From d85928448558f36ecf9541cb99360a33573516c5 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Mon, 1 Jun 2026 11:06:05 -0700 Subject: [PATCH 01/30] chore(windows): gate Unix-only workspace code for MSVC Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- Cargo.lock | 73 + Cargo.toml | 1 + crates/openshell-cli/src/run.rs | 6 +- .../sandbox_create_lifecycle_integration.rs | 2 + crates/openshell-core/Cargo.toml | 5 + crates/openshell-core/build.rs | 35 +- crates/openshell-core/src/forward.rs | 1 + crates/openshell-core/src/paths.rs | 12 + crates/openshell-prover/Cargo.toml | 5 +- crates/openshell-prover/src/lib.rs | 299 +- crates/openshell-prover/src/lib_unix.rs | 293 ++ crates/openshell-sandbox/Cargo.toml | 9 +- crates/openshell-sandbox/src/lib.rs | 3773 +---------------- crates/openshell-sandbox/src/lib_unix.rs | 3773 +++++++++++++++++ crates/openshell-sandbox/src/main.rs | 750 +--- crates/openshell-sandbox/src/main_unix.rs | 749 ++++ .../openshell-sandbox/tests/stdout_logging.rs | 2 + crates/openshell-server/src/grpc/policy.rs | 71 +- .../tests/system_inference.rs | 2 + .../tests/websocket_upgrade.rs | 1 + crates/openshell-vfio/src/bind.rs | 6 +- crates/openshell-vfio/src/gpu.rs | 2 +- crates/openshell-vfio/src/lib.rs | 2 +- crates/openshell-vfio/src/pci.rs | 2 +- crates/openshell-vfio/src/reconcile.rs | 2 +- crates/openshell-vfio/src/sysfs.rs | 4 +- 26 files changed, 5047 insertions(+), 4833 deletions(-) create mode 100644 crates/openshell-prover/src/lib_unix.rs create mode 100644 crates/openshell-sandbox/src/lib_unix.rs create mode 100644 crates/openshell-sandbox/src/main_unix.rs diff --git a/Cargo.lock b/Cargo.lock index 31e2104987..803b544b3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3894,6 +3894,7 @@ dependencies = [ "prost", "prost-types", "protobuf-src", + "protoc-bin-vendored", "reqwest 0.12.28", "serde", "serde_json", @@ -4106,8 +4107,11 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "anyhow", "clap", "futures", + "hex", + "hmac", "miette", "nix", "openshell-core", @@ -4119,11 +4123,15 @@ dependencies = [ "openshell-supervisor-process", "prost", "prost-types", + "rand_core 0.6.4", + "russh", "rustls 0.23.38", "serde", "serde_json", + "sha2 0.10.9", "temp-env", "tempfile", + "thiserror 2.0.18", "tokio", "tokio-tungstenite 0.26.2", "tonic", @@ -4947,6 +4955,70 @@ dependencies = [ "autotools", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "pulldown-cmark" version = "0.13.3" @@ -5450,6 +5522,7 @@ dependencies = [ "pkcs8 0.10.2", "rand 0.9.4", "rand_core 0.10.0-rc-3", + "ring", "rsa 0.10.0-rc.12", "russh-cryptovec", "russh-util", diff --git a/Cargo.toml b/Cargo.toml index 4ec6a0d44f..40bc57ee67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ tonic-prost-build = "0.14" prost = "0.14" prost-types = "0.14" prost-reflect = { version = "0.16.5", features = ["serde"] } +protoc-bin-vendored = "3.2.0" # HTTP server axum = { version = "0.8", features = ["ws"] } diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 4843475e5d..834e9cd7fb 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1752,8 +1752,10 @@ impl Drop for RawModeGuard { } } +#[cfg(unix)] struct TaskGuard(tokio::task::JoinHandle<()>); +#[cfg(unix)] impl Drop for TaskGuard { fn drop(&mut self) { self.0.abort(); @@ -1768,7 +1770,9 @@ async fn sandbox_exec_interactive_grpc( timeout_seconds: u32, environment: &HashMap, ) -> Result { - use openshell_core::proto::{ExecSandboxInput, ExecSandboxWindowResize, exec_sandbox_input}; + #[cfg(unix)] + use openshell_core::proto::ExecSandboxWindowResize; + use openshell_core::proto::{ExecSandboxInput, exec_sandbox_input}; use tokio_stream::wrappers::ReceiverStream; let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24)); diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 7ed148304c..a1979a9022 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![cfg(not(target_os = "windows"))] + mod helpers; use helpers::{ diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index 35a3732cf9..1f7cbf60cb 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -40,8 +40,13 @@ telemetry = ["dep:reqwest", "dep:chrono"] [build-dependencies] tonic-prost-build = { workspace = true } + +[target.'cfg(not(target_os = "windows"))'.build-dependencies] protobuf-src = { workspace = true } +[target.'cfg(target_os = "windows")'.build-dependencies] +protoc-bin-vendored = { workspace = true } + [dev-dependencies] tempfile = "3" diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 7955772a67..26cbedbca3 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -22,19 +22,23 @@ fn main() -> Result<(), Box> { // --- Protobuf compilation --- // Re-run when anything under proto/ changes (including newly added .proto files). println!("cargo:rerun-if-changed={PROTO_REL}"); - // Use bundled protoc from protobuf-src. The system protoc (from apt-get) - // does not bundle the well-known type includes (google/protobuf/struct.proto - // etc.), so we must use protobuf-src which ships both the binary and the - // include tree. + // Use a bundled protoc. The system protoc (from apt-get) does not bundle + // the well-known type includes (google/protobuf/struct.proto etc.), so the + // build script picks a vendored provider per platform. // SAFETY: This is run at build time in a single-threaded build script context. // No other threads are reading environment variables concurrently. #[allow(unsafe_code)] unsafe { - env::set_var("PROTOC", protobuf_src::protoc()); + env::set_var("PROTOC", protoc_path()?); } let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); + let proto_includes = proto_include_dirs(&proto_root)?; + let proto_include_refs = proto_includes + .iter() + .map(PathBuf::as_path) + .collect::>(); let mut proto_files = Vec::new(); collect_proto_files(&proto_root, &mut proto_files)?; @@ -50,7 +54,7 @@ fn main() -> Result<(), Box> { // Emit a binary FileDescriptorSet so the server can enumerate every // RPC at runtime (used by the per-handler auth exhaustiveness test). .file_descriptor_set_path(&descriptor_path) - .compile_protos(&proto_files, &[proto_root])?; + .compile_protos(&proto_files, &proto_include_refs)?; println!( "cargo:rustc-env=OPENSHELL_DESCRIPTOR_PATH={}", @@ -60,6 +64,25 @@ fn main() -> Result<(), Box> { Ok(()) } +#[cfg(target_os = "windows")] +fn protoc_path() -> Result> { + Ok(protoc_bin_vendored::protoc_bin_path()?) +} + +#[cfg(not(target_os = "windows"))] +fn protoc_path() -> Result> { + Ok(protobuf_src::protoc()) +} + +fn proto_include_dirs(proto_root: &Path) -> Result, Box> { + let mut includes = vec![proto_root.to_path_buf()]; + #[cfg(target_os = "windows")] + { + includes.push(protoc_bin_vendored::include_path()?); + } + Ok(includes) +} + fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let path = entry?.path(); diff --git a/crates/openshell-core/src/forward.rs b/crates/openshell-core/src/forward.rs index 70ab74edd0..b32dc305d4 100644 --- a/crates/openshell-core/src/forward.rs +++ b/crates/openshell-core/src/forward.rs @@ -1341,6 +1341,7 @@ mod tests { ); } + #[cfg(not(target_os = "windows"))] #[test] fn check_port_available_occupied_ipv6_wildcard() { // Bind on [::]:0 (IPv6 wildcard) — this simulates a server like diff --git a/crates/openshell-core/src/paths.rs b/crates/openshell-core/src/paths.rs index 9445347c79..c1c327122e 100644 --- a/crates/openshell-core/src/paths.rs +++ b/crates/openshell-core/src/paths.rs @@ -20,6 +20,10 @@ pub fn xdg_config_dir() -> Result { if let Ok(path) = std::env::var("XDG_CONFIG_HOME") { return Ok(PathBuf::from(path)); } + #[cfg(target_os = "windows")] + if let Ok(path) = std::env::var("APPDATA") { + return Ok(PathBuf::from(path)); + } let home = std::env::var("HOME") .into_diagnostic() .wrap_err("HOME is not set")?; @@ -38,6 +42,10 @@ pub fn xdg_state_dir() -> Result { if let Ok(path) = std::env::var("XDG_STATE_HOME") { return Ok(PathBuf::from(path)); } + #[cfg(target_os = "windows")] + if let Ok(path) = std::env::var("LOCALAPPDATA") { + return Ok(PathBuf::from(path)); + } let home = std::env::var("HOME") .into_diagnostic() .wrap_err("HOME is not set")?; @@ -56,6 +64,10 @@ pub fn xdg_data_dir() -> Result { if let Ok(path) = std::env::var("XDG_DATA_HOME") { return Ok(PathBuf::from(path)); } + #[cfg(target_os = "windows")] + if let Ok(path) = std::env::var("LOCALAPPDATA") { + return Ok(PathBuf::from(path)); + } let home = std::env::var("HOME") .into_diagnostic() .wrap_err("HOME is not set")?; diff --git a/crates/openshell-prover/Cargo.toml b/crates/openshell-prover/Cargo.toml index ee815f3a3f..3175acf36c 100644 --- a/crates/openshell-prover/Cargo.toml +++ b/crates/openshell-prover/Cargo.toml @@ -13,7 +13,10 @@ repository.workspace = true [features] bundled-z3 = ["z3/bundled"] -[dependencies] +[target.'cfg(target_os = "windows")'.dependencies] +miette = { workspace = true } + +[target.'cfg(not(target_os = "windows"))'.dependencies] z3 = { workspace = true } serde = { workspace = true } serde_yml = { workspace = true } diff --git a/crates/openshell-prover/src/lib.rs b/crates/openshell-prover/src/lib.rs index 0fb8757577..15e4f633f6 100644 --- a/crates/openshell-prover/src/lib.rs +++ b/crates/openshell-prover/src/lib.rs @@ -1,293 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Formal policy verification for `OpenShell` sandboxes. -//! -//! Encodes sandbox policies, binary capabilities, and credential scopes as Z3 -//! SMT constraints, then checks reachability queries to detect data exfiltration -//! paths and write-bypass violations. +#[cfg(not(target_os = "windows"))] +include!("lib_unix.rs"); -pub mod accepted_risks; -pub mod credentials; -pub mod finding; -pub mod model; -pub mod policy; -pub mod queries; -pub mod registry; -pub mod report; - -use std::path::Path; - -use miette::Result; - -use accepted_risks::{apply_accepted_risks, load_accepted_risks}; -use model::build_model; -use policy::parse_policy; -use queries::run_all_queries; -use report::{render_compact, render_report}; - -/// Run the prover end-to-end and return a result containing an exit code. -/// -/// - `Ok(0)` — pass (no findings, or all accepted) -/// - `Ok(1)` — fail (one or more unaccepted findings present) -/// - `Err(_)` — input or registry loading error -/// -/// Binary and API capability registries are embedded at compile time. -/// Pass `registry_dir` to override with a custom filesystem registry. +#[cfg(target_os = "windows")] pub fn prove( - policy_path: &str, - credentials_path: &str, - registry_dir: Option<&str>, - accepted_risks_path: Option<&str>, - compact: bool, -) -> Result { - let policy = parse_policy(Path::new(policy_path))?; - - let (credential_set, binary_registry) = match registry_dir { - Some(dir) => { - let dir = Path::new(dir); - ( - credentials::load_credential_set_from_dir(Path::new(credentials_path), dir)?, - registry::load_binary_registry_from_dir(dir)?, - ) - } - None => ( - credentials::load_credential_set_embedded(Path::new(credentials_path))?, - registry::load_embedded_binary_registry()?, - ), - }; - - let z3_model = build_model(policy, credential_set, binary_registry); - let mut findings = run_all_queries(&z3_model); - - if let Some(ar_path) = accepted_risks_path { - let accepted = load_accepted_risks(Path::new(ar_path))?; - findings = apply_accepted_risks(findings, &accepted); - } - - let exit_code = if compact { - render_compact(&findings, policy_path, credentials_path) - } else { - render_report(&findings, policy_path, credentials_path) - }; - - Ok(exit_code) -} - -// =========================================================================== -// Tests -// =========================================================================== - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - fn testdata_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") - } - - // 1. Parse testdata/policy.yaml, verify structure. - #[test] - fn test_parse_policy() { - let path = testdata_dir().join("policy.yaml"); - let model = parse_policy(&path).expect("failed to parse policy"); - assert_eq!(model.version, 1); - assert!(model.network_policies.contains_key("github_api")); - let rule = &model.network_policies["github_api"]; - assert_eq!(rule.name, "github-api"); - assert_eq!(rule.endpoints.len(), 2); - assert!(rule.binaries.len() >= 4); - } - - // 2. Verify readable_paths. - #[test] - fn test_filesystem_policy() { - let path = testdata_dir().join("policy.yaml"); - let model = parse_policy(&path).expect("failed to parse policy"); - let readable = model.filesystem_policy.readable_paths(); - assert!(readable.contains(&"/usr".to_owned())); - assert!(readable.contains(&"/sandbox".to_owned())); - assert!(readable.contains(&"/tmp".to_owned())); - } - - // 3. Workdir NOT included by default (matches runtime behavior). - #[test] - fn test_include_workdir_default() { - let yaml = r" -version: 1 -filesystem_policy: - read_only: - - /usr -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); - assert!(!readable.contains(&"/sandbox".to_owned())); - } - - // 4. Workdir excluded when include_workdir: false. - #[test] - fn test_include_workdir_false() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); - assert!(!readable.contains(&"/sandbox".to_owned())); - } - - // 5. No duplicate when workdir already in read_write. - #[test] - fn test_include_workdir_no_duplicate() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: true - read_write: - - /sandbox - - /tmp -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); - let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); - assert_eq!(sandbox_count, 1); - } - - // 6. End-to-end: testdata policy with a github credential in scope and a - // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. - // The prover output is categorical, not severity-graded. - #[test] - fn test_findings_for_github_policy() { - use finding::category; - - let policy_path = testdata_dir().join("policy.yaml"); - let creds_path = testdata_dir().join("credentials.yaml"); - - let pol = parse_policy(&policy_path).expect("parse policy"); - let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(pol, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - let categories: std::collections::HashSet<&str> = - findings.iter().map(|f| f.query.as_str()).collect(); - assert!( - categories.contains(category::L7_BYPASS_CREDENTIALED), - "expected l7_bypass_credentialed finding for bypass-L7 binary with credential in scope; \ - got categories: {categories:?}" - ); - // Every emitted category must be one of the four v1 categories. - let allowed: std::collections::HashSet<&str> = [ - category::LINK_LOCAL_REACH, - category::L7_BYPASS_CREDENTIALED, - category::CREDENTIAL_REACH_EXPANSION, - category::CAPABILITY_EXPANSION, - ] - .into_iter() - .collect(); - for c in &categories { - assert!( - allowed.contains(*c), - "unexpected category {c} emitted by the prover" - ); - } - } - - #[test] - fn test_wildcard_endpoint_covering_credential_host_emits_credential_reach() { - use finding::{FindingPath, category}; - - let policy = policy::parse_policy_str( - r#" -version: 1 -network_policies: - github_wildcard: - name: github-wildcard - endpoints: - - host: "*.github.com" - port: 443 - protocol: rest - enforcement: enforce - access: read-write - binaries: - - path: /usr/bin/curl -"#, - ) - .expect("parse policy"); - let cred_set = - credentials::load_credential_set_embedded(&testdata_dir().join("credentials.yaml")) - .expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(policy, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - let reach = findings - .iter() - .find(|finding| finding.query == category::CREDENTIAL_REACH_EXPANSION) - .expect("wildcard host covering api.github.com must be credentialed"); - assert!(reach.paths.iter().any(|path| matches!( - path, - FindingPath::Exfil(exfil) - if exfil.endpoint_host == "*.github.com" && exfil.binary == "/usr/bin/curl" - ))); - } - - #[test] - fn test_known_metadata_hostname_emits_link_local_finding() { - use finding::{FindingPath, category}; - - let policy = policy::parse_policy_str( - r" -version: 1 -network_policies: - metadata: - name: metadata - endpoints: - - host: metadata.google.internal - port: 80 - binaries: - - path: /usr/bin/curl -", - ) - .expect("parse policy"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(policy, credentials::CredentialSet::default(), bin_reg); - let findings = run_all_queries(&z3_model); - - let link_local = findings - .iter() - .find(|finding| finding.query == category::LINK_LOCAL_REACH) - .expect("known metadata hostname must emit link-local/metadata finding"); - assert!(link_local.paths.iter().any(|path| matches!( - path, - FindingPath::Exfil(exfil) - if exfil.endpoint_host == "metadata.google.internal" - ))); - } - - // 7. Empty policy produces no findings. - #[test] - fn test_empty_policy_no_findings() { - let policy_path = testdata_dir().join("empty-policy.yaml"); - let creds_path = testdata_dir().join("credentials.yaml"); - - let pol = parse_policy(&policy_path).expect("parse policy"); - let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(pol, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - assert!( - findings.is_empty(), - "deny-all policy should produce no findings, got: {findings:?}" - ); - } + _policy_path: &str, + _credentials_path: &str, + _registry_dir: Option<&str>, + _accepted_risks_path: Option<&str>, + _compact: bool, +) -> miette::Result { + Err(miette::miette!( + "policy prover is not available on Windows in this build" + )) } diff --git a/crates/openshell-prover/src/lib_unix.rs b/crates/openshell-prover/src/lib_unix.rs new file mode 100644 index 0000000000..0fb8757577 --- /dev/null +++ b/crates/openshell-prover/src/lib_unix.rs @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Formal policy verification for `OpenShell` sandboxes. +//! +//! Encodes sandbox policies, binary capabilities, and credential scopes as Z3 +//! SMT constraints, then checks reachability queries to detect data exfiltration +//! paths and write-bypass violations. + +pub mod accepted_risks; +pub mod credentials; +pub mod finding; +pub mod model; +pub mod policy; +pub mod queries; +pub mod registry; +pub mod report; + +use std::path::Path; + +use miette::Result; + +use accepted_risks::{apply_accepted_risks, load_accepted_risks}; +use model::build_model; +use policy::parse_policy; +use queries::run_all_queries; +use report::{render_compact, render_report}; + +/// Run the prover end-to-end and return a result containing an exit code. +/// +/// - `Ok(0)` — pass (no findings, or all accepted) +/// - `Ok(1)` — fail (one or more unaccepted findings present) +/// - `Err(_)` — input or registry loading error +/// +/// Binary and API capability registries are embedded at compile time. +/// Pass `registry_dir` to override with a custom filesystem registry. +pub fn prove( + policy_path: &str, + credentials_path: &str, + registry_dir: Option<&str>, + accepted_risks_path: Option<&str>, + compact: bool, +) -> Result { + let policy = parse_policy(Path::new(policy_path))?; + + let (credential_set, binary_registry) = match registry_dir { + Some(dir) => { + let dir = Path::new(dir); + ( + credentials::load_credential_set_from_dir(Path::new(credentials_path), dir)?, + registry::load_binary_registry_from_dir(dir)?, + ) + } + None => ( + credentials::load_credential_set_embedded(Path::new(credentials_path))?, + registry::load_embedded_binary_registry()?, + ), + }; + + let z3_model = build_model(policy, credential_set, binary_registry); + let mut findings = run_all_queries(&z3_model); + + if let Some(ar_path) = accepted_risks_path { + let accepted = load_accepted_risks(Path::new(ar_path))?; + findings = apply_accepted_risks(findings, &accepted); + } + + let exit_code = if compact { + render_compact(&findings, policy_path, credentials_path) + } else { + render_report(&findings, policy_path, credentials_path) + }; + + Ok(exit_code) +} + +// =========================================================================== +// Tests +// =========================================================================== + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn testdata_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") + } + + // 1. Parse testdata/policy.yaml, verify structure. + #[test] + fn test_parse_policy() { + let path = testdata_dir().join("policy.yaml"); + let model = parse_policy(&path).expect("failed to parse policy"); + assert_eq!(model.version, 1); + assert!(model.network_policies.contains_key("github_api")); + let rule = &model.network_policies["github_api"]; + assert_eq!(rule.name, "github-api"); + assert_eq!(rule.endpoints.len(), 2); + assert!(rule.binaries.len() >= 4); + } + + // 2. Verify readable_paths. + #[test] + fn test_filesystem_policy() { + let path = testdata_dir().join("policy.yaml"); + let model = parse_policy(&path).expect("failed to parse policy"); + let readable = model.filesystem_policy.readable_paths(); + assert!(readable.contains(&"/usr".to_owned())); + assert!(readable.contains(&"/sandbox".to_owned())); + assert!(readable.contains(&"/tmp".to_owned())); + } + + // 3. Workdir NOT included by default (matches runtime behavior). + #[test] + fn test_include_workdir_default() { + let yaml = r" +version: 1 +filesystem_policy: + read_only: + - /usr +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model.filesystem_policy.readable_paths(); + assert!(!readable.contains(&"/sandbox".to_owned())); + } + + // 4. Workdir excluded when include_workdir: false. + #[test] + fn test_include_workdir_false() { + let yaml = r" +version: 1 +filesystem_policy: + include_workdir: false + read_only: + - /usr +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model.filesystem_policy.readable_paths(); + assert!(!readable.contains(&"/sandbox".to_owned())); + } + + // 5. No duplicate when workdir already in read_write. + #[test] + fn test_include_workdir_no_duplicate() { + let yaml = r" +version: 1 +filesystem_policy: + include_workdir: true + read_write: + - /sandbox + - /tmp +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model.filesystem_policy.readable_paths(); + let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); + assert_eq!(sandbox_count, 1); + } + + // 6. End-to-end: testdata policy with a github credential in scope and a + // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. + // The prover output is categorical, not severity-graded. + #[test] + fn test_findings_for_github_policy() { + use finding::category; + + let policy_path = testdata_dir().join("policy.yaml"); + let creds_path = testdata_dir().join("credentials.yaml"); + + let pol = parse_policy(&policy_path).expect("parse policy"); + let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(pol, cred_set, bin_reg); + let findings = run_all_queries(&z3_model); + + let categories: std::collections::HashSet<&str> = + findings.iter().map(|f| f.query.as_str()).collect(); + assert!( + categories.contains(category::L7_BYPASS_CREDENTIALED), + "expected l7_bypass_credentialed finding for bypass-L7 binary with credential in scope; \ + got categories: {categories:?}" + ); + // Every emitted category must be one of the four v1 categories. + let allowed: std::collections::HashSet<&str> = [ + category::LINK_LOCAL_REACH, + category::L7_BYPASS_CREDENTIALED, + category::CREDENTIAL_REACH_EXPANSION, + category::CAPABILITY_EXPANSION, + ] + .into_iter() + .collect(); + for c in &categories { + assert!( + allowed.contains(*c), + "unexpected category {c} emitted by the prover" + ); + } + } + + #[test] + fn test_wildcard_endpoint_covering_credential_host_emits_credential_reach() { + use finding::{FindingPath, category}; + + let policy = policy::parse_policy_str( + r#" +version: 1 +network_policies: + github_wildcard: + name: github-wildcard + endpoints: + - host: "*.github.com" + port: 443 + protocol: rest + enforcement: enforce + access: read-write + binaries: + - path: /usr/bin/curl +"#, + ) + .expect("parse policy"); + let cred_set = + credentials::load_credential_set_embedded(&testdata_dir().join("credentials.yaml")) + .expect("load creds"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(policy, cred_set, bin_reg); + let findings = run_all_queries(&z3_model); + + let reach = findings + .iter() + .find(|finding| finding.query == category::CREDENTIAL_REACH_EXPANSION) + .expect("wildcard host covering api.github.com must be credentialed"); + assert!(reach.paths.iter().any(|path| matches!( + path, + FindingPath::Exfil(exfil) + if exfil.endpoint_host == "*.github.com" && exfil.binary == "/usr/bin/curl" + ))); + } + + #[test] + fn test_known_metadata_hostname_emits_link_local_finding() { + use finding::{FindingPath, category}; + + let policy = policy::parse_policy_str( + r" +version: 1 +network_policies: + metadata: + name: metadata + endpoints: + - host: metadata.google.internal + port: 80 + binaries: + - path: /usr/bin/curl +", + ) + .expect("parse policy"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(policy, credentials::CredentialSet::default(), bin_reg); + let findings = run_all_queries(&z3_model); + + let link_local = findings + .iter() + .find(|finding| finding.query == category::LINK_LOCAL_REACH) + .expect("known metadata hostname must emit link-local/metadata finding"); + assert!(link_local.paths.iter().any(|path| matches!( + path, + FindingPath::Exfil(exfil) + if exfil.endpoint_host == "metadata.google.internal" + ))); + } + + // 7. Empty policy produces no findings. + #[test] + fn test_empty_policy_no_findings() { + let policy_path = testdata_dir().join("empty-policy.yaml"); + let creds_path = testdata_dir().join("credentials.yaml"); + + let pol = parse_policy(&policy_path).expect("parse policy"); + let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(pol, cred_set, bin_reg); + let findings = run_all_queries(&z3_model); + + assert!( + findings.is_empty(), + "deny-all policy should produce no findings, got: {findings:?}" + ); + } +} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 6a51635b14..3fff53439c 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -14,7 +14,7 @@ repository.workspace = true name = "openshell-sandbox" path = "src/main.rs" -[dependencies] +[target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } @@ -35,6 +35,13 @@ clap = { workspace = true } # Error handling miette = { workspace = true } +thiserror = { workspace = true } +anyhow = { workspace = true } +hmac = "0.12" +sha2 = { workspace = true } +hex = "0.4" +russh = { version = "0.57", default-features = false, features = ["flate2", "ring", "rsa"] } +rand_core = "0.6" # Unix ownership for Kubernetes sidecar init setup nix = { workspace = true } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index e696403f39..4d431aa1e6 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1,3773 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Sandbox library. -//! -//! This crate provides process sandboxing and monitoring capabilities. - -mod activity_aggregator; -mod denial_aggregator; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod google_cloud_metadata; -mod mechanistic_mapper; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod metadata_server; -mod sidecar_control; - -use miette::{IntoDiagnostic, Result, WrapErr}; -use std::future::Future; -use std::sync::Arc; -#[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicBool, AtomicU32}; -use std::time::Duration; -use tracing::{debug, info, warn}; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, - DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, -}; - -// --------------------------------------------------------------------------- -// OCSF Context -// --------------------------------------------------------------------------- -// -// The following log sites intentionally remain as plain `tracing` macros -// and are NOT migrated to OCSF builders: -// -// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) -// - Transient "about to do X" events where the result is logged separately -// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") -// - Internal SSH channel warnings (unknown channel, PTY resize failures) -// - Denial flush telemetry (the individual denials are already OCSF events) -// - Status reporting failures (sync to gateway, non-actionable) -// - Route refresh interval validation warnings -// -// These are operational plumbing that don't represent security decisions, -// policy changes, or observable sandbox behavior worth structuring. -// --------------------------------------------------------------------------- - -/// Re-export the process-wide OCSF sandbox context getter. -/// -/// The singleton lives in `openshell-ocsf` so both supervisor leaves can -/// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. -pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; - -use openshell_core::denial::DenialEvent; -use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_supervisor_network::opa::OpaEngine; -use openshell_supervisor_process::process::ProcessEnforcementMode; -pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; -use openshell_supervisor_process::skills; -use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] -use tokio::time::timeout; - -const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; -const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; -const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; -const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; -const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; -const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; - -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, - inference_routes: Option, - ocsf_enabled: Arc, - network_enabled: bool, - process_enabled: bool, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, -) -> Result { - let (program, args) = command - .split_first() - .ok_or_else(|| miette::miette!("No command specified"))?; - - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; they are only significant for network - // events which happen after the netns is created. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(SandboxContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); - let process_uses_sidecar_control = - process_enabled && !network_enabled && sidecar_network_enforcement; - let mut process_control_connection = None; - let sidecar_bootstrap = if process_uses_sidecar_control { - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let (bootstrap, connection) = sidecar_control::connect_process_client( - &socket, - Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), - ) - .await?; - process_control_connection = Some(connection); - Some(bootstrap) - } else { - None - }; - - // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); - let sandbox_name_for_agg = sandbox.clone(); - let ( - mut policy, - opa_engine, - retained_proto, - middleware_registry_status, - loaded_policy_origin, - initial_agent_proposals_enabled, - ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let (policy, opa_engine, retained_proto, loaded_policy_origin) = - load_policy_from_sidecar_bootstrap(bootstrap)?; - ( - policy, - opa_engine, - retained_proto, - MiddlewareRegistryStatus::Synchronized, - loaded_policy_origin, - bootstrap.agent_proposals_enabled, - ) - } else { - load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - ) - .await? - }; - - // Override the policy's process identity with the driver-resolved UID/GID - // from the pod environment. The policy defaults to the name "sandbox" which - // resolves via /etc/passwd, but the driver may have chosen a different - // numeric UID (e.g. from OpenShift SCC annotations). - // Validate overrides against the same rules as the policy layer to prevent - // env-injected values (e.g. GID 0) from bypassing policy restrictions. - if let Ok(uid) = std::env::var(openshell_core::sandbox_env::SANDBOX_UID) - && !uid.is_empty() - { - if !openshell_policy::is_valid_sandbox_identity(&uid) { - return Err(miette::miette!( - "OPENSHELL_SANDBOX_UID contains invalid sandbox identity '{uid}'; \ - expected 'sandbox' or a numeric UID in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - policy.process.run_as_user = Some(uid); - } - if let Ok(gid) = std::env::var(openshell_core::sandbox_env::SANDBOX_GID) - && !gid.is_empty() - { - if !openshell_policy::is_valid_sandbox_identity(&gid) { - return Err(miette::miette!( - "OPENSHELL_SANDBOX_GID contains invalid sandbox identity '{gid}'; \ - expected 'sandbox' or a numeric GID in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - policy.process.run_as_group = Some(gid); - } - - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = - if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - bootstrap.provider_env_revision, - bootstrap.provider_child_env.clone(), - ); - (provider_credentials, bootstrap.provider_child_env.clone()) - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .message(format!( - "Failed to fetch provider environment, continuing without: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - } - } - } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - }; - - let provider_credentials = ProviderCredentialState::from_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ); - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - - // Shared agent-proposals feature flag. Seed from the same initial settings - // snapshot that produced the policy so networking and process setup agree - // before the poll loop starts reconciling later changes. - let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); - - let process_control_writer = process_control_connection - .as_ref() - .map(|connection| connection.writer.clone()); - let mut process_control_closed = None; - if let Some(connection) = process_control_connection { - process_control_closed = Some(connection.closed); - spawn_sidecar_control_update_watcher( - connection.updates, - provider_credentials.clone(), - agent_proposals.clone(), - ); - } - - // Shared PID: set after process spawn so the proxy can look up - // the entrypoint process's /proc/net/tcp for identity binding. - let entrypoint_pid = Arc::new(AtomicU32::new(0)); - - // Create the workload's network namespace. It is shared infrastructure: - // the proxy binds to its host-side veth IP, the bypass monitor reads - // /dev/kmsg from inside it, and the workload child / SSH sessions enter - // it via setns(). The RAII handle lives in this frame for the duration - // of the sandbox. - #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { - openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? - } else { - None - }; - - // The denial channel is owned by the orchestrator: the proxy (in the - // networking leaf) and the bypass monitor (in the process leaf) both - // produce DenialEvents that the denial aggregator (orchestrator-side) - // consumes via the matching receiver. Both leaves are pure producers; - // the orchestrator owns the consumer task spawned below. - let (denial_tx, denial_rx, bypass_denial_tx): ( - Option>, - _, - Option>, - ) = if sandbox_id.is_some() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_denial_tx); - - // Anonymous activity channel: same orchestrator-owned pattern as the - // denial channel. The proxy and the bypass monitor both emit per-event - // activity records; the orchestrator-side aggregator drains, sanitizes, - // and flushes anonymous summaries to the gateway. - let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { - let (tx, rx) = - tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_activity_tx); - - // Workspace watch: the policy poll loop learns the workspace from - // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local - // API read the current value so proposals target the correct workspace. - let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - - let networking = if network_enabled { - #[cfg(target_os = "linux")] - let proxy_bind_ip = netns - .as_ref() - .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); - #[cfg(not(target_os = "linux"))] - let proxy_bind_ip: Option = None; - - Some( - openshell_supervisor_network::run::run_networking( - &policy, - proxy_bind_ip, - opa_engine.as_ref(), - retained_proto.as_ref(), - entrypoint_pid.clone(), - process_enabled, - &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), - inference_routes.as_deref(), - denial_tx, - activity_tx, - agent_proposals.clone(), - workspace_rx.clone(), - &upstream_proxy_args, - ) - .await?, - ) - } else { - None - }; - - #[cfg(target_os = "linux")] - let sidecar_control_server = if network_enabled && sidecar_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" - )); - } - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let proto = retained_proto.as_ref().ok_or_else(|| { - miette::miette!( - "sidecar topology requires gateway policy data for the process supervisor" - ) - })?; - let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); - Some(sidecar_control::spawn_server( - &socket, - sidecar_control::BootstrapData { - policy_proto: proto.clone(), - provider_env_revision: provider_credentials.snapshot().revision, - provider_child_env: provider_env.clone(), - agent_proposals_enabled: agent_proposals.enabled(), - proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), - proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), - }, - sidecar_expected_peer()?, - )?) - } else { - None - }; - #[cfg(not(target_os = "linux"))] - let sidecar_control_server: Option = None; - - let sidecar_control_publisher = sidecar_control_server - .as_ref() - .map(sidecar_control::ServerHandle::publisher); - - #[cfg(target_os = "linux")] - let mut sidecar_control_task = None; - - #[cfg(target_os = "linux")] - if network_enabled - && sidecar_network_enforcement - && let Some(server) = sidecar_control_server - { - let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar network topology", - openshell_core::sandbox_env::SSH_SOCKET_PATH - ) - })?; - let (entrypoint_rx, connection_task) = server.into_runtime_parts(); - sidecar_control_task = Some(connection_task); - spawn_sidecar_entrypoint_handler( - entrypoint_rx, - entrypoint_pid.clone(), - opa_engine.clone(), - retained_proto.clone(), - openshell_endpoint.clone(), - sandbox_id.clone(), - std::path::PathBuf::from(trusted_ssh_socket_path), - ); - } - - #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { - return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" - )); - } - - // Spawn the denial-aggregator flush task. The aggregator drains denial - // events from the proxy + bypass monitor, batches them, and ships - // summaries to the gateway via `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { - // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall - // back to the ID when the name isn't set. - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - - let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); - let denial_workspace_gate = workspace_rx.clone(); - let denial_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - |summaries| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = denial_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_proposals_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summaries, - ) - .await - { - warn!(error = %e, "Failed to flush denial summaries to gateway"); - } - } - }, - move || !denial_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn the activity-aggregator flush task. The aggregator drains - // anonymous activity events from the proxy, sanitizes deny groups, - // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( - std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") - .ok() - .as_deref(), - ); - - let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); - let activity_workspace_gate = workspace_rx.clone(); - let activity_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - move |summary| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = activity_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_activity_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summary, - ) - .await - { - warn!(error = %e, "Failed to flush activity summary to gateway"); - } - } - }, - move || !activity_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn background policy poll task (gRPC mode only). - if !process_uses_sidecar_control - && let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) - { - let poll_id = id.to_string(); - let poll_endpoint = endpoint.to_string(); - let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_pid = entrypoint_pid.clone(); - let poll_provider_credentials = provider_credentials.clone(); - let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); - let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, - sandbox_id: poll_id, - opa_engine: poll_engine, - loaded_policy_origin, - entrypoint_pid: poll_pid, - interval_secs: poll_interval_secs, - ocsf_enabled: poll_ocsf_enabled, - provider_credentials: poll_provider_credentials, - policy_local_ctx: poll_policy_local, - agent_proposals: agent_proposals.clone(), - middleware_registry_status, - sidecar_control_publisher: sidecar_control_publisher.clone(), - workspace_tx, - }; - - tokio::spawn(async move { - if let Err(e) = run_policy_poll_loop(poll_ctx).await { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .message(format!("Policy poll loop exited with error: {e}")) - .build() - ); - } - }); - } - - // Start GCE metadata loopback server inside the network namespace so - // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) - // can reach it via direct TCP. Must start before the process leaf so SSH - // sessions also see corrected env vars on bind failure. - #[cfg(target_os = "linux")] - if let Some(ns) = netns.as_ref() - && provider_credentials - .snapshot() - .child_env - .contains_key("GCE_METADATA_HOST") - { - let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); - let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); - match ns - .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) - .await - { - Ok(listener) => { - tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); - if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { - info!(addr = %addr, "GCE metadata loopback server ready"); - } else { - warn!("GCE metadata server failed to become ready, removing metadata env vars"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - Err(e) => { - warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - } - - let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { - bootstrap - .proxy_ca_cert_path - .clone() - .zip(bootstrap.proxy_ca_bundle_path.clone()) - }); - - let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None - } - }); - - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok(pid) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid).await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); - } - } - }); - Some(tx) - } else { - None - }; - - let process = openshell_supervisor_process::run::run_process( - program, - args, - workdir.as_deref(), - timeout_secs, - interactive, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - &process_policy, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - provider_credentials, - provider_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); - - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - } - } else { - process.await? - } - } else { - // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until shutdown. If the - // sole authenticated process-supervisor control connection closes, - // exit non-zero so Kubernetes restarts the network sidecar and creates - // a fresh one-client bootstrap listener for the restarted agent. - #[cfg(target_os = "linux")] - if let Some(control_task) = sidecar_control_task { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - result = control_task => { - warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); - 1 - } - } - } else { - wait_for_shutdown_signal().await; - 0 - } - #[cfg(not(target_os = "linux"))] - { - wait_for_shutdown_signal().await; - 0 - } - }; - - // Drop networking explicitly so the proxy + bypass monitor RAII - // handles tear down before we return. - drop(networking); - - Ok(exit_code) -} - -/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is -/// no entrypoint child whose lifetime drives the supervisor's exit. -async fn wait_for_shutdown_signal() { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(s) => s, - Err(e) => { - tracing::warn!( - error = %e, - "Failed to install SIGTERM handler; waiting on SIGINT only" - ); - let _ = tokio::signal::ctrl_c().await; - return; - } - }; - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Received SIGINT, shutting down network-only supervisor"); - } - _ = sigterm.recv() => { - info!("Received SIGTERM, shutting down network-only supervisor"); - } - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - info!("Received Ctrl-C, shutting down network-only supervisor"); - } -} - -fn sidecar_network_enforcement_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) - .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) -} - -fn process_enforcement_mode() -> ProcessEnforcementMode { - match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .ok() - .as_deref() - { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, - _ => ProcessEnforcementMode::Full, - } -} - -fn sidecar_control_socket() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) - .ok() - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn sidecar_expected_peer() -> Result { - fn required_numeric_env(name: &str) -> Result { - let value = std::env::var(name) - .into_diagnostic() - .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; - value.parse::().into_diagnostic().wrap_err_with(|| { - format!("{name} must be a numeric ID for sidecar control authentication") - }) - } - - Ok(sidecar_control::ExpectedPeer { - uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, - gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, - }) -} - -type LoadedPolicyBundle = ( - SandboxPolicy, - Option>, - Option, - LoadedPolicyOrigin, -); - -fn load_policy_from_sidecar_bootstrap( - bootstrap: &sidecar_control::BootstrapData, -) -> Result { - let proto = bootstrap.policy_proto.clone(); - let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); - let policy = SandboxPolicy::try_from(proto.clone())?; - info!("Loaded sidecar policy from control socket bootstrap"); - Ok(( - policy, - opa_engine, - Some(proto), - LoadedPolicyOrigin::Gateway { revision: None }, - )) -} - -fn spawn_sidecar_control_update_watcher( - mut updates: tokio::sync::mpsc::UnboundedReceiver, - provider_credentials: ProviderCredentialState, - agent_proposals: AgentProposals, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - while let Some(update) = updates.recv().await { - match update { - sidecar_control::ControlUpdate::ProviderEnv { - revision, - provider_child_env, - } => { - if revision <= provider_credentials.snapshot().revision { - continue; - } - let env_count = provider_credentials - .install_child_env_snapshot(revision, provider_child_env); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("provider_env_revision", serde_json::json!(revision)) - .message(format!( - "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" - )) - .build() - ); - } - sidecar_control::ControlUpdate::Policy { - policy_proto, - policy_hash, - config_revision, - } => { - debug!( - version = policy_proto.version, - policy_hash, - config_revision, - "Received sidecar policy update for process supervisor" - ); - } - sidecar_control::ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - apply_agent_proposals_enabled( - &agent_proposals, - enabled, - "sidecar control", - Some(config_revision), - None, - skills::install_static_skills, - ); - } - } - } - }) -} - -#[cfg(target_os = "linux")] -fn spawn_sidecar_entrypoint_handler( - mut entrypoint_rx: tokio::sync::mpsc::Receiver, - entrypoint_pid: Arc, - opa_engine: Option>, - retained_proto: Option, - openshell_endpoint: Option, - sandbox_id: Option, - trusted_ssh_socket_path: std::path::PathBuf, -) { - tokio::spawn(async move { - let mut session_started = false; - let mut trusted_supervisor_pid = None; - let terminating = Arc::new(AtomicBool::new(false)); - while let Some(started) = entrypoint_rx.recv().await { - entrypoint_pid.store(started.pid, Ordering::Release); - if started.start_session { - info!( - pid = started.pid, - ssh_socket = %trusted_ssh_socket_path.display(), - "Sidecar process supervisor reported entrypoint start" - ); - } else { - trusted_supervisor_pid = Some(started.pid); - info!( - pid = started.pid, - "Sidecar process supervisor reported initial process anchor" - ); - } - - if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { - match engine.reload_from_proto_with_pid(proto, started.pid) { - Ok(()) => info!( - pid = started.pid, - "Policy binary symlink resolution complete for sidecar process anchor" - ), - Err(err) => warn!( - error = %err, - pid = started.pid, - "Failed to rebuild OPA engine with sidecar process anchor PID" - ), - } - } - - if started.start_session - && !session_started - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let Some(supervisor_pid) = trusted_supervisor_pid else { - warn!( - pid = started.pid, - "Ignoring sidecar entrypoint event before authenticated supervisor anchor" - ); - continue; - }; - openshell_supervisor_process::supervisor_session::spawn( - endpoint.clone(), - id.clone(), - trusted_ssh_socket_path.clone(), - None, - Some(supervisor_pid), - Arc::clone(&terminating), - ); - session_started = true; - info!("sidecar supervisor session task spawned"); - } - } - terminating.store(true, Ordering::Release); - }); -} - -fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) - .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); - let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); - let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); - (cert.exists() && bundle.exists()).then_some((cert, bundle)) -} - -fn process_policy_for_topology( - policy: &SandboxPolicy, - sidecar_network_enforcement: bool, -) -> Result { - let mut process_policy = policy.clone(); - if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { - let proxy = process_policy - .network - .proxy - .get_or_insert(ProxyPolicy { http_addr: None }); - if proxy.http_addr.is_none() { - proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); - } - } - Ok(process_policy) -} - -/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. -async fn flush_proposals_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summaries: Vec, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialSummary, L7RequestSample}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summaries: Vec = summaries - .into_iter() - .map(|s| DenialSummary { - sandbox_id: String::new(), - host: s.host, - port: u32::from(s.port), - binary: s.binary, - ancestors: s.ancestors, - deny_reason: s.deny_reason, - first_seen_ms: s.first_seen_ms, - last_seen_ms: s.last_seen_ms, - count: s.count, - suppressed_count: 0, - total_count: s.count, - sample_cmdlines: s.sample_cmdlines, - binary_sha256: String::new(), - persistent: false, - denial_stage: s.denial_stage, - l7_request_samples: s - .l7_samples - .into_iter() - .map(|l| L7RequestSample { - method: l.method, - path: l.path, - decision: "deny".to_string(), - count: l.count, - }) - .collect(), - l7_inspection_active: false, - }) - .collect(); - - // Run the mechanistic mapper sandbox-side to generate proposals. - // The gateway is a thin persistence + validation layer — it never - // generates proposals itself. - let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); - - info!( - sandbox_name = %sandbox_name, - summaries = proto_summaries.len(), - proposals = proposals.len(), - "Flushed denial analysis to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - proto_summaries, - proposals, - Vec::new(), - "mechanistic", - ) - .await?; - - Ok(()) -} - -/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. -async fn flush_activity_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summary: activity_aggregator::FlushableActivitySummary, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summary = NetworkActivitySummary { - network_activity_count: summary.network_activity_count, - denied_action_count: summary.denied_action_count, - denials_by_group: summary - .denials_by_group - .into_iter() - .map(|(group, count)| DenialGroupCount { - deny_group: group, - denied_count: count, - }) - .collect(), - }; - - info!( - sandbox_name = %sandbox_name, - network_activity_count = proto_summary.network_activity_count, - denied_action_count = proto_summary.denied_action_count, - "Flushed activity summary to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - Vec::new(), - Vec::new(), - vec![proto_summary], - "activity", - ) - .await?; - - Ok(()) -} - -// ============================================================================ -// Baseline filesystem path enrichment -// ============================================================================ - -/// Minimum read-only paths required for a proxy-mode sandbox child process to -/// function: dynamic linker, shared libraries, DNS resolution, CA certs, -/// Python venv, openshell logs, process info, and random bytes. -/// -/// `/proc` and `/dev/urandom` are included here for the same reasons they -/// appear in `restrictive_default_policy()`: virtually every process needs -/// them. Before the Landlock per-path fix (#677) these were effectively free -/// because a missing path silently disabled the entire ruleset; now they must -/// be explicit. -const PROXY_BASELINE_READ_ONLY: &[&str] = &[ - "/usr", - "/lib", - "/etc", - "/app", - "/var/log", - "/proc", - "/dev/urandom", -]; - -/// Minimum read-write paths required for a proxy-mode sandbox child process: -/// user working directory and temporary files. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; - -/// GPU read-only paths. -/// -/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced -/// socket at init time. If the directory exists but Landlock denies traversal -/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` -/// even though the daemon is optional. Only read/traversal access is needed. -/// -/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, -/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` -/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may -/// not be covered by the parent-directory Landlock rule when the mount crosses -/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal -/// is permitted regardless of Landlock's cross-mount behaviour. -const GPU_BASELINE_READ_ONLY: &[&str] = &[ - "/run/nvidia-persistenced", - "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory -]; - -/// GPU read-write paths (static). -/// -/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, -/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native -/// Linux. Landlock restricts `open(2)` on device files even when DAC allows -/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. -/// These devices do not exist on WSL2 and will be skipped by the existence -/// check in `enrich_proto_baseline_paths()`. -/// -/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver -/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects -/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux -/// and will be skipped there by the existence check. -/// -/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` -/// to set thread names. Without write access, `cuInit()` returns error 304. -/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to -/// inodes and child processes have different procfs inodes than the parent. -/// -/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by -/// `enumerate_gpu_device_nodes()` since the count varies. -const GPU_BASELINE_READ_WRITE: &[&str] = &[ - "/dev/nvidiactl", - "/dev/nvidia-uvm", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-modeset", - "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) - "/proc", -]; - -/// Returns true if GPU devices are present in the container. -/// -/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and -/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these -/// depending on the host kernel; the other will not exist. -fn has_gpu_devices() -> bool { - std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() -} - -/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). -fn enumerate_gpu_device_nodes() -> Vec { - let mut paths = Vec::new(); - if let Ok(entries) = std::fs::read_dir("/dev") { - for entry in entries.flatten() { - let name = entry.file_name(); - let name = name.to_string_lossy(); - if let Some(suffix) = name.strip_prefix("nvidia") { - if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { - continue; - } - paths.push(entry.path().to_string_lossy().into_owned()); - } - } - } - paths -} - -fn push_unique(paths: &mut Vec, path: String) { - if !paths.iter().any(|p| p == &path) { - paths.push(path); - } -} - -fn collect_baseline_enrichment_paths( - include_proxy: bool, - include_gpu: bool, - gpu_device_nodes: Vec, -) -> (Vec, Vec) { - let mut ro = Vec::new(); - let mut rw = Vec::new(); - - if include_proxy { - for &path in PROXY_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in PROXY_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - } - - if include_gpu { - for &path in GPU_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in GPU_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - for path in gpu_device_nodes { - push_unique(&mut rw, path); - } - } - - // A path promoted to read_write (e.g. /proc for GPU) should not also - // appear in read_only — Landlock handles the overlap correctly but the - // duplicate is confusing when inspecting the effective policy. - ro.retain(|p| !rw.contains(p)); - - (ro, rw) -} - -fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { - let include_gpu = has_gpu_devices(); - let gpu_device_nodes = if include_gpu { - enumerate_gpu_device_nodes() - } else { - Vec::new() - }; - collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) -} - -/// Collect all active baseline paths for tests and diagnostics. -/// Returns `(read_only, read_write)` as owned `String` vecs. -#[cfg(test)] -fn baseline_enrichment_paths() -> (Vec, Vec) { - active_baseline_enrichment_paths(true) -} - -fn enrich_proto_baseline_paths_with( - proto: &mut openshell_core::proto::SandboxPolicy, - ro: &[String], - rw: &[String], - path_exists: F, -) -> bool -where - F: Fn(&str) -> bool, -{ - if ro.is_empty() && rw.is_empty() { - return false; - } - - let fs = proto - .filesystem - .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { - include_workdir: true, - ..Default::default() - }); - - let mut modified = false; - for path in ro { - if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { - if !path_exists(path) { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - fs.read_only.push(path.clone()); - modified = true; - } - } - for path in rw { - if fs.read_write.iter().any(|p| p == path) { - continue; - } - if !path_exists(path) { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - if fs.read_only.iter().any(|p| p == path) { - if path == "/proc" { - info!( - path, - "Promoting /proc from read-only to read-write for GPU runtime compatibility" - ); - fs.read_only.retain(|p| p != path); - fs.read_write.push(path.clone()); - modified = true; - } - continue; - } - fs.read_write.push(path.clone()); - modified = true; - } - - modified -} - -/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths -/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if -/// missing; user-specified paths are never removed. -/// -/// Returns `true` if the policy was modified (caller may want to sync back). -fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); - - // Baseline paths are system-injected, not user-specified. Skip paths - // that do not exist in this container image to avoid noisy warnings from - // Landlock and, more critically, to prevent a single missing baseline - // path from abandoning the entire Landlock ruleset under best-effort - // mode (see issue #664). - let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { - std::path::Path::new(path).exists() - }); - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } - - modified -} - -fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - openshell_policy::strip_provider_rule_names(proto) -} - -fn proto_sync_payload_for_enriched_policy( - proto: &openshell_core::proto::SandboxPolicy, - enriched: bool, -) -> Option { - if !enriched { - return None; - } - - let mut sync_policy = proto.clone(); - strip_proto_provider_policy_entries(&mut sync_policy); - Some(sync_policy) -} - -/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem -/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the -/// local-file code path where no proto is available. -fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { - let (ro, rw) = - active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); - if ro.is_empty() && rw.is_empty() { - return; - } - - let mut modified = false; - for path in &ro { - let p = std::path::PathBuf::from(path); - if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { - if !p.exists() { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_only.push(p); - modified = true; - } - } - for path in &rw { - let p = std::path::PathBuf::from(path); - if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { - continue; - } - if !p.exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_write.push(p); - modified = true; - } - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod baseline_tests { - use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; - - #[test] - fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { - // When GPU devices are present, /proc is promoted to read_write - // (CUDA needs to write /proc//task//comm). It should - // NOT also appear in read_only. - if !has_gpu_devices() { - // Can't test GPU dedup without GPU devices; skip silently. - return; - } - let (ro, rw) = baseline_enrichment_paths(); - assert!( - rw.contains(&"/proc".to_string()), - "/proc should be in read_write when GPU is present" - ); - assert!( - !ro.contains(&"/proc".to_string()), - "/proc should NOT be in read_only when it is already in read_write" - ); - } - - #[test] - fn proc_in_read_only_without_gpu() { - if has_gpu_devices() { - // On a GPU host we can't test the non-GPU path; skip silently. - return; - } - let (ro, _rw) = baseline_enrichment_paths(); - assert!( - ro.contains(&"/proc".to_string()), - "/proc should be in read_only when GPU is not present" - ); - } - - #[test] - fn baseline_read_write_always_includes_sandbox_and_tmp() { - let (_ro, rw) = baseline_enrichment_paths(); - assert!(rw.contains(&"/sandbox".to_string())); - assert!(rw.contains(&"/tmp".to_string())); - } - - #[test] - fn enumerate_gpu_device_nodes_skips_bare_nvidia() { - // "nvidia" (without a trailing digit) is a valid /dev entry on some - // systems but is not a per-GPU device node. The enumerator must - // not match it. - let nodes = enumerate_gpu_device_nodes(); - assert!( - !nodes.contains(&"/dev/nvidia".to_string()), - "bare /dev/nvidia should not be enumerated: {nodes:?}" - ); - } - - #[test] - fn no_duplicate_paths_in_baseline() { - let (ro, rw) = baseline_enrichment_paths(); - // No path should appear in both lists. - for path in &ro { - assert!( - !rw.contains(path), - "path {path} appears in both read_only and read_write" - ); - } - } - - #[test] - fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { - read_only: vec!["/tmp".to_string()], - read_write: vec![], - include_workdir: false, - }); - policy.network_policies.insert( - "test".into(), - openshell_core::proto::NetworkPolicyRule { - name: "test-rule".into(), - endpoints: vec![openshell_core::proto::NetworkEndpoint { - host: "example.com".into(), - port: 443, - ..Default::default() - }], - ..Default::default() - }, - ); - - enrich_proto_baseline_paths(&mut policy); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - filesystem.read_only.contains(&"/tmp".to_string()), - "explicit read_only baseline path should be preserved" - ); - assert!( - !filesystem.read_write.contains(&"/tmp".to_string()), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - assert!(strip_proto_provider_policy_entries(&mut policy)); - assert!( - !policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(policy.network_policies.contains_key("sandbox_only")); - assert!(!strip_proto_provider_policy_entries(&mut policy)); - } - - #[test] - fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - - assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "provider-derived rules alone must not trigger sync or mutate runtime policy" - ); - } - - #[test] - fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - runtime_policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) - .expect("enrichment should create a sync payload"); - - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "runtime policy must retain provider-derived rules for OPA input" - ); - assert!( - !sync_policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(sync_policy.network_policies.contains_key("sandbox_only")); - } - - #[test] - fn proto_gpu_enrichment_promotes_proc_without_network_policy() { - let mut policy = openshell_policy::restrictive_default_policy(); - assert!( - policy.network_policies.is_empty(), - "regression setup must exercise the no-network default path" - ); - let (ro, rw) = - collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); - - let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { - matches!(path, "/proc" | "/dev/nvidia0") - }); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - enriched, - "GPU enrichment should not require network policies" - ); - assert!( - filesystem.read_write.contains(&"/dev/nvidia0".to_string()), - "GPU enrichment should add enumerated device nodes without network policies" - ); - assert!( - !filesystem.read_only.contains(&"/proc".to_string()), - "GPU enrichment should remove /proc from read_only" - ); - assert!( - filesystem.read_write.contains(&"/proc".to_string()), - "GPU enrichment should promote /proc to read_write" - ); - } - - #[test] - fn gpu_baseline_read_write_contains_dxg() { - // /dev/dxg must be present so WSL2 sandboxes get the Landlock - // read-write rule for the CDI-injected DXG device. The existence - // check in enrich_proto_baseline_paths() skips it on native Linux. - assert!( - GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), - "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" - ); - } - - #[test] - fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only: vec![std::path::PathBuf::from("/tmp")], - read_write: vec![], - include_workdir: false, - }, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - }; - - enrich_sandbox_baseline_paths(&mut policy); - - assert!( - policy - .filesystem - .read_only - .contains(&std::path::PathBuf::from("/tmp")), - "explicit read_only baseline path should be preserved" - ); - assert!( - !policy - .filesystem - .read_write - .contains(&std::path::PathBuf::from("/tmp")), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn gpu_baseline_read_only_contains_usr_lib_wsl() { - // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library - // bind-mounts are accessible under Landlock. Skipped on native Linux. - assert!( - GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), - "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" - ); - } - - #[test] - fn has_gpu_devices_reflects_dxg_or_nvidiactl() { - // Verify the OR logic: result must match the manual disjunction of - // the two path checks. Passes in all environments. - let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); - let dxg = std::path::Path::new("/dev/dxg").exists(); - assert_eq!( - has_gpu_devices(), - nvidiactl || dxg, - "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" - ); - } -} - -/// Returns `true` if the error is transient and worth retrying. -/// -/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If -/// found, only the gRPC codes that represent transient failures are retryable. -/// If no `tonic::Status` is present (e.g. a raw connection error), assume the -/// failure is transient. -fn is_retryable_error(err: &miette::Report) -> bool { - let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); - while let Some(e) = source { - if let Some(status) = e.downcast_ref::() { - return matches!( - status.code(), - tonic::Code::Unavailable - | tonic::Code::DeadlineExceeded - | tonic::Code::ResourceExhausted - | tonic::Code::Aborted - | tonic::Code::Internal - | tonic::Code::Unknown - ); - } - source = e.source(); - } - true -} - -/// Retry a gRPC operation with exponential backoff (capped at 4 s). -/// -/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, -/// `PERMISSION_DENIED`) are returned immediately without retrying. -async fn grpc_retry(op_name: &str, f: F) -> Result -where - F: Fn() -> Fut, - Fut: Future>, -{ - let mut last_err = None; - for attempt in 1..=5u32 { - match f().await { - Ok(val) => return Ok(val), - Err(e) => { - if !is_retryable_error(&e) { - return Err(e); - } - if attempt < 5 { - warn!( - attempt, - max_attempts = 5, - error = %e, - "{op_name} failed, retrying" - ); - let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); - tokio::time::sleep(backoff).await; - } - last_err = Some(e); - } - } - } - Err(miette::miette!( - "{op_name} failed after 5 attempts: {}", - last_err.expect("loop executed at least once") - )) -} - -/// Load sandbox policy from local files or gRPC. -/// -/// Priority: -/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files -/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC -/// 3. If the server returns no policy, discover from disk or use restrictive default -/// 4. Otherwise, return an error -/// -/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto -/// policy. The proto is retained so the OPA engine can be rebuilt with symlink -/// resolution after the container entrypoint starts. -async fn load_policy( - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, -) -> Result<( - SandboxPolicy, - Option>, - Option, - MiddlewareRegistryStatus, - LoadedPolicyOrigin, - bool, -)> { - // File mode: load OPA engine from rego rules + YAML data (dev override) - if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "loading") - .unmapped("policy_rules", serde_json::json!(policy_file)) - .unmapped("policy_data", serde_json::json!(data_file)) - .message(format!( - "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" - )) - .build()); - let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { - openshell_supervisor_middleware_builtins::validate_config(implementation, config) - .map_err(|error| error.to_string()) - }; - let engine = OpaEngine::from_files_with_middleware_config( - std::path::Path::new(policy_file), - std::path::Path::new(data_file), - Some(&validate_middleware_config), - )?; - let middleware_registry = - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - engine.replace_middleware_registry(middleware_registry)?; - let config = engine.query_sandbox_config()?; - let mut policy = SandboxPolicy { - version: 1, - filesystem: config.filesystem, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: config.landlock, - process: config.process, - }; - enrich_sandbox_baseline_paths(&mut policy); - // File mode has no operator-registered middleware to connect. - return Ok(( - policy, - Some(Arc::new(engine)), - None, - MiddlewareRegistryStatus::Synchronized, - LoadedPolicyOrigin::LocalOverride, - false, - )); - } - - // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - info!( - sandbox_id = %id, - endpoint = %endpoint, - "Fetching sandbox policy via gRPC" - ); - let mut snapshot = grpc_retry("Policy fetch", || { - openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) - }) - .await?; - - let mut proto_policy = if let Some(p) = snapshot.policy.clone() { - p - } else { - // No policy configured on the server. Discover from disk or - // fall back to the restrictive default, then sync to the - // gateway so it becomes the authoritative baseline. - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "discovery") - .message("Server returned no policy; attempting local discovery") - .build() - ); - let mut discovered = discover_policy_from_disk_or_default(); - // Enrich before syncing so the gateway baseline includes - // baseline paths from the start. - enrich_proto_baseline_paths(&mut discovered); - strip_proto_provider_policy_entries(&mut discovered); - let sandbox = sandbox.as_deref().ok_or_else(|| { - miette::miette!( - "Cannot sync discovered policy: sandbox not available.\n\ - Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." - ) - })?; - - // Sync and re-fetch over a single connection to avoid extra - // TLS handshakes. - let ws = snapshot.workspace.clone(); - snapshot = grpc_retry("Policy discovery sync", || { - openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox, - &discovered, - &ws, - ) - }) - .await?; - snapshot.policy.clone().ok_or_else(|| { - miette::miette!("Server still returned no policy after sync — this is a bug") - })? - }; - - // True only while `snapshot` describes the exact policy that will be - // constructed below. If enrichment cannot be synced and re-fetched, - // the policy remains enforceable but cannot be acknowledged by - // inferred structural equality. - let mut policy_bound_to_snapshot = true; - - // Ensure baseline filesystem paths are present for proxy-mode - // sandboxes. If the policy was enriched, sync the updated version - // back to the gateway so users can see the effective policy. - let enriched = enrich_proto_baseline_paths(&mut proto_policy); - let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); - if let Some(sync_policy) = sync_policy { - if let Some(sandbox_name) = sandbox.as_deref() { - match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox_name, - &sync_policy, - &snapshot.workspace, - ) - .await - { - Ok(canonical) => { - if let Some(policy) = canonical.policy.clone() { - proto_policy = policy; - snapshot = canonical; - } else { - policy_bound_to_snapshot = false; - warn!( - "Gateway returned no policy after enrichment sync; initial revision will be reconciled" - ); - } - } - Err(e) => { - policy_bound_to_snapshot = false; - warn!( - error = %e, - "Failed to sync enriched policy back to gateway; initial revision will be reconciled" - ); - } - } - } else { - policy_bound_to_snapshot = false; - } - } - - let loaded_policy_revision = - policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); - - // Build OPA engine from baked-in rules + typed proto data. - // In cluster mode, proxy networking is always enabled so OPA is - // always required for allow/deny decisions. - // The initial load uses pid=0 (no symlink resolution) because the - // container hasn't started yet. After the entrypoint spawns, the - // engine is rebuilt with the real PID for symlink resolution. - info!("Creating OPA engine from proto policy data"); - let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => engine, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - - // Install the in-process catalog before any external connection can - // fail. A newly started sandbox must always be able to resolve built-in - // bindings, even while operator-run services are unavailable. - install_builtin_middleware_registry(&engine).await?; - - // Connect operator-registered middleware services. A connect/describe - // failure keeps the built-in registry active so each request's - // `on_error` policy governs matched traffic. The policy poll loop - // retries the install without waiting for a config change. - let middleware_services = snapshot.supervisor_middleware_services.clone(); - let middleware_registry_status = if middleware_services.is_empty() { - MiddlewareRegistryStatus::Synchronized - } else if let Err(error) = grpc_retry("Middleware connect", || { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - middleware_services.clone(), - ) - }) - .await - .and_then(|registry| engine.replace_middleware_registry(registry)) - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(middleware_services.len()) - ) - .message(format!( - "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" - )) - .build() - ); - MiddlewareRegistryStatus::NeedsReconciliation - } else { - MiddlewareRegistryStatus::Synchronized - }; - let opa_engine = Some(Arc::new(engine)); - - let policy = match SandboxPolicy::try_from(proto_policy.clone()) { - Ok(policy) => policy, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - return Ok(( - policy, - opa_engine, - Some(proto_policy), - middleware_registry_status, - LoadedPolicyOrigin::Gateway { - revision: loaded_policy_revision, - }, - agent_proposals_enabled_from_settings(&snapshot.settings), - )); - } - - // No policy source available - Err(miette::miette!( - "Sandbox policy required. Provide one of:\n\ - - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ - - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" - )) -} - -/// Try to discover a sandbox policy from the well-known disk path, falling -/// back to the legacy path, then to the hardcoded restrictive default. -fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { - let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); - if primary.exists() { - return discover_policy_from_path(primary); - } - let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); - if legacy.exists() { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "legacy_path", - serde_json::json!(legacy.display().to_string()) - ) - .unmapped("new_path", serde_json::json!(primary.display().to_string())) - .message(format!( - "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", - legacy.display(), - primary.display() - )) - .build() - ); - return discover_policy_from_path(legacy); - } - discover_policy_from_path(primary) -} - -/// Try to read a sandbox policy YAML from `path`, falling back to the -/// hardcoded restrictive default if the file is missing or invalid. -fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { - use openshell_policy::{ - parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, - }; - - let Ok(yaml) = std::fs::read_to_string(path) else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "default") - .message(format!( - "No policy file on disk, using restrictive default [path:{}]", - path.display() - )) - .build() - ); - return restrictive_default_policy(); - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Loaded sandbox policy from container disk [path:{}]", - path.display() - )) - .build() - ); - match parse_sandbox_policy(&yaml) { - Ok(policy) => { - // Validate the disk-loaded policy for safety. - if let Err(violations) = validate_sandbox_policy(&policy) { - let messages: Vec = violations.iter().map(ToString::to_string).collect(); - ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Medium) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .finding_info( - FindingInfo::new( - "unsafe-disk-policy", - "Unsafe Disk Policy Content", - ) - .with_desc(&format!( - "Disk policy at {} contains unsafe content: {}", - path.display(), - messages.join("; "), - )), - ) - .message(format!( - "Disk policy contains unsafe content, using restrictive default [path:{}]", - path.display() - )) - .build()); - return restrictive_default_policy(); - } - policy - } - Err(e) => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "fallback") - .message(format!( - "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", - path.display() - )) - .build()); - restrictive_default_policy() - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum MiddlewareRegistryStatus { - Synchronized, - NeedsReconciliation, -} - -/// True when the installed middleware registry no longer matches the desired -/// service set and must be rebuilt (reconnecting every delivered service). -/// -/// A policy-only change never requires a rebuild: middleware configs were -/// validated at gateway admission and the installed registry's manifests -/// already cover the unchanged service set, so requiring the services to be -/// reachable would only let a middleware outage block the policy update. -fn middleware_registry_needs_rebuild( - registry_status: MiddlewareRegistryStatus, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> bool { - registry_status == MiddlewareRegistryStatus::NeedsReconciliation - || current_services != desired_services -} - -fn gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy: bool, - current_policy_hash: &str, - desired_policy_hash: &str, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - registry_status: MiddlewareRegistryStatus, -) -> bool { - reloads_gateway_policy - && (current_policy_hash != desired_policy_hash - || middleware_registry_needs_rebuild( - registry_status, - current_services, - desired_services, - )) -} - -/// Identity returned with the exact policy snapshot used to construct OPA. -#[derive(Clone, Debug, PartialEq, Eq)] -struct LoadedPolicyRevision { - version: u32, - policy_hash: String, - config_revision: u64, - policy_source: openshell_core::proto::PolicySource, -} - -/// Identifies where the policy currently loaded into OPA came from. -/// -/// A missing gateway revision means the policy was loaded from the gateway but -/// could not be bound to an authoritative snapshot (for example, enrichment -/// sync failed). That state must reconcile on the first successful poll. A -/// local-file override is different: gateway policy revisions are observed for -/// settings/provider refreshes but must never replace the explicit local OPA -/// policy. -#[derive(Clone, Debug, PartialEq, Eq)] -enum LoadedPolicyOrigin { - LocalOverride, - Gateway { - revision: Option, - }, -} - -impl LoadedPolicyOrigin { - fn allows_gateway_policy_reload(&self) -> bool { - matches!(self, Self::Gateway { .. }) - } -} - -impl LoadedPolicyRevision { - fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { - Self { - version: snapshot.version, - policy_hash: snapshot.policy_hash.clone(), - config_revision: snapshot.config_revision, - policy_source: snapshot.policy_source, - } - } -} - -/// A sandbox-scoped policy revision that was constructed successfully at -/// startup and must be acknowledged to the gateway exactly once. -#[derive(Clone, Debug, PartialEq, Eq)] -struct InitialPolicyAck { - version: u32, - policy_hash: String, - config_revision: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PolicyStatusUpdate { - version: u32, - loaded: bool, - error: String, - initial_policy_hash: Option, -} - -impl PolicyStatusUpdate { - fn initial_loaded(ack: &InitialPolicyAck) -> Self { - Self { - version: ack.version, - loaded: true, - error: String::new(), - initial_policy_hash: Some(ack.policy_hash.clone()), - } - } - - fn loaded(version: u32) -> Self { - Self { - version, - loaded: true, - error: String::new(), - initial_policy_hash: None, - } - } - - fn failed(version: u32, error: String) -> Self { - Self { - version, - loaded: false, - error, - initial_policy_hash: None, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum InitialPollDisposition { - Acknowledge(InitialPolicyAck), - Reconcile, - TrackOnly, -} - -/// Determine whether the initially loaded policy corresponds to an -/// authoritative sandbox-scoped revision that must be acknowledged. -/// -/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose -/// captured gateway identity matches the current version and hash. Global -/// policies, local-file development policies, version zero, and changed -/// identities yield `None`, so those paths never emit a sandbox-revision -/// acknowledgement. -fn initial_policy_ack_candidate( - loaded: Option<&LoadedPolicyRevision>, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - let loaded = loaded?; - if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox - || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox - { - return None; - } - if loaded.version == 0 || canonical.version == 0 { - return None; - } - if loaded.version != canonical.version - || loaded.policy_hash != canonical.policy_hash - || canonical.config_revision < loaded.config_revision - { - return None; - } - Some(InitialPolicyAck { - version: loaded.version, - policy_hash: loaded.policy_hash.clone(), - config_revision: canonical.config_revision, - }) -} - -fn initial_poll_disposition( - origin: &LoadedPolicyOrigin, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> InitialPollDisposition { - match origin { - LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision } => { - initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( - InitialPollDisposition::Reconcile, - InitialPollDisposition::Acknowledge, - ) - } - } -} - -/// Deliver policy status updates independently from policy reconciliation. -/// -/// The channel is FIFO, so a delayed older status can never arrive after a -/// newer status and move the gateway's active version backward. Delivery uses -/// the existing bounded retry, but failures never delay policy enforcement. -async fn run_policy_status_reporter( - client: openshell_core::grpc_client::CachedOpenShellClient, - sandbox_id: String, - mut updates: tokio::sync::mpsc::UnboundedReceiver, -) { - 'updates: while let Some(update) = updates.recv().await { - let operation = if update.initial_policy_hash.is_some() { - "Initial policy acknowledgement" - } else { - "Policy status report" - }; - let mut attempt = 1_u32; - loop { - let sandbox_id = sandbox_id.clone(); - let error = update.error.clone(); - let client = client.clone(); - match client - .report_policy_status(&sandbox_id, update.version, update.loaded, &error) - .await - { - Ok(()) => break, - Err(error) if is_retryable_error(&error) => { - let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); - warn!( - %error, - attempt, - version = update.version, - loaded = update.loaded, - retry_in_secs = backoff.as_secs(), - "{operation} failed transiently; retaining ordered update" - ); - tokio::time::sleep(backoff).await; - attempt = attempt.saturating_add(1); - } - Err(error) => { - warn!( - %error, - version = update.version, - loaded = update.loaded, - "Discarding terminal policy status update" - ); - continue 'updates; - } - } - } - - if let Some(policy_hash) = update.initial_policy_hash { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("version", serde_json::json!(update.version)) - .unmapped("policy_hash", serde_json::json!(policy_hash)) - .message(format!( - "Acknowledged initial policy revision as loaded [version:{}]", - update.version - )) - .build() - ); - } - } -} - -fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { - let version = update.version; - if let Err(error) = sender.send(update) { - warn!( - %error, - version, - "Policy status reporter unavailable during shutdown" - ); - } -} - -/// Best-effort `FAILED` acknowledgement when initial policy construction or -/// conversion fails. -/// -/// Uses the revision identity captured with the policy that failed to build, -/// and preserves the original construction error as the reported message. A -/// delivery failure here is swallowed so it can never mask that error. -async fn report_initial_policy_failure( - endpoint: &str, - sandbox_id: &str, - revision: Option<&LoadedPolicyRevision>, - error: &miette::Report, -) { - let Some(revision) = revision.filter(|revision| { - revision.version > 0 - && revision.policy_source == openshell_core::proto::PolicySource::Sandbox - }) else { - return; - }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { - Ok(client) => client, - Err(e) => { - warn!(error = %e, "Failed to connect to report initial policy failure"); - return; - } - }; - let message = error.to_string(); - if let Err(e) = grpc_retry("Initial policy failure report", || { - let client = client.clone(); - let message = message.clone(); - async move { - client - .report_policy_status(sandbox_id, revision.version, false, &message) - .await - } - }) - .await - { - warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); - } +#[cfg(target_os = "windows")] +pub fn windows_unsupported() -> &'static str { + "openshell-sandbox supervisor is not available on Windows" } -/// Background loop that polls the server for policy updates. -/// -/// When a new version is detected, attempts to reload the OPA engine via -/// `reload_from_proto_with_pid()`. Reports load success/failure back to the -/// server. On failure, the previous engine is untouched (LKG behavior). -/// -/// When the entrypoint PID is available, policy reloads include symlink -/// resolution for binary paths via the container filesystem. -struct PolicyPollLoopContext { - endpoint: String, - sandbox_id: String, - opa_engine: Arc, - /// Source of the policy currently loaded into OPA. This distinguishes an - /// explicit local-file override from an unbound gateway revision so the - /// former is never replaced by policy polling. - loaded_policy_origin: LoadedPolicyOrigin, - entrypoint_pid: Arc, - interval_secs: u64, - ocsf_enabled: Arc, - provider_credentials: ProviderCredentialState, - policy_local_ctx: Option>, - agent_proposals: AgentProposals, - middleware_registry_status: MiddlewareRegistryStatus, - sidecar_control_publisher: Option, - workspace_tx: tokio::sync::watch::Sender, -} - -async fn connect_middleware_registry( - services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> Result { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - ) - .await -} - -async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - opa_engine.replace_middleware_registry(registry) -} - -async fn reconcile_middleware_registry( - opa_engine: &OpaEngine, - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - current_services: &mut Vec, - status: &mut MiddlewareRegistryStatus, -) { - if *status == MiddlewareRegistryStatus::Synchronized - && desired_services == current_services.as_slice() - { - return; - } - - match connect_middleware_registry(desired_services) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - current_services.clear(); - current_services.extend_from_slice(desired_services); - *status = MiddlewareRegistryStatus::Synchronized; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(current_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - current_services.len() - )) - .build() - ); - } - Err(error) => { - // Emit only on the transition into the failed state to avoid - // repeating the same finding on every poll during an outage. - if *status == MiddlewareRegistryStatus::Synchronized { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - } - *status = MiddlewareRegistryStatus::NeedsReconciliation; - } - } -} - -async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::PolicySource; - use std::sync::atomic::Ordering; - - let client = CachedOpenShellClient::connect(&ctx.endpoint).await?; - let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); - tokio::spawn(run_policy_status_reporter( - client.clone(), - ctx.sandbox_id.clone(), - status_receiver, - )); - - let mut current_config_revision: u64 = 0; - let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; - let mut current_policy_hash = String::new(); - let mut current_middleware_services = Vec::new(); - let mut middleware_registry_status = ctx.middleware_registry_status; - let mut current_settings: std::collections::HashMap< - String, - openshell_core::proto::EffectiveSetting, - > = std::collections::HashMap::new(); - let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); - let mut last_failed_runtime_revision: Option<(u64, String)> = None; - - // A first poll that does not match the policy already loaded into OPA must - // pass through the normal reconciliation path immediately. It must never - // seed the applied-state trackers before OPA actually loads it. - let mut pending_result = None; - - // Initialize revision from the first poll and acknowledge the initial - // policy revision the supervisor actually loaded. A mismatched result is - // reconciled below instead of being recorded as already applied. - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { - InitialPollDisposition::Acknowledge(candidate) => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(candidate.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = candidate.config_revision; - current_policy_hash.clone_from(&candidate.policy_hash); - current_middleware_services = result.supervisor_middleware_services; - current_settings = result.settings; - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::initial_loaded(&candidate), - ); - debug!( - config_revision = current_config_revision, - "Settings poll: initial policy matches loaded revision" - ); - } - InitialPollDisposition::Reconcile => pending_result = Some(result), - InitialPollDisposition::TrackOnly => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash.clone(); - current_middleware_services = result.supervisor_middleware_services; - current_settings = result.settings; - debug!( - config_revision = current_config_revision, - "Settings poll: tracking gateway config while preserving local policy override" - ); - } - } - } - Err(e) => { - warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); - } - } - - let interval = Duration::from_secs(ctx.interval_secs); - loop { - let result = if let Some(result) = pending_result.take() { - result - } else { - tokio::time::sleep(interval).await; - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - result - } - Err(e) => { - debug!(error = %e, "Settings poll: server unreachable, will retry"); - continue; - } - } - }; - - let config_changed = result.config_revision != current_config_revision; - let provider_env_changed = result.provider_env_revision != current_provider_env_revision; - let policy_changed = result.policy_hash != current_policy_hash; - let middleware_registry_changed = middleware_registry_needs_rebuild( - middleware_registry_status, - ¤t_middleware_services, - &result.supervisor_middleware_services, - ); - let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); - - // A local policy override is not coupled to the gateway policy - // snapshot, so its service registry can still be reconciled alone. - // Gateway policy snapshots, however, must install policy and registry - // as one generation below. - if !reloads_gateway_policy { - reconcile_middleware_registry( - &ctx.opa_engine, - &result.supervisor_middleware_services, - &mut current_middleware_services, - &mut middleware_registry_status, - ) - .await; - } - - if !config_changed && !provider_env_changed && !policy_runtime_changed { - continue; - } - - if config_changed || provider_env_changed { - // Log which settings changed. - log_setting_changes(¤t_settings, &result.settings); - - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "detected") - .unmapped("old_config_revision", serde_json::json!(current_config_revision)) - .unmapped("new_config_revision", serde_json::json!(result.config_revision)) - .unmapped("policy_changed", serde_json::json!(policy_changed)) - .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) - .message(format!( - "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", - result.config_revision - )) - .build()); - } - - if provider_env_changed { - match openshell_core::grpc_client::fetch_provider_environment( - &ctx.endpoint, - &ctx.sandbox_id, - ) - .await - { - Ok(env_result) => { - ctx.provider_credentials.install_environment( - env_result.provider_env_revision, - env_result.environment, - env_result.credential_expires_at_ms, - env_result.dynamic_credentials, - ); - let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); - let env_count = child_env.len(); - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_provider_env( - env_result.provider_env_revision, - child_env.clone(), - ); - } - current_provider_env_revision = env_result.provider_env_revision; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "provider_env_revision", - serde_json::json!(env_result.provider_env_revision) - ) - .message(format!( - "Provider environment refreshed [revision:{} env_count:{env_count}]", - env_result.provider_env_revision - )) - .build() - ); - } - Err(e) => { - warn!( - error = %e, - provider_env_revision = result.provider_env_revision, - "Settings poll: failed to refresh provider environment" - ); - } - } - } - - if policy_runtime_changed { - let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = match result.policy.as_ref() { - Some(policy) if middleware_registry_changed => { - match connect_middleware_registry(&result.supervisor_middleware_services).await - { - Ok(registry) => ctx - .opa_engine - .reload_policy_and_middleware_from_proto_with_pid( - policy, pid, registry, - ), - Err(error) => Err(error), - } - } - // Policy-only change: the installed registry already matches - // the delivered service set, so swap the engine alone. This - // must not require middleware reachability. - Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), - None => Err(miette::miette!( - "runtime reload requires a policy payload but none was returned" - )), - }; - - match runtime_result { - Ok(()) => { - let policy = result - .policy - .as_ref() - .expect("successful runtime reload requires a policy payload"); - if policy_changed { - if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { - policy_local_ctx.set_current_policy(policy.clone()).await; - } - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_policy( - policy.clone(), - result.policy_hash.clone(), - result.config_revision, - ); - } - if result.global_policy_version > 0 { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .unmapped("global_version", serde_json::json!(result.global_policy_version)) - .message(format!( - "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", - result.policy_hash, - result.global_policy_version - )) - .build()); - } else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - } - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - } - } - - if middleware_registry_changed { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(result.supervisor_middleware_services.len()) - ) - .message(format!( - "Supervisor policy runtime reloaded atomically [service_count:{}]", - result.supervisor_middleware_services.len() - )) - .build()); - } - - current_policy_hash.clone_from(&result.policy_hash); - current_middleware_services.clone_from(&result.supervisor_middleware_services); - middleware_registry_status = MiddlewareRegistryStatus::Synchronized; - last_failed_runtime_revision = None; - } - Err(e) => { - let failed_revision = (result.config_revision, result.policy_hash.clone()); - if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(e.to_string())) - .message(format!( - "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", - result.version - )) - .build()); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, e.to_string()), - ); - } - } - last_failed_runtime_revision = Some(failed_revision); - // Nothing was installed, so the registry status still - // describes the live registry. The retry is driven by the - // persisting hash/service-set mismatch (or an existing - // NeedsReconciliation), not by degrading the status here. - } - } - } - - // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - - // Apply the agent-proposals feature toggle. On a false→true transition - // we lazily install the skill so a sandbox that started with the flag - // off picks up the surface without a recreate. We never uninstall on - // a true→false transition: stale skill content on disk is harmless - // because route_request and agent_next_steps both gate on the live - // shared flag, so the agent that reads the skill will see 404s and an - // empty `next_steps` array regardless. - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - - current_config_revision = result.config_revision; - if !reloads_gateway_policy { - current_policy_hash = result.policy_hash; - } - current_settings = result.settings; - } -} - -fn apply_ocsf_json_setting( - enabled: &AtomicBool, - settings: &std::collections::HashMap, -) { - use std::sync::atomic::Ordering; - - let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); - let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); - if new_ocsf != prev_ocsf { - info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); - } -} - -/// Extract a bool value from an effective setting, if present. -fn extract_bool_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) -} - -fn agent_proposals_enabled_from_settings( - settings: &std::collections::HashMap, -) -> bool { - extract_bool_setting( - settings, - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, - ) - .unwrap_or(false) -} - -fn apply_agent_proposals_enabled( - agent_proposals: &AgentProposals, - enabled: bool, - source: &'static str, - config_revision: Option, - sidecar_control_publisher: Option<&sidecar_control::Publisher>, - install_static_skills: impl FnOnce() -> Result, -) { - let previously_enabled = agent_proposals.swap_enabled(enabled); - if enabled == previously_enabled { - return; - } - - info!( - agent_policy_proposals_enabled = enabled, - source, config_revision, "agent-driven policy proposals toggled" - ); - - if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { - publisher.publish_agent_proposals(enabled, config_revision); - } - - if enabled && !previously_enabled { - match install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill on toggle-on" - ), - Err(error) => warn!( - error = %error, - "Failed to install sandbox agent skill on toggle-on" - ), - } - } -} - -/// Log individual setting changes between two snapshots. -fn log_setting_changes( - old: &std::collections::HashMap, - new: &std::collections::HashMap, -) { - for (key, new_es) in new { - let new_val = format_setting_value(new_es); - match old.get(key) { - Some(old_es) => { - let old_val = format_setting_value(old_es); - if old_val != new_val { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "updated") - .unmapped("key", serde_json::json!(key)) - .unmapped("old", serde_json::json!(old_val.clone())) - .unmapped("new", serde_json::json!(new_val.clone())) - .message(format!( - "Setting changed [key:{key} old:{old_val} new:{new_val}]" - )) - .build() - ); - } - } - None => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enabled") - .unmapped("key", serde_json::json!(key)) - .unmapped("value", serde_json::json!(new_val.clone())) - .message(format!("Setting added [key:{key} value:{new_val}]")) - .build() - ); - } - } - } - for key in old.keys() { - if !new.contains_key(key) { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Disabled, "disabled") - .unmapped("key", serde_json::json!(key)) - .message(format!("Setting removed [key:{key}]")) - .build() - ); - } - } -} - -/// Format an `EffectiveSetting` value for log display. -fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { - use openshell_core::proto::setting_value; - match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { - None => "".to_string(), - Some(setting_value::Value::StringValue(v)) => v.clone(), - Some(setting_value::Value::BoolValue(v)) => v.to_string(), - Some(setting_value::Value::IntValue(v)) => v.to_string(), - Some(setting_value::Value::BytesValue(_)) => "".to_string(), - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod tests { - use super::*; - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - fn proxy_policy(http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { - openshell_core::proto::EffectiveSetting { - value: Some(openshell_core::proto::SettingValue { - value: Some(openshell_core::proto::setting_value::Value::BoolValue( - value, - )), - }), - scope: openshell_core::proto::SettingScope::Global.into(), - } - } - - #[test] - fn sidecar_process_policy_sets_loopback_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, true).unwrap(); - - let http_addr = process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .expect("sidecar process policy should set proxy address"); - assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); - assert!( - policy - .network - .proxy - .as_ref() - .expect("original policy should keep proxy config") - .http_addr - .is_none(), - "process policy normalization must not mutate the network policy" - ); - } - - #[test] - fn non_sidecar_process_policy_preserves_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, false).unwrap(); - - assert!( - process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .is_none() - ); - } - - #[tokio::test] - async fn sidecar_control_provider_env_update_installs_newer_revision() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - 1, - std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), - ); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials.clone(), - AgentProposals::default(), - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "new".to_string(), - )]), - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 2 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 2); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 1, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "stale".to_string(), - )]), - }) - .unwrap(); - tokio::time::sleep(Duration::from_millis(20)).await; - assert_eq!( - provider_credentials - .snapshot() - .child_env - .get("TOKEN") - .map(String::as_str), - Some("new") - ); - handle.abort(); - } - - #[tokio::test] - async fn sidecar_control_agent_proposals_update_flips_shared_state() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = - ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); - let agent_proposals = AgentProposals::new(true); - let handle = - spawn_sidecar_control_update_watcher(rx, provider_credentials, agent_proposals.clone()); - - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 5, - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if !agent_proposals.enabled() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - handle.abort(); - } - - #[test] - fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { - let agent_proposals = AgentProposals::default(); - let installs = AtomicUsize::new(0); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(!agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - } - - #[test] - fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { - let enabled = AtomicBool::new(false); - let mut settings = std::collections::HashMap::new(); - settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(enabled.load(Ordering::Relaxed)); - } - - #[test] - fn apply_ocsf_json_setting_disables_when_setting_is_unset() { - let enabled = AtomicBool::new(true); - let settings = std::collections::HashMap::new(); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(!enabled.load(Ordering::Relaxed)); - } - - #[test] - fn agent_proposals_setting_enables_from_initial_settings_snapshot() { - let mut settings = std::collections::HashMap::new(); - settings.insert( - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), - effective_bool(true), - ); - - assert!(agent_proposals_enabled_from_settings(&settings)); - } - - #[test] - fn agent_proposals_setting_defaults_false_when_unset() { - let settings = std::collections::HashMap::new(); - - assert!(!agent_proposals_enabled_from_settings(&settings)); - } - - // ---- Policy disk discovery tests ---- - - #[test] - fn discover_policy_from_nonexistent_path_returns_restrictive_default() { - let path = std::path::Path::new("/nonexistent/policy.yaml"); - let policy = discover_policy_from_path(path); - // Restrictive default has no network policies. - assert!(policy.network_policies.is_empty()); - // But does have filesystem and process policies. - assert!(policy.filesystem.is_some()); - assert!(policy.process.is_some()); - } - - #[test] - fn discover_policy_from_valid_yaml_file() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr - read_write: - - /tmp -network_policies: - test: - name: test - endpoints: - - { host: example.com, port: 443 } - binaries: - - { path: /usr/bin/curl } -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - assert_eq!(policy.network_policies.len(), 1); - assert!(policy.network_policies.contains_key("test")); - let fs = policy.filesystem.unwrap(); - assert!(!fs.include_workdir); - } - - #[test] - fn discover_policy_from_invalid_yaml_returns_restrictive_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default. - assert!(policy.network_policies.is_empty()); - assert!(policy.filesystem.is_some()); - } - - #[test] - fn discover_policy_from_unsafe_yaml_falls_back_to_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -process: - run_as_user: root - run_as_group: root -filesystem_policy: - include_workdir: true - read_only: - - /usr - read_write: - - /tmp -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default because of root user. - let proc = policy.process.unwrap(); - assert_eq!(proc.run_as_user, "sandbox"); - assert_eq!(proc.run_as_group, "sandbox"); - } - - #[test] - fn discover_policy_restrictive_default_blocks_network() { - // In cluster mode we keep proxy mode enabled so `inference.local` - // can always be routed through proxy/OPA controls. - let proto = openshell_policy::restrictive_default_policy(); - let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); - assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); - } - - // ---- Initial policy acknowledgement tests ---- - - fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::restrictive_default_policy() - } - - fn settings_poll_result( - policy: Option, - version: u32, - source: openshell_core::proto::PolicySource, - ) -> openshell_core::grpc_client::SettingsPollResult { - openshell_core::grpc_client::SettingsPollResult { - policy, - version, - policy_hash: format!("hash-v{version}"), - config_revision: u64::from(version) * 100, - policy_source: source, - settings: std::collections::HashMap::new(), - global_policy_version: 0, - provider_env_revision: 0, - supervisor_middleware_services: Vec::new(), - workspace: String::new(), - } - } - - #[tokio::test] - async fn failed_external_startup_registry_build_preserves_installed_builtins() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let builtins_generation = engine.current_generation(); - assert_eq!(builtins_generation, 1); - - let invalid_external = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_body_bytes: 1024, - ..Default::default() - }; - connect_middleware_registry(&[invalid_external]) - .await - .expect_err("unavailable external service must not replace built-ins"); - - assert_eq!(engine.current_generation(), builtins_generation); - } - - #[test] - fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { - let services = Vec::new(); - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - } - - #[test] - fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &no_services, - &no_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &no_services, - &desired_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - false, - "local-policy", - "hash-v2", - &no_services, - &desired_services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - } - - #[test] - fn policy_only_change_does_not_rebuild_middleware_registry() { - let services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - // The runtime must reconcile, but the registry (and therefore - // middleware reachability) is not part of that reconciliation. - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &services, - &services, - )); - } - - #[test] - fn registry_rebuild_requires_service_set_change_or_degraded_registry() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &no_services, - &desired_services, - )); - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::NeedsReconciliation, - &desired_services, - &desired_services, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &desired_services, - &desired_services, - )); - } - - #[test] - fn initial_ack_candidate_matches_sandbox_revision() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) - .expect("sandbox-sourced matching revision should be acknowledged"); - - assert_eq!(ack.version, 2); - assert_eq!(ack.policy_hash, "hash-v2"); - assert_eq!(ack.config_revision, 200); - } - - #[test] - fn initial_ack_candidate_ignores_global_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Global, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_version_zero() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 0, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_local_file_mode() { - // Local-file mode retains no proto policy, so there is nothing to - // acknowledge to the gateway. - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(None, &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_rejects_mismatched_identity() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let mut newer = proto_policy_fixture(); - newer.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule::default(), - ); - let canonical = - settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); - let canonical = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "hash-provider-change".to_string(), - config_revision: loaded.config_revision + 1, - ..canonical - }; - - assert_eq!( - initial_poll_disposition( - &LoadedPolicyOrigin::Gateway { - revision: Some(loaded), - }, - &canonical, - ), - InitialPollDisposition::Reconcile - ); - } - - #[test] - fn initial_poll_tracks_local_override_without_reconciliation() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert_eq!( - initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), - InitialPollDisposition::TrackOnly - ); - assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); - } - - #[test] - fn initial_poll_reconciles_unbound_gateway_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let origin = LoadedPolicyOrigin::Gateway { revision: None }; - - assert_eq!( - initial_poll_disposition(&origin, &canonical), - InitialPollDisposition::Reconcile - ); - assert!(origin.allows_gateway_policy_reload()); - } - - #[test] - fn policy_status_outbox_preserves_all_revision_order() { - let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - for version in 1..=128 { - enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); - } - - for version in 1..=128 { - assert_eq!( - receiver.try_recv().unwrap(), - PolicyStatusUpdate::loaded(version) - ); - } - } - - #[test] - fn settings_snapshot_carries_workspace_for_policy_sync() { - let mut snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - snapshot.workspace = "beta".to_string(); - - let revision = LoadedPolicyRevision::from_snapshot(&snapshot); - assert_eq!(revision.version, 1); - assert_eq!( - snapshot.workspace, "beta", - "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" - ); - } -} +#[cfg(not(target_os = "windows"))] +include!("lib_unix.rs"); diff --git a/crates/openshell-sandbox/src/lib_unix.rs b/crates/openshell-sandbox/src/lib_unix.rs new file mode 100644 index 0000000000..e696403f39 --- /dev/null +++ b/crates/openshell-sandbox/src/lib_unix.rs @@ -0,0 +1,3773 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `OpenShell` Sandbox library. +//! +//! This crate provides process sandboxing and monitoring capabilities. + +mod activity_aggregator; +mod denial_aggregator; +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +mod google_cloud_metadata; +mod mechanistic_mapper; +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +mod metadata_server; +mod sidecar_control; + +use miette::{IntoDiagnostic, Result, WrapErr}; +use std::future::Future; +use std::sync::Arc; +#[cfg(target_os = "linux")] +use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::time::Duration; +use tracing::{debug, info, warn}; + +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, + DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, +}; + +// --------------------------------------------------------------------------- +// OCSF Context +// --------------------------------------------------------------------------- +// +// The following log sites intentionally remain as plain `tracing` macros +// and are NOT migrated to OCSF builders: +// +// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) +// - Transient "about to do X" events where the result is logged separately +// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") +// - Internal SSH channel warnings (unknown channel, PTY resize failures) +// - Denial flush telemetry (the individual denials are already OCSF events) +// - Status reporting failures (sync to gateway, non-actionable) +// - Route refresh interval validation warnings +// +// These are operational plumbing that don't represent security decisions, +// policy changes, or observable sandbox behavior worth structuring. +// --------------------------------------------------------------------------- + +/// Re-export the process-wide OCSF sandbox context getter. +/// +/// The singleton lives in `openshell-ocsf` so both supervisor leaves can +/// reach it without depending on `openshell-sandbox`. Initialised once during +/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; + +use openshell_core::denial::DenialEvent; +use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_supervisor_network::opa::OpaEngine; +use openshell_supervisor_process::process::ProcessEnforcementMode; +pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; +use openshell_supervisor_process::skills; +use tokio::sync::mpsc::UnboundedSender; +#[cfg(any(test, target_os = "linux"))] +use tokio::time::timeout; + +const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; +const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; +const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; +const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; + +/// Run a command in the sandbox. +/// +/// # Errors +/// +/// Returns an error if the command fails to start or encounters a fatal error. +#[allow( + clippy::too_many_arguments, + clippy::similar_names, + clippy::fn_params_excessive_bools +)] +pub async fn run_sandbox( + command: Vec, + workdir: Option, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, + ssh_socket_path: Option, + _health_check: bool, + _health_port: u16, + inference_routes: Option, + ocsf_enabled: Arc, + network_enabled: bool, + process_enabled: bool, + upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, +) -> Result { + let (program, args) = command + .split_first() + .ok_or_else(|| miette::miette!("No command specified"))?; + + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; they are only significant for network + // events which happen after the netns is created. + { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + + if !openshell_ocsf::ctx::set_ctx(SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + }) { + debug!("OCSF context already initialized, keeping existing"); + } + } + + let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); + let process_enforcement_mode = process_enforcement_mode(); + let process_uses_sidecar_control = + process_enabled && !network_enabled && sidecar_network_enforcement; + let mut process_control_connection = None; + let sidecar_bootstrap = if process_uses_sidecar_control { + let socket = sidecar_control_socket().ok_or_else(|| { + miette::miette!( + "{} is required for process-only sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET + ) + })?; + let (bootstrap, connection) = sidecar_control::connect_process_client( + &socket, + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), + ) + .await?; + process_control_connection = Some(connection); + Some(bootstrap) + } else { + None + }; + + // Load policy and initialize OPA engine + let openshell_endpoint_for_proxy = openshell_endpoint.clone(); + let sandbox_name_for_agg = sandbox.clone(); + let ( + mut policy, + opa_engine, + retained_proto, + middleware_registry_status, + loaded_policy_origin, + initial_agent_proposals_enabled, + ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let (policy, opa_engine, retained_proto, loaded_policy_origin) = + load_policy_from_sidecar_bootstrap(bootstrap)?; + ( + policy, + opa_engine, + retained_proto, + MiddlewareRegistryStatus::Synchronized, + loaded_policy_origin, + bootstrap.agent_proposals_enabled, + ) + } else { + load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + ) + .await? + }; + + // Override the policy's process identity with the driver-resolved UID/GID + // from the pod environment. The policy defaults to the name "sandbox" which + // resolves via /etc/passwd, but the driver may have chosen a different + // numeric UID (e.g. from OpenShift SCC annotations). + // Validate overrides against the same rules as the policy layer to prevent + // env-injected values (e.g. GID 0) from bypassing policy restrictions. + if let Ok(uid) = std::env::var(openshell_core::sandbox_env::SANDBOX_UID) + && !uid.is_empty() + { + if !openshell_policy::is_valid_sandbox_identity(&uid) { + return Err(miette::miette!( + "OPENSHELL_SANDBOX_UID contains invalid sandbox identity '{uid}'; \ + expected 'sandbox' or a numeric UID in range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID, + )); + } + policy.process.run_as_user = Some(uid); + } + if let Ok(gid) = std::env::var(openshell_core::sandbox_env::SANDBOX_GID) + && !gid.is_empty() + { + if !openshell_policy::is_valid_sandbox_identity(&gid) { + return Err(miette::miette!( + "OPENSHELL_SANDBOX_GID contains invalid sandbox identity '{gid}'; \ + expected 'sandbox' or a numeric GID in range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID, + )); + } + policy.process.run_as_group = Some(gid); + } + + #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] + let (provider_credentials, mut provider_env) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + bootstrap.provider_env_revision, + bootstrap.provider_child_env.clone(), + ); + (provider_credentials, bootstrap.provider_child_env.clone()) + } else { + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .message(format!( + "Failed to fetch provider environment, continuing without: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + }; + + let provider_credentials = ProviderCredentialState::from_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ); + let provider_env = provider_credentials.child_env_with_gcp_resolved(); + (provider_credentials, provider_env) + }; + + // Shared agent-proposals feature flag. Seed from the same initial settings + // snapshot that produced the policy so networking and process setup agree + // before the poll loop starts reconciling later changes. + let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); + + let process_control_writer = process_control_connection + .as_ref() + .map(|connection| connection.writer.clone()); + let mut process_control_closed = None; + if let Some(connection) = process_control_connection { + process_control_closed = Some(connection.closed); + spawn_sidecar_control_update_watcher( + connection.updates, + provider_credentials.clone(), + agent_proposals.clone(), + ); + } + + // Shared PID: set after process spawn so the proxy can look up + // the entrypoint process's /proc/net/tcp for identity binding. + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + + // Create the workload's network namespace. It is shared infrastructure: + // the proxy binds to its host-side veth IP, the bypass monitor reads + // /dev/kmsg from inside it, and the workload child / SSH sessions enter + // it via setns(). The RAII handle lives in this frame for the duration + // of the sandbox. + #[cfg(target_os = "linux")] + let netns = if network_enabled && !sidecar_network_enforcement { + openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? + } else { + None + }; + + // The denial channel is owned by the orchestrator: the proxy (in the + // networking leaf) and the bypass monitor (in the process leaf) both + // produce DenialEvents that the denial aggregator (orchestrator-side) + // consumes via the matching receiver. Both leaves are pure producers; + // the orchestrator owns the consumer task spawned below. + let (denial_tx, denial_rx, bypass_denial_tx): ( + Option>, + _, + Option>, + ) = if sandbox_id.is_some() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let bypass_tx = tx.clone(); + (Some(tx), Some(rx), Some(bypass_tx)) + } else { + (None, None, None) + }; + #[cfg(not(target_os = "linux"))] + drop(bypass_denial_tx); + + // Anonymous activity channel: same orchestrator-owned pattern as the + // denial channel. The proxy and the bypass monitor both emit per-event + // activity records; the orchestrator-side aggregator drains, sanitizes, + // and flushes anonymous summaries to the gateway. + let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { + let (tx, rx) = + tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); + let bypass_tx = tx.clone(); + (Some(tx), Some(rx), Some(bypass_tx)) + } else { + (None, None, None) + }; + #[cfg(not(target_os = "linux"))] + drop(bypass_activity_tx); + + // Workspace watch: the policy poll loop learns the workspace from + // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local + // API read the current value so proposals target the correct workspace. + let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); + + let networking = if network_enabled { + #[cfg(target_os = "linux")] + let proxy_bind_ip = netns + .as_ref() + .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); + #[cfg(not(target_os = "linux"))] + let proxy_bind_ip: Option = None; + + Some( + openshell_supervisor_network::run::run_networking( + &policy, + proxy_bind_ip, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + process_enabled, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + ) + .await?, + ) + } else { + None + }; + + #[cfg(target_os = "linux")] + let sidecar_control_server = if network_enabled && sidecar_network_enforcement { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Err(miette::miette!( + "sidecar network enforcement requires proxy network mode" + )); + } + let socket = sidecar_control_socket().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET + ) + })?; + let proto = retained_proto.as_ref().ok_or_else(|| { + miette::miette!( + "sidecar topology requires gateway policy data for the process supervisor" + ) + })?; + let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); + Some(sidecar_control::spawn_server( + &socket, + sidecar_control::BootstrapData { + policy_proto: proto.clone(), + provider_env_revision: provider_credentials.snapshot().revision, + provider_child_env: provider_env.clone(), + agent_proposals_enabled: agent_proposals.enabled(), + proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), + proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), + }, + sidecar_expected_peer()?, + )?) + } else { + None + }; + #[cfg(not(target_os = "linux"))] + let sidecar_control_server: Option = None; + + let sidecar_control_publisher = sidecar_control_server + .as_ref() + .map(sidecar_control::ServerHandle::publisher); + + #[cfg(target_os = "linux")] + let mut sidecar_control_task = None; + + #[cfg(target_os = "linux")] + if network_enabled + && sidecar_network_enforcement + && let Some(server) = sidecar_control_server + { + let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar network topology", + openshell_core::sandbox_env::SSH_SOCKET_PATH + ) + })?; + let (entrypoint_rx, connection_task) = server.into_runtime_parts(); + sidecar_control_task = Some(connection_task); + spawn_sidecar_entrypoint_handler( + entrypoint_rx, + entrypoint_pid.clone(), + opa_engine.clone(), + retained_proto.clone(), + openshell_endpoint.clone(), + sandbox_id.clone(), + std::path::PathBuf::from(trusted_ssh_socket_path), + ); + } + + #[cfg(not(target_os = "linux"))] + if network_enabled && sidecar_network_enforcement { + return Err(miette::miette!( + "sidecar network enforcement is only supported on Linux" + )); + } + + // Spawn the denial-aggregator flush task. The aggregator drains denial + // events from the proxy + bypass monitor, batches them, and ships + // summaries to the gateway via `SubmitPolicyAnalysis`. + if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall + // back to the ID when the name isn't set. + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + + let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); + let denial_workspace_gate = workspace_rx.clone(); + let denial_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + |summaries| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = denial_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_proposals_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summaries, + ) + .await + { + warn!(error = %e, "Failed to flush denial summaries to gateway"); + } + } + }, + move || !denial_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn the activity-aggregator flush task. The aggregator drains + // anonymous activity events from the proxy, sanitizes deny groups, + // and ships periodic summaries to the gateway. + if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( + std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") + .ok() + .as_deref(), + ); + + let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); + let activity_workspace_gate = workspace_rx.clone(); + let activity_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + move |summary| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = activity_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_activity_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summary, + ) + .await + { + warn!(error = %e, "Failed to flush activity summary to gateway"); + } + } + }, + move || !activity_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn background policy poll task (gRPC mode only). + if !process_uses_sidecar_control + && let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) + { + let poll_id = id.to_string(); + let poll_endpoint = endpoint.to_string(); + let poll_engine = engine.clone(); + let poll_ocsf_enabled = ocsf_enabled.clone(); + let poll_pid = entrypoint_pid.clone(); + let poll_provider_credentials = provider_credentials.clone(); + let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); + let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let poll_ctx = PolicyPollLoopContext { + endpoint: poll_endpoint, + sandbox_id: poll_id, + opa_engine: poll_engine, + loaded_policy_origin, + entrypoint_pid: poll_pid, + interval_secs: poll_interval_secs, + ocsf_enabled: poll_ocsf_enabled, + provider_credentials: poll_provider_credentials, + policy_local_ctx: poll_policy_local, + agent_proposals: agent_proposals.clone(), + middleware_registry_status, + sidecar_control_publisher: sidecar_control_publisher.clone(), + workspace_tx, + }; + + tokio::spawn(async move { + if let Err(e) = run_policy_poll_loop(poll_ctx).await { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .message(format!("Policy poll loop exited with error: {e}")) + .build() + ); + } + }); + } + + // Start GCE metadata loopback server inside the network namespace so + // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) + // can reach it via direct TCP. Must start before the process leaf so SSH + // sessions also see corrected env vars on bind failure. + #[cfg(target_os = "linux")] + if let Some(ns) = netns.as_ref() + && provider_credentials + .snapshot() + .child_env + .contains_key("GCE_METADATA_HOST") + { + let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + match ns + .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) + .await + { + Ok(listener) => { + tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); + if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { + info!(addr = %addr, "GCE metadata loopback server ready"); + } else { + warn!("GCE metadata server failed to become ready, removing metadata env vars"); + provider_env.remove("GCE_METADATA_HOST"); + provider_env.remove("GCE_METADATA_IP"); + provider_env.remove("METADATA_SERVER_DETECTION"); + provider_credentials.remove_env_key("GCE_METADATA_HOST"); + } + } + Err(e) => { + warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); + provider_env.remove("GCE_METADATA_HOST"); + provider_env.remove("GCE_METADATA_IP"); + provider_env.remove("METADATA_SERVER_DETECTION"); + provider_credentials.remove_env_key("GCE_METADATA_HOST"); + } + } + } + + let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; + let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { + bootstrap + .proxy_ca_cert_path + .clone() + .zip(bootstrap.proxy_ca_bundle_path.clone()) + }); + + let exit_code = if process_enabled { + let ca_file_paths = networking + .as_ref() + .and_then(|n| n.ca_file_paths.clone()) + .or_else(|| { + if sidecar_network_enforcement { + sidecar_bootstrap_ca_file_paths + .clone() + .or_else(sidecar_ca_file_paths) + } else { + None + } + }); + + let entrypoint_started_tx = + if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + match rx.await { + Ok(pid) => { + if let Err(err) = + sidecar_control::send_entrypoint_started(&writer, pid).await + { + warn!(error = %err, "Failed to send sidecar entrypoint event"); + } + } + Err(_closed) => { + debug!("Entrypoint exited before sidecar entrypoint event was sent"); + } + } + }); + Some(tx) + } else { + None + }; + + let process = openshell_supervisor_process::run::run_process( + program, + args, + workdir.as_deref(), + timeout_secs, + interactive, + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path, + sidecar_network_enforcement, + &process_policy, + process_enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + provider_credentials, + provider_env, + ca_file_paths, + agent_proposals.clone(), + #[cfg(target_os = "linux")] + netns.as_ref(), + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + ); + + if let Some(control_closed) = process_control_closed.as_mut() { + tokio::select! { + result = process => result?, + _ = control_closed => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Authoritative network-sidecar control channel closed; terminating process container" + ) + .build() + ); + return Err(miette::miette!( + "authoritative network-sidecar control channel closed" + )); + } + } + } else { + process.await? + } + } else { + // Network-only sidecar mode: keep the proxy and its background + // tasks alive (held via the `networking` value) until shutdown. If the + // sole authenticated process-supervisor control connection closes, + // exit non-zero so Kubernetes restarts the network sidecar and creates + // a fresh one-client bootstrap listener for the restarted agent. + #[cfg(target_os = "linux")] + if let Some(control_task) = sidecar_control_task { + tokio::select! { + () = wait_for_shutdown_signal() => 0, + result = control_task => { + warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); + 1 + } + } + } else { + wait_for_shutdown_signal().await; + 0 + } + #[cfg(not(target_os = "linux"))] + { + wait_for_shutdown_signal().await; + 0 + } + }; + + // Drop networking explicitly so the proxy + bypass monitor RAII + // handles tear down before we return. + drop(networking); + + Ok(exit_code) +} + +/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is +/// no entrypoint child whose lifetime drives the supervisor's exit. +async fn wait_for_shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut sigterm = match signal(SignalKind::terminate()) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + error = %e, + "Failed to install SIGTERM handler; waiting on SIGINT only" + ); + let _ = tokio::signal::ctrl_c().await; + return; + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => { + info!("Received SIGINT, shutting down network-only supervisor"); + } + _ = sigterm.recv() => { + info!("Received SIGTERM, shutting down network-only supervisor"); + } + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + info!("Received Ctrl-C, shutting down network-only supervisor"); + } +} + +fn sidecar_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) + .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) +} + +fn process_enforcement_mode() -> ProcessEnforcementMode { + match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) + .ok() + .as_deref() + { + Some("sidecar") => ProcessEnforcementMode::NetworkOnly, + _ => ProcessEnforcementMode::Full, + } +} + +fn sidecar_control_socket() -> Option { + std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) + .ok() + .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn sidecar_expected_peer() -> Result { + fn required_numeric_env(name: &str) -> Result { + let value = std::env::var(name) + .into_diagnostic() + .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; + value.parse::().into_diagnostic().wrap_err_with(|| { + format!("{name} must be a numeric ID for sidecar control authentication") + }) + } + + Ok(sidecar_control::ExpectedPeer { + uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, + gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, + }) +} + +type LoadedPolicyBundle = ( + SandboxPolicy, + Option>, + Option, + LoadedPolicyOrigin, +); + +fn load_policy_from_sidecar_bootstrap( + bootstrap: &sidecar_control::BootstrapData, +) -> Result { + let proto = bootstrap.policy_proto.clone(); + let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); + let policy = SandboxPolicy::try_from(proto.clone())?; + info!("Loaded sidecar policy from control socket bootstrap"); + Ok(( + policy, + opa_engine, + Some(proto), + LoadedPolicyOrigin::Gateway { revision: None }, + )) +} + +fn spawn_sidecar_control_update_watcher( + mut updates: tokio::sync::mpsc::UnboundedReceiver, + provider_credentials: ProviderCredentialState, + agent_proposals: AgentProposals, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + while let Some(update) = updates.recv().await { + match update { + sidecar_control::ControlUpdate::ProviderEnv { + revision, + provider_child_env, + } => { + if revision <= provider_credentials.snapshot().revision { + continue; + } + let env_count = provider_credentials + .install_child_env_snapshot(revision, provider_child_env); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("provider_env_revision", serde_json::json!(revision)) + .message(format!( + "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" + )) + .build() + ); + } + sidecar_control::ControlUpdate::Policy { + policy_proto, + policy_hash, + config_revision, + } => { + debug!( + version = policy_proto.version, + policy_hash, + config_revision, + "Received sidecar policy update for process supervisor" + ); + } + sidecar_control::ControlUpdate::AgentProposals { + enabled, + config_revision, + } => { + apply_agent_proposals_enabled( + &agent_proposals, + enabled, + "sidecar control", + Some(config_revision), + None, + skills::install_static_skills, + ); + } + } + } + }) +} + +#[cfg(target_os = "linux")] +fn spawn_sidecar_entrypoint_handler( + mut entrypoint_rx: tokio::sync::mpsc::Receiver, + entrypoint_pid: Arc, + opa_engine: Option>, + retained_proto: Option, + openshell_endpoint: Option, + sandbox_id: Option, + trusted_ssh_socket_path: std::path::PathBuf, +) { + tokio::spawn(async move { + let mut session_started = false; + let mut trusted_supervisor_pid = None; + let terminating = Arc::new(AtomicBool::new(false)); + while let Some(started) = entrypoint_rx.recv().await { + entrypoint_pid.store(started.pid, Ordering::Release); + if started.start_session { + info!( + pid = started.pid, + ssh_socket = %trusted_ssh_socket_path.display(), + "Sidecar process supervisor reported entrypoint start" + ); + } else { + trusted_supervisor_pid = Some(started.pid); + info!( + pid = started.pid, + "Sidecar process supervisor reported initial process anchor" + ); + } + + if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { + match engine.reload_from_proto_with_pid(proto, started.pid) { + Ok(()) => info!( + pid = started.pid, + "Policy binary symlink resolution complete for sidecar process anchor" + ), + Err(err) => warn!( + error = %err, + pid = started.pid, + "Failed to rebuild OPA engine with sidecar process anchor PID" + ), + } + } + + if started.start_session + && !session_started + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_ref(), sandbox_id.as_ref()) + { + let Some(supervisor_pid) = trusted_supervisor_pid else { + warn!( + pid = started.pid, + "Ignoring sidecar entrypoint event before authenticated supervisor anchor" + ); + continue; + }; + openshell_supervisor_process::supervisor_session::spawn( + endpoint.clone(), + id.clone(), + trusted_ssh_socket_path.clone(), + None, + Some(supervisor_pid), + Arc::clone(&terminating), + ); + session_started = true; + info!("sidecar supervisor session task spawned"); + } + } + terminating.store(true, Ordering::Release); + }); +} + +fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { + let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) + .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); + let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); + let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); + (cert.exists() && bundle.exists()).then_some((cert, bundle)) +} + +fn process_policy_for_topology( + policy: &SandboxPolicy, + sidecar_network_enforcement: bool, +) -> Result { + let mut process_policy = policy.clone(); + if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { + let proxy = process_policy + .network + .proxy + .get_or_insert(ProxyPolicy { http_addr: None }); + if proxy.http_addr.is_none() { + proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); + } + } + Ok(process_policy) +} + +/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. +async fn flush_proposals_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summaries: Vec, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialSummary, L7RequestSample}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summaries: Vec = summaries + .into_iter() + .map(|s| DenialSummary { + sandbox_id: String::new(), + host: s.host, + port: u32::from(s.port), + binary: s.binary, + ancestors: s.ancestors, + deny_reason: s.deny_reason, + first_seen_ms: s.first_seen_ms, + last_seen_ms: s.last_seen_ms, + count: s.count, + suppressed_count: 0, + total_count: s.count, + sample_cmdlines: s.sample_cmdlines, + binary_sha256: String::new(), + persistent: false, + denial_stage: s.denial_stage, + l7_request_samples: s + .l7_samples + .into_iter() + .map(|l| L7RequestSample { + method: l.method, + path: l.path, + decision: "deny".to_string(), + count: l.count, + }) + .collect(), + l7_inspection_active: false, + }) + .collect(); + + // Run the mechanistic mapper sandbox-side to generate proposals. + // The gateway is a thin persistence + validation layer — it never + // generates proposals itself. + let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); + + info!( + sandbox_name = %sandbox_name, + summaries = proto_summaries.len(), + proposals = proposals.len(), + "Flushed denial analysis to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + proto_summaries, + proposals, + Vec::new(), + "mechanistic", + ) + .await?; + + Ok(()) +} + +/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. +async fn flush_activity_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summary: activity_aggregator::FlushableActivitySummary, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summary = NetworkActivitySummary { + network_activity_count: summary.network_activity_count, + denied_action_count: summary.denied_action_count, + denials_by_group: summary + .denials_by_group + .into_iter() + .map(|(group, count)| DenialGroupCount { + deny_group: group, + denied_count: count, + }) + .collect(), + }; + + info!( + sandbox_name = %sandbox_name, + network_activity_count = proto_summary.network_activity_count, + denied_action_count = proto_summary.denied_action_count, + "Flushed activity summary to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + Vec::new(), + Vec::new(), + vec![proto_summary], + "activity", + ) + .await?; + + Ok(()) +} + +// ============================================================================ +// Baseline filesystem path enrichment +// ============================================================================ + +/// Minimum read-only paths required for a proxy-mode sandbox child process to +/// function: dynamic linker, shared libraries, DNS resolution, CA certs, +/// Python venv, openshell logs, process info, and random bytes. +/// +/// `/proc` and `/dev/urandom` are included here for the same reasons they +/// appear in `restrictive_default_policy()`: virtually every process needs +/// them. Before the Landlock per-path fix (#677) these were effectively free +/// because a missing path silently disabled the entire ruleset; now they must +/// be explicit. +const PROXY_BASELINE_READ_ONLY: &[&str] = &[ + "/usr", + "/lib", + "/etc", + "/app", + "/var/log", + "/proc", + "/dev/urandom", +]; + +/// Minimum read-write paths required for a proxy-mode sandbox child process: +/// user working directory and temporary files. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; + +/// GPU read-only paths. +/// +/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced +/// socket at init time. If the directory exists but Landlock denies traversal +/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` +/// even though the daemon is optional. Only read/traversal access is needed. +/// +/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, +/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` +/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may +/// not be covered by the parent-directory Landlock rule when the mount crosses +/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal +/// is permitted regardless of Landlock's cross-mount behaviour. +const GPU_BASELINE_READ_ONLY: &[&str] = &[ + "/run/nvidia-persistenced", + "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory +]; + +/// GPU read-write paths (static). +/// +/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, +/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native +/// Linux. Landlock restricts `open(2)` on device files even when DAC allows +/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. +/// These devices do not exist on WSL2 and will be skipped by the existence +/// check in `enrich_proto_baseline_paths()`. +/// +/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver +/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects +/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux +/// and will be skipped there by the existence check. +/// +/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` +/// to set thread names. Without write access, `cuInit()` returns error 304. +/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to +/// inodes and child processes have different procfs inodes than the parent. +/// +/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by +/// `enumerate_gpu_device_nodes()` since the count varies. +const GPU_BASELINE_READ_WRITE: &[&str] = &[ + "/dev/nvidiactl", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia-modeset", + "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) + "/proc", +]; + +/// Returns true if GPU devices are present in the container. +/// +/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and +/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these +/// depending on the host kernel; the other will not exist. +fn has_gpu_devices() -> bool { + std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() +} + +/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). +fn enumerate_gpu_device_nodes() -> Vec { + let mut paths = Vec::new(); + if let Ok(entries) = std::fs::read_dir("/dev") { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(suffix) = name.strip_prefix("nvidia") { + if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { + continue; + } + paths.push(entry.path().to_string_lossy().into_owned()); + } + } + } + paths +} + +fn push_unique(paths: &mut Vec, path: String) { + if !paths.iter().any(|p| p == &path) { + paths.push(path); + } +} + +fn collect_baseline_enrichment_paths( + include_proxy: bool, + include_gpu: bool, + gpu_device_nodes: Vec, +) -> (Vec, Vec) { + let mut ro = Vec::new(); + let mut rw = Vec::new(); + + if include_proxy { + for &path in PROXY_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in PROXY_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + } + + if include_gpu { + for &path in GPU_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in GPU_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + for path in gpu_device_nodes { + push_unique(&mut rw, path); + } + } + + // A path promoted to read_write (e.g. /proc for GPU) should not also + // appear in read_only — Landlock handles the overlap correctly but the + // duplicate is confusing when inspecting the effective policy. + ro.retain(|p| !rw.contains(p)); + + (ro, rw) +} + +fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { + let include_gpu = has_gpu_devices(); + let gpu_device_nodes = if include_gpu { + enumerate_gpu_device_nodes() + } else { + Vec::new() + }; + collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) +} + +/// Collect all active baseline paths for tests and diagnostics. +/// Returns `(read_only, read_write)` as owned `String` vecs. +#[cfg(test)] +fn baseline_enrichment_paths() -> (Vec, Vec) { + active_baseline_enrichment_paths(true) +} + +fn enrich_proto_baseline_paths_with( + proto: &mut openshell_core::proto::SandboxPolicy, + ro: &[String], + rw: &[String], + path_exists: F, +) -> bool +where + F: Fn(&str) -> bool, +{ + if ro.is_empty() && rw.is_empty() { + return false; + } + + let fs = proto + .filesystem + .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { + include_workdir: true, + ..Default::default() + }); + + let mut modified = false; + for path in ro { + if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { + if !path_exists(path) { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + fs.read_only.push(path.clone()); + modified = true; + } + } + for path in rw { + if fs.read_write.iter().any(|p| p == path) { + continue; + } + if !path_exists(path) { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + if fs.read_only.iter().any(|p| p == path) { + if path == "/proc" { + info!( + path, + "Promoting /proc from read-only to read-write for GPU runtime compatibility" + ); + fs.read_only.retain(|p| p != path); + fs.read_write.push(path.clone()); + modified = true; + } + continue; + } + fs.read_write.push(path.clone()); + modified = true; + } + + modified +} + +/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths +/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if +/// missing; user-specified paths are never removed. +/// +/// Returns `true` if the policy was modified (caller may want to sync back). +fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); + + // Baseline paths are system-injected, not user-specified. Skip paths + // that do not exist in this container image to avoid noisy warnings from + // Landlock and, more critically, to prevent a single missing baseline + // path from abandoning the entire Landlock ruleset under best-effort + // mode (see issue #664). + let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { + std::path::Path::new(path).exists() + }); + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } + + modified +} + +fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + openshell_policy::strip_provider_rule_names(proto) +} + +fn proto_sync_payload_for_enriched_policy( + proto: &openshell_core::proto::SandboxPolicy, + enriched: bool, +) -> Option { + if !enriched { + return None; + } + + let mut sync_policy = proto.clone(); + strip_proto_provider_policy_entries(&mut sync_policy); + Some(sync_policy) +} + +/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem +/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the +/// local-file code path where no proto is available. +fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { + let (ro, rw) = + active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); + if ro.is_empty() && rw.is_empty() { + return; + } + + let mut modified = false; + for path in &ro { + let p = std::path::PathBuf::from(path); + if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { + if !p.exists() { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_only.push(p); + modified = true; + } + } + for path in &rw { + let p = std::path::PathBuf::from(path); + if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { + continue; + } + if !p.exists() { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_write.push(p); + modified = true; + } + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod baseline_tests { + use super::*; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; + + #[test] + fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { + // When GPU devices are present, /proc is promoted to read_write + // (CUDA needs to write /proc//task//comm). It should + // NOT also appear in read_only. + if !has_gpu_devices() { + // Can't test GPU dedup without GPU devices; skip silently. + return; + } + let (ro, rw) = baseline_enrichment_paths(); + assert!( + rw.contains(&"/proc".to_string()), + "/proc should be in read_write when GPU is present" + ); + assert!( + !ro.contains(&"/proc".to_string()), + "/proc should NOT be in read_only when it is already in read_write" + ); + } + + #[test] + fn proc_in_read_only_without_gpu() { + if has_gpu_devices() { + // On a GPU host we can't test the non-GPU path; skip silently. + return; + } + let (ro, _rw) = baseline_enrichment_paths(); + assert!( + ro.contains(&"/proc".to_string()), + "/proc should be in read_only when GPU is not present" + ); + } + + #[test] + fn baseline_read_write_always_includes_sandbox_and_tmp() { + let (_ro, rw) = baseline_enrichment_paths(); + assert!(rw.contains(&"/sandbox".to_string())); + assert!(rw.contains(&"/tmp".to_string())); + } + + #[test] + fn enumerate_gpu_device_nodes_skips_bare_nvidia() { + // "nvidia" (without a trailing digit) is a valid /dev entry on some + // systems but is not a per-GPU device node. The enumerator must + // not match it. + let nodes = enumerate_gpu_device_nodes(); + assert!( + !nodes.contains(&"/dev/nvidia".to_string()), + "bare /dev/nvidia should not be enumerated: {nodes:?}" + ); + } + + #[test] + fn no_duplicate_paths_in_baseline() { + let (ro, rw) = baseline_enrichment_paths(); + // No path should appear in both lists. + for path in &ro { + assert!( + !rw.contains(path), + "path {path} appears in both read_only and read_write" + ); + } + } + + #[test] + fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + read_only: vec!["/tmp".to_string()], + read_write: vec![], + include_workdir: false, + }); + policy.network_policies.insert( + "test".into(), + openshell_core::proto::NetworkPolicyRule { + name: "test-rule".into(), + endpoints: vec![openshell_core::proto::NetworkEndpoint { + host: "example.com".into(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + + enrich_proto_baseline_paths(&mut policy); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + filesystem.read_only.contains(&"/tmp".to_string()), + "explicit read_only baseline path should be preserved" + ); + assert!( + !filesystem.read_write.contains(&"/tmp".to_string()), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + assert!(strip_proto_provider_policy_entries(&mut policy)); + assert!( + !policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(policy.network_policies.contains_key("sandbox_only")); + assert!(!strip_proto_provider_policy_entries(&mut policy)); + } + + #[test] + fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + + assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "provider-derived rules alone must not trigger sync or mutate runtime policy" + ); + } + + #[test] + fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + runtime_policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) + .expect("enrichment should create a sync payload"); + + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "runtime policy must retain provider-derived rules for OPA input" + ); + assert!( + !sync_policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(sync_policy.network_policies.contains_key("sandbox_only")); + } + + #[test] + fn proto_gpu_enrichment_promotes_proc_without_network_policy() { + let mut policy = openshell_policy::restrictive_default_policy(); + assert!( + policy.network_policies.is_empty(), + "regression setup must exercise the no-network default path" + ); + let (ro, rw) = + collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); + + let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { + matches!(path, "/proc" | "/dev/nvidia0") + }); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + enriched, + "GPU enrichment should not require network policies" + ); + assert!( + filesystem.read_write.contains(&"/dev/nvidia0".to_string()), + "GPU enrichment should add enumerated device nodes without network policies" + ); + assert!( + !filesystem.read_only.contains(&"/proc".to_string()), + "GPU enrichment should remove /proc from read_only" + ); + assert!( + filesystem.read_write.contains(&"/proc".to_string()), + "GPU enrichment should promote /proc to read_write" + ); + } + + #[test] + fn gpu_baseline_read_write_contains_dxg() { + // /dev/dxg must be present so WSL2 sandboxes get the Landlock + // read-write rule for the CDI-injected DXG device. The existence + // check in enrich_proto_baseline_paths() skips it on native Linux. + assert!( + GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), + "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" + ); + } + + #[test] + fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![std::path::PathBuf::from("/tmp")], + read_write: vec![], + include_workdir: false, + }, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }; + + enrich_sandbox_baseline_paths(&mut policy); + + assert!( + policy + .filesystem + .read_only + .contains(&std::path::PathBuf::from("/tmp")), + "explicit read_only baseline path should be preserved" + ); + assert!( + !policy + .filesystem + .read_write + .contains(&std::path::PathBuf::from("/tmp")), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn gpu_baseline_read_only_contains_usr_lib_wsl() { + // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library + // bind-mounts are accessible under Landlock. Skipped on native Linux. + assert!( + GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), + "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" + ); + } + + #[test] + fn has_gpu_devices_reflects_dxg_or_nvidiactl() { + // Verify the OR logic: result must match the manual disjunction of + // the two path checks. Passes in all environments. + let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); + let dxg = std::path::Path::new("/dev/dxg").exists(); + assert_eq!( + has_gpu_devices(), + nvidiactl || dxg, + "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" + ); + } +} + +/// Returns `true` if the error is transient and worth retrying. +/// +/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If +/// found, only the gRPC codes that represent transient failures are retryable. +/// If no `tonic::Status` is present (e.g. a raw connection error), assume the +/// failure is transient. +fn is_retryable_error(err: &miette::Report) -> bool { + let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); + while let Some(e) = source { + if let Some(status) = e.downcast_ref::() { + return matches!( + status.code(), + tonic::Code::Unavailable + | tonic::Code::DeadlineExceeded + | tonic::Code::ResourceExhausted + | tonic::Code::Aborted + | tonic::Code::Internal + | tonic::Code::Unknown + ); + } + source = e.source(); + } + true +} + +/// Retry a gRPC operation with exponential backoff (capped at 4 s). +/// +/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, +/// `PERMISSION_DENIED`) are returned immediately without retrying. +async fn grpc_retry(op_name: &str, f: F) -> Result +where + F: Fn() -> Fut, + Fut: Future>, +{ + let mut last_err = None; + for attempt in 1..=5u32 { + match f().await { + Ok(val) => return Ok(val), + Err(e) => { + if !is_retryable_error(&e) { + return Err(e); + } + if attempt < 5 { + warn!( + attempt, + max_attempts = 5, + error = %e, + "{op_name} failed, retrying" + ); + let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); + tokio::time::sleep(backoff).await; + } + last_err = Some(e); + } + } + } + Err(miette::miette!( + "{op_name} failed after 5 attempts: {}", + last_err.expect("loop executed at least once") + )) +} + +/// Load sandbox policy from local files or gRPC. +/// +/// Priority: +/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files +/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC +/// 3. If the server returns no policy, discover from disk or use restrictive default +/// 4. Otherwise, return an error +/// +/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto +/// policy. The proto is retained so the OPA engine can be rebuilt with symlink +/// resolution after the container entrypoint starts. +async fn load_policy( + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, +) -> Result<( + SandboxPolicy, + Option>, + Option, + MiddlewareRegistryStatus, + LoadedPolicyOrigin, + bool, +)> { + // File mode: load OPA engine from rego rules + YAML data (dev override) + if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "loading") + .unmapped("policy_rules", serde_json::json!(policy_file)) + .unmapped("policy_data", serde_json::json!(data_file)) + .message(format!( + "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" + )) + .build()); + let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { + openshell_supervisor_middleware_builtins::validate_config(implementation, config) + .map_err(|error| error.to_string()) + }; + let engine = OpaEngine::from_files_with_middleware_config( + std::path::Path::new(policy_file), + std::path::Path::new(data_file), + Some(&validate_middleware_config), + )?; + let middleware_registry = + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + engine.replace_middleware_registry(middleware_registry)?; + let config = engine.query_sandbox_config()?; + let mut policy = SandboxPolicy { + version: 1, + filesystem: config.filesystem, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: config.landlock, + process: config.process, + }; + enrich_sandbox_baseline_paths(&mut policy); + // File mode has no operator-registered middleware to connect. + return Ok(( + policy, + Some(Arc::new(engine)), + None, + MiddlewareRegistryStatus::Synchronized, + LoadedPolicyOrigin::LocalOverride, + false, + )); + } + + // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data + if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + info!( + sandbox_id = %id, + endpoint = %endpoint, + "Fetching sandbox policy via gRPC" + ); + let mut snapshot = grpc_retry("Policy fetch", || { + openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) + }) + .await?; + + let mut proto_policy = if let Some(p) = snapshot.policy.clone() { + p + } else { + // No policy configured on the server. Discover from disk or + // fall back to the restrictive default, then sync to the + // gateway so it becomes the authoritative baseline. + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "discovery") + .message("Server returned no policy; attempting local discovery") + .build() + ); + let mut discovered = discover_policy_from_disk_or_default(); + // Enrich before syncing so the gateway baseline includes + // baseline paths from the start. + enrich_proto_baseline_paths(&mut discovered); + strip_proto_provider_policy_entries(&mut discovered); + let sandbox = sandbox.as_deref().ok_or_else(|| { + miette::miette!( + "Cannot sync discovered policy: sandbox not available.\n\ + Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." + ) + })?; + + // Sync and re-fetch over a single connection to avoid extra + // TLS handshakes. + let ws = snapshot.workspace.clone(); + snapshot = grpc_retry("Policy discovery sync", || { + openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox, + &discovered, + &ws, + ) + }) + .await?; + snapshot.policy.clone().ok_or_else(|| { + miette::miette!("Server still returned no policy after sync — this is a bug") + })? + }; + + // True only while `snapshot` describes the exact policy that will be + // constructed below. If enrichment cannot be synced and re-fetched, + // the policy remains enforceable but cannot be acknowledged by + // inferred structural equality. + let mut policy_bound_to_snapshot = true; + + // Ensure baseline filesystem paths are present for proxy-mode + // sandboxes. If the policy was enriched, sync the updated version + // back to the gateway so users can see the effective policy. + let enriched = enrich_proto_baseline_paths(&mut proto_policy); + let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); + if let Some(sync_policy) = sync_policy { + if let Some(sandbox_name) = sandbox.as_deref() { + match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox_name, + &sync_policy, + &snapshot.workspace, + ) + .await + { + Ok(canonical) => { + if let Some(policy) = canonical.policy.clone() { + proto_policy = policy; + snapshot = canonical; + } else { + policy_bound_to_snapshot = false; + warn!( + "Gateway returned no policy after enrichment sync; initial revision will be reconciled" + ); + } + } + Err(e) => { + policy_bound_to_snapshot = false; + warn!( + error = %e, + "Failed to sync enriched policy back to gateway; initial revision will be reconciled" + ); + } + } + } else { + policy_bound_to_snapshot = false; + } + } + + let loaded_policy_revision = + policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); + + // Build OPA engine from baked-in rules + typed proto data. + // In cluster mode, proxy networking is always enabled so OPA is + // always required for allow/deny decisions. + // The initial load uses pid=0 (no symlink resolution) because the + // container hasn't started yet. After the entrypoint spawns, the + // engine is rebuilt with the real PID for symlink resolution. + info!("Creating OPA engine from proto policy data"); + let engine = match OpaEngine::from_proto(&proto_policy) { + Ok(engine) => engine, + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + return Err(e); + } + }; + + // Install the in-process catalog before any external connection can + // fail. A newly started sandbox must always be able to resolve built-in + // bindings, even while operator-run services are unavailable. + install_builtin_middleware_registry(&engine).await?; + + // Connect operator-registered middleware services. A connect/describe + // failure keeps the built-in registry active so each request's + // `on_error` policy governs matched traffic. The policy poll loop + // retries the install without waiting for a config change. + let middleware_services = snapshot.supervisor_middleware_services.clone(); + let middleware_registry_status = if middleware_services.is_empty() { + MiddlewareRegistryStatus::Synchronized + } else if let Err(error) = grpc_retry("Middleware connect", || { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + middleware_services.clone(), + ) + }) + .await + .and_then(|registry| engine.replace_middleware_registry(registry)) + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(middleware_services.len()) + ) + .message(format!( + "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" + )) + .build() + ); + MiddlewareRegistryStatus::NeedsReconciliation + } else { + MiddlewareRegistryStatus::Synchronized + }; + let opa_engine = Some(Arc::new(engine)); + + let policy = match SandboxPolicy::try_from(proto_policy.clone()) { + Ok(policy) => policy, + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + return Err(e); + } + }; + return Ok(( + policy, + opa_engine, + Some(proto_policy), + middleware_registry_status, + LoadedPolicyOrigin::Gateway { + revision: loaded_policy_revision, + }, + agent_proposals_enabled_from_settings(&snapshot.settings), + )); + } + + // No policy source available + Err(miette::miette!( + "Sandbox policy required. Provide one of:\n\ + - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ + - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" + )) +} + +/// Try to discover a sandbox policy from the well-known disk path, falling +/// back to the legacy path, then to the hardcoded restrictive default. +fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { + let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); + if primary.exists() { + return discover_policy_from_path(primary); + } + let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); + if legacy.exists() { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "legacy_path", + serde_json::json!(legacy.display().to_string()) + ) + .unmapped("new_path", serde_json::json!(primary.display().to_string())) + .message(format!( + "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", + legacy.display(), + primary.display() + )) + .build() + ); + return discover_policy_from_path(legacy); + } + discover_policy_from_path(primary) +} + +/// Try to read a sandbox policy YAML from `path`, falling back to the +/// hardcoded restrictive default if the file is missing or invalid. +fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { + use openshell_policy::{ + parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, + }; + + let Ok(yaml) = std::fs::read_to_string(path) else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "default") + .message(format!( + "No policy file on disk, using restrictive default [path:{}]", + path.display() + )) + .build() + ); + return restrictive_default_policy(); + }; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Loaded sandbox policy from container disk [path:{}]", + path.display() + )) + .build() + ); + match parse_sandbox_policy(&yaml) { + Ok(policy) => { + // Validate the disk-loaded policy for safety. + if let Err(violations) = validate_sandbox_policy(&policy) { + let messages: Vec = violations.iter().map(ToString::to_string).collect(); + ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Medium) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .finding_info( + FindingInfo::new( + "unsafe-disk-policy", + "Unsafe Disk Policy Content", + ) + .with_desc(&format!( + "Disk policy at {} contains unsafe content: {}", + path.display(), + messages.join("; "), + )), + ) + .message(format!( + "Disk policy contains unsafe content, using restrictive default [path:{}]", + path.display() + )) + .build()); + return restrictive_default_policy(); + } + policy + } + Err(e) => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "fallback") + .message(format!( + "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", + path.display() + )) + .build()); + restrictive_default_policy() + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MiddlewareRegistryStatus { + Synchronized, + NeedsReconciliation, +} + +/// True when the installed middleware registry no longer matches the desired +/// service set and must be rebuilt (reconnecting every delivered service). +/// +/// A policy-only change never requires a rebuild: middleware configs were +/// validated at gateway admission and the installed registry's manifests +/// already cover the unchanged service set, so requiring the services to be +/// reachable would only let a middleware outage block the policy update. +fn middleware_registry_needs_rebuild( + registry_status: MiddlewareRegistryStatus, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> bool { + registry_status == MiddlewareRegistryStatus::NeedsReconciliation + || current_services != desired_services +} + +fn gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy: bool, + current_policy_hash: &str, + desired_policy_hash: &str, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + registry_status: MiddlewareRegistryStatus, +) -> bool { + reloads_gateway_policy + && (current_policy_hash != desired_policy_hash + || middleware_registry_needs_rebuild( + registry_status, + current_services, + desired_services, + )) +} + +/// Identity returned with the exact policy snapshot used to construct OPA. +#[derive(Clone, Debug, PartialEq, Eq)] +struct LoadedPolicyRevision { + version: u32, + policy_hash: String, + config_revision: u64, + policy_source: openshell_core::proto::PolicySource, +} + +/// Identifies where the policy currently loaded into OPA came from. +/// +/// A missing gateway revision means the policy was loaded from the gateway but +/// could not be bound to an authoritative snapshot (for example, enrichment +/// sync failed). That state must reconcile on the first successful poll. A +/// local-file override is different: gateway policy revisions are observed for +/// settings/provider refreshes but must never replace the explicit local OPA +/// policy. +#[derive(Clone, Debug, PartialEq, Eq)] +enum LoadedPolicyOrigin { + LocalOverride, + Gateway { + revision: Option, + }, +} + +impl LoadedPolicyOrigin { + fn allows_gateway_policy_reload(&self) -> bool { + matches!(self, Self::Gateway { .. }) + } +} + +impl LoadedPolicyRevision { + fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { + Self { + version: snapshot.version, + policy_hash: snapshot.policy_hash.clone(), + config_revision: snapshot.config_revision, + policy_source: snapshot.policy_source, + } + } +} + +/// A sandbox-scoped policy revision that was constructed successfully at +/// startup and must be acknowledged to the gateway exactly once. +#[derive(Clone, Debug, PartialEq, Eq)] +struct InitialPolicyAck { + version: u32, + policy_hash: String, + config_revision: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PolicyStatusUpdate { + version: u32, + loaded: bool, + error: String, + initial_policy_hash: Option, +} + +impl PolicyStatusUpdate { + fn initial_loaded(ack: &InitialPolicyAck) -> Self { + Self { + version: ack.version, + loaded: true, + error: String::new(), + initial_policy_hash: Some(ack.policy_hash.clone()), + } + } + + fn loaded(version: u32) -> Self { + Self { + version, + loaded: true, + error: String::new(), + initial_policy_hash: None, + } + } + + fn failed(version: u32, error: String) -> Self { + Self { + version, + loaded: false, + error, + initial_policy_hash: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum InitialPollDisposition { + Acknowledge(InitialPolicyAck), + Reconcile, + TrackOnly, +} + +/// Determine whether the initially loaded policy corresponds to an +/// authoritative sandbox-scoped revision that must be acknowledged. +/// +/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose +/// captured gateway identity matches the current version and hash. Global +/// policies, local-file development policies, version zero, and changed +/// identities yield `None`, so those paths never emit a sandbox-revision +/// acknowledgement. +fn initial_policy_ack_candidate( + loaded: Option<&LoadedPolicyRevision>, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> Option { + let loaded = loaded?; + if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox + || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox + { + return None; + } + if loaded.version == 0 || canonical.version == 0 { + return None; + } + if loaded.version != canonical.version + || loaded.policy_hash != canonical.policy_hash + || canonical.config_revision < loaded.config_revision + { + return None; + } + Some(InitialPolicyAck { + version: loaded.version, + policy_hash: loaded.policy_hash.clone(), + config_revision: canonical.config_revision, + }) +} + +fn initial_poll_disposition( + origin: &LoadedPolicyOrigin, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> InitialPollDisposition { + match origin { + LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, + LoadedPolicyOrigin::Gateway { revision } => { + initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( + InitialPollDisposition::Reconcile, + InitialPollDisposition::Acknowledge, + ) + } + } +} + +/// Deliver policy status updates independently from policy reconciliation. +/// +/// The channel is FIFO, so a delayed older status can never arrive after a +/// newer status and move the gateway's active version backward. Delivery uses +/// the existing bounded retry, but failures never delay policy enforcement. +async fn run_policy_status_reporter( + client: openshell_core::grpc_client::CachedOpenShellClient, + sandbox_id: String, + mut updates: tokio::sync::mpsc::UnboundedReceiver, +) { + 'updates: while let Some(update) = updates.recv().await { + let operation = if update.initial_policy_hash.is_some() { + "Initial policy acknowledgement" + } else { + "Policy status report" + }; + let mut attempt = 1_u32; + loop { + let sandbox_id = sandbox_id.clone(); + let error = update.error.clone(); + let client = client.clone(); + match client + .report_policy_status(&sandbox_id, update.version, update.loaded, &error) + .await + { + Ok(()) => break, + Err(error) if is_retryable_error(&error) => { + let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); + warn!( + %error, + attempt, + version = update.version, + loaded = update.loaded, + retry_in_secs = backoff.as_secs(), + "{operation} failed transiently; retaining ordered update" + ); + tokio::time::sleep(backoff).await; + attempt = attempt.saturating_add(1); + } + Err(error) => { + warn!( + %error, + version = update.version, + loaded = update.loaded, + "Discarding terminal policy status update" + ); + continue 'updates; + } + } + } + + if let Some(policy_hash) = update.initial_policy_hash { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("version", serde_json::json!(update.version)) + .unmapped("policy_hash", serde_json::json!(policy_hash)) + .message(format!( + "Acknowledged initial policy revision as loaded [version:{}]", + update.version + )) + .build() + ); + } + } +} + +fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { + let version = update.version; + if let Err(error) = sender.send(update) { + warn!( + %error, + version, + "Policy status reporter unavailable during shutdown" + ); + } +} + +/// Best-effort `FAILED` acknowledgement when initial policy construction or +/// conversion fails. +/// +/// Uses the revision identity captured with the policy that failed to build, +/// and preserves the original construction error as the reported message. A +/// delivery failure here is swallowed so it can never mask that error. +async fn report_initial_policy_failure( + endpoint: &str, + sandbox_id: &str, + revision: Option<&LoadedPolicyRevision>, + error: &miette::Report, +) { + let Some(revision) = revision.filter(|revision| { + revision.version > 0 + && revision.policy_source == openshell_core::proto::PolicySource::Sandbox + }) else { + return; + }; + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + Ok(client) => client, + Err(e) => { + warn!(error = %e, "Failed to connect to report initial policy failure"); + return; + } + }; + let message = error.to_string(); + if let Err(e) = grpc_retry("Initial policy failure report", || { + let client = client.clone(); + let message = message.clone(); + async move { + client + .report_policy_status(sandbox_id, revision.version, false, &message) + .await + } + }) + .await + { + warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); + } +} + +/// Background loop that polls the server for policy updates. +/// +/// When a new version is detected, attempts to reload the OPA engine via +/// `reload_from_proto_with_pid()`. Reports load success/failure back to the +/// server. On failure, the previous engine is untouched (LKG behavior). +/// +/// When the entrypoint PID is available, policy reloads include symlink +/// resolution for binary paths via the container filesystem. +struct PolicyPollLoopContext { + endpoint: String, + sandbox_id: String, + opa_engine: Arc, + /// Source of the policy currently loaded into OPA. This distinguishes an + /// explicit local-file override from an unbound gateway revision so the + /// former is never replaced by policy polling. + loaded_policy_origin: LoadedPolicyOrigin, + entrypoint_pid: Arc, + interval_secs: u64, + ocsf_enabled: Arc, + provider_credentials: ProviderCredentialState, + policy_local_ctx: Option>, + agent_proposals: AgentProposals, + middleware_registry_status: MiddlewareRegistryStatus, + sidecar_control_publisher: Option, + workspace_tx: tokio::sync::watch::Sender, +} + +async fn connect_middleware_registry( + services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> Result { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + ) + .await +} + +async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + opa_engine.replace_middleware_registry(registry) +} + +async fn reconcile_middleware_registry( + opa_engine: &OpaEngine, + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + current_services: &mut Vec, + status: &mut MiddlewareRegistryStatus, +) { + if *status == MiddlewareRegistryStatus::Synchronized + && desired_services == current_services.as_slice() + { + return; + } + + match connect_middleware_registry(desired_services) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + current_services.clear(); + current_services.extend_from_slice(desired_services); + *status = MiddlewareRegistryStatus::Synchronized; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(current_services.len()) + ) + .message(format!( + "Supervisor middleware registry reloaded [service_count:{}]", + current_services.len() + )) + .build() + ); + } + Err(error) => { + // Emit only on the transition into the failed state to avoid + // repeating the same finding on every poll during an outage. + if *status == MiddlewareRegistryStatus::Synchronized { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + } + *status = MiddlewareRegistryStatus::NeedsReconciliation; + } + } +} + +async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::PolicySource; + use std::sync::atomic::Ordering; + + let client = CachedOpenShellClient::connect(&ctx.endpoint).await?; + let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(run_policy_status_reporter( + client.clone(), + ctx.sandbox_id.clone(), + status_receiver, + )); + + let mut current_config_revision: u64 = 0; + let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; + let mut current_policy_hash = String::new(); + let mut current_middleware_services = Vec::new(); + let mut middleware_registry_status = ctx.middleware_registry_status; + let mut current_settings: std::collections::HashMap< + String, + openshell_core::proto::EffectiveSetting, + > = std::collections::HashMap::new(); + let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); + let mut last_failed_runtime_revision: Option<(u64, String)> = None; + + // A first poll that does not match the policy already loaded into OPA must + // pass through the normal reconciliation path immediately. It must never + // seed the applied-state trackers before OPA actually loads it. + let mut pending_result = None; + + // Initialize revision from the first poll and acknowledge the initial + // policy revision the supervisor actually loaded. A mismatched result is + // reconciled below instead of being recorded as already applied. + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { + InitialPollDisposition::Acknowledge(candidate) => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(candidate.config_revision), + ctx.sidecar_control_publisher.as_ref(), + skills::install_static_skills, + ); + current_config_revision = candidate.config_revision; + current_policy_hash.clone_from(&candidate.policy_hash); + current_middleware_services = result.supervisor_middleware_services; + current_settings = result.settings; + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::initial_loaded(&candidate), + ); + debug!( + config_revision = current_config_revision, + "Settings poll: initial policy matches loaded revision" + ); + } + InitialPollDisposition::Reconcile => pending_result = Some(result), + InitialPollDisposition::TrackOnly => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(result.config_revision), + ctx.sidecar_control_publisher.as_ref(), + skills::install_static_skills, + ); + current_config_revision = result.config_revision; + current_policy_hash = result.policy_hash.clone(); + current_middleware_services = result.supervisor_middleware_services; + current_settings = result.settings; + debug!( + config_revision = current_config_revision, + "Settings poll: tracking gateway config while preserving local policy override" + ); + } + } + } + Err(e) => { + warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); + } + } + + let interval = Duration::from_secs(ctx.interval_secs); + loop { + let result = if let Some(result) = pending_result.take() { + result + } else { + tokio::time::sleep(interval).await; + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + result + } + Err(e) => { + debug!(error = %e, "Settings poll: server unreachable, will retry"); + continue; + } + } + }; + + let config_changed = result.config_revision != current_config_revision; + let provider_env_changed = result.provider_env_revision != current_provider_env_revision; + let policy_changed = result.policy_hash != current_policy_hash; + let middleware_registry_changed = middleware_registry_needs_rebuild( + middleware_registry_status, + ¤t_middleware_services, + &result.supervisor_middleware_services, + ); + let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); + + // A local policy override is not coupled to the gateway policy + // snapshot, so its service registry can still be reconciled alone. + // Gateway policy snapshots, however, must install policy and registry + // as one generation below. + if !reloads_gateway_policy { + reconcile_middleware_registry( + &ctx.opa_engine, + &result.supervisor_middleware_services, + &mut current_middleware_services, + &mut middleware_registry_status, + ) + .await; + } + + if !config_changed && !provider_env_changed && !policy_runtime_changed { + continue; + } + + if config_changed || provider_env_changed { + // Log which settings changed. + log_setting_changes(¤t_settings, &result.settings); + + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "detected") + .unmapped("old_config_revision", serde_json::json!(current_config_revision)) + .unmapped("new_config_revision", serde_json::json!(result.config_revision)) + .unmapped("policy_changed", serde_json::json!(policy_changed)) + .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) + .message(format!( + "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", + result.config_revision + )) + .build()); + } + + if provider_env_changed { + match openshell_core::grpc_client::fetch_provider_environment( + &ctx.endpoint, + &ctx.sandbox_id, + ) + .await + { + Ok(env_result) => { + ctx.provider_credentials.install_environment( + env_result.provider_env_revision, + env_result.environment, + env_result.credential_expires_at_ms, + env_result.dynamic_credentials, + ); + let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); + let env_count = child_env.len(); + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_provider_env( + env_result.provider_env_revision, + child_env.clone(), + ); + } + current_provider_env_revision = env_result.provider_env_revision; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "provider_env_revision", + serde_json::json!(env_result.provider_env_revision) + ) + .message(format!( + "Provider environment refreshed [revision:{} env_count:{env_count}]", + env_result.provider_env_revision + )) + .build() + ); + } + Err(e) => { + warn!( + error = %e, + provider_env_revision = result.provider_env_revision, + "Settings poll: failed to refresh provider environment" + ); + } + } + } + + if policy_runtime_changed { + let pid = ctx.entrypoint_pid.load(Ordering::Acquire); + let runtime_result = match result.policy.as_ref() { + Some(policy) if middleware_registry_changed => { + match connect_middleware_registry(&result.supervisor_middleware_services).await + { + Ok(registry) => ctx + .opa_engine + .reload_policy_and_middleware_from_proto_with_pid( + policy, pid, registry, + ), + Err(error) => Err(error), + } + } + // Policy-only change: the installed registry already matches + // the delivered service set, so swap the engine alone. This + // must not require middleware reachability. + Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), + None => Err(miette::miette!( + "runtime reload requires a policy payload but none was returned" + )), + }; + + match runtime_result { + Ok(()) => { + let policy = result + .policy + .as_ref() + .expect("successful runtime reload requires a policy payload"); + if policy_changed { + if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { + policy_local_ctx.set_current_policy(policy.clone()).await; + } + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_policy( + policy.clone(), + result.policy_hash.clone(), + result.config_revision, + ); + } + if result.global_policy_version > 0 { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .unmapped("global_version", serde_json::json!(result.global_policy_version)) + .message(format!( + "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", + result.policy_hash, + result.global_policy_version + )) + .build()); + } else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + } + if result.version > 0 && result.policy_source == PolicySource::Sandbox { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + } + } + + if middleware_registry_changed { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(result.supervisor_middleware_services.len()) + ) + .message(format!( + "Supervisor policy runtime reloaded atomically [service_count:{}]", + result.supervisor_middleware_services.len() + )) + .build()); + } + + current_policy_hash.clone_from(&result.policy_hash); + current_middleware_services.clone_from(&result.supervisor_middleware_services); + middleware_registry_status = MiddlewareRegistryStatus::Synchronized; + last_failed_runtime_revision = None; + } + Err(e) => { + let failed_revision = (result.config_revision, result.policy_hash.clone()); + if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .unmapped("version", serde_json::json!(result.version)) + .unmapped("error", serde_json::json!(e.to_string())) + .message(format!( + "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", + result.version + )) + .build()); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, e.to_string()), + ); + } + } + last_failed_runtime_revision = Some(failed_revision); + // Nothing was installed, so the registry status still + // describes the live registry. The retry is driven by the + // persisting hash/service-set mismatch (or an existing + // NeedsReconciliation), not by degrading the status here. + } + } + } + + // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + + // Apply the agent-proposals feature toggle. On a false→true transition + // we lazily install the skill so a sandbox that started with the flag + // off picks up the surface without a recreate. We never uninstall on + // a true→false transition: stale skill content on disk is harmless + // because route_request and agent_next_steps both gate on the live + // shared flag, so the agent that reads the skill will see 404s and an + // empty `next_steps` array regardless. + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "settings poll", + Some(result.config_revision), + ctx.sidecar_control_publisher.as_ref(), + skills::install_static_skills, + ); + + current_config_revision = result.config_revision; + if !reloads_gateway_policy { + current_policy_hash = result.policy_hash; + } + current_settings = result.settings; + } +} + +fn apply_ocsf_json_setting( + enabled: &AtomicBool, + settings: &std::collections::HashMap, +) { + use std::sync::atomic::Ordering; + + let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); + let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); + if new_ocsf != prev_ocsf { + info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); + } +} + +/// Extract a bool value from an effective setting, if present. +fn extract_bool_setting( + settings: &std::collections::HashMap, + key: &str, +) -> Option { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|v| match v { + setting_value::Value::BoolValue(b) => Some(*b), + _ => None, + }) +} + +fn agent_proposals_enabled_from_settings( + settings: &std::collections::HashMap, +) -> bool { + extract_bool_setting( + settings, + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, + ) + .unwrap_or(false) +} + +fn apply_agent_proposals_enabled( + agent_proposals: &AgentProposals, + enabled: bool, + source: &'static str, + config_revision: Option, + sidecar_control_publisher: Option<&sidecar_control::Publisher>, + install_static_skills: impl FnOnce() -> Result, +) { + let previously_enabled = agent_proposals.swap_enabled(enabled); + if enabled == previously_enabled { + return; + } + + info!( + agent_policy_proposals_enabled = enabled, + source, config_revision, "agent-driven policy proposals toggled" + ); + + if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { + publisher.publish_agent_proposals(enabled, config_revision); + } + + if enabled && !previously_enabled { + match install_static_skills() { + Ok(installed) => info!( + path = %installed.policy_advisor.display(), + "Installed sandbox agent skill on toggle-on" + ), + Err(error) => warn!( + error = %error, + "Failed to install sandbox agent skill on toggle-on" + ), + } + } +} + +/// Log individual setting changes between two snapshots. +fn log_setting_changes( + old: &std::collections::HashMap, + new: &std::collections::HashMap, +) { + for (key, new_es) in new { + let new_val = format_setting_value(new_es); + match old.get(key) { + Some(old_es) => { + let old_val = format_setting_value(old_es); + if old_val != new_val { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "updated") + .unmapped("key", serde_json::json!(key)) + .unmapped("old", serde_json::json!(old_val.clone())) + .unmapped("new", serde_json::json!(new_val.clone())) + .message(format!( + "Setting changed [key:{key} old:{old_val} new:{new_val}]" + )) + .build() + ); + } + } + None => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enabled") + .unmapped("key", serde_json::json!(key)) + .unmapped("value", serde_json::json!(new_val.clone())) + .message(format!("Setting added [key:{key} value:{new_val}]")) + .build() + ); + } + } + } + for key in old.keys() { + if !new.contains_key(key) { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Disabled, "disabled") + .unmapped("key", serde_json::json!(key)) + .message(format!("Setting removed [key:{key}]")) + .build() + ); + } + } +} + +/// Format an `EffectiveSetting` value for log display. +fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { + use openshell_core::proto::setting_value; + match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { + None => "".to_string(), + Some(setting_value::Value::StringValue(v)) => v.clone(), + Some(setting_value::Value::BoolValue(v)) => v.to_string(), + Some(setting_value::Value::IntValue(v)) => v.to_string(), + Some(setting_value::Value::BytesValue(_)) => "".to_string(), + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, + }; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + fn proxy_policy(http_addr: Option) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { + openshell_core::proto::EffectiveSetting { + value: Some(openshell_core::proto::SettingValue { + value: Some(openshell_core::proto::setting_value::Value::BoolValue( + value, + )), + }), + scope: openshell_core::proto::SettingScope::Global.into(), + } + } + + #[test] + fn sidecar_process_policy_sets_loopback_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, true).unwrap(); + + let http_addr = process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .expect("sidecar process policy should set proxy address"); + assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); + assert!( + policy + .network + .proxy + .as_ref() + .expect("original policy should keep proxy config") + .http_addr + .is_none(), + "process policy normalization must not mutate the network policy" + ); + } + + #[test] + fn non_sidecar_process_policy_preserves_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, false).unwrap(); + + assert!( + process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .is_none() + ); + } + + #[tokio::test] + async fn sidecar_control_provider_env_update_installs_newer_revision() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + 1, + std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), + ); + let handle = spawn_sidecar_control_update_watcher( + rx, + provider_credentials.clone(), + AgentProposals::default(), + ); + + tx.send(sidecar_control::ControlUpdate::ProviderEnv { + revision: 2, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "new".to_string(), + )]), + }) + .unwrap(); + + timeout(Duration::from_secs(1), async { + loop { + if provider_credentials.snapshot().revision == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let snapshot = provider_credentials.snapshot(); + assert_eq!(snapshot.revision, 2); + assert_eq!( + snapshot.child_env.get("TOKEN").map(String::as_str), + Some("new") + ); + + tx.send(sidecar_control::ControlUpdate::ProviderEnv { + revision: 1, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "stale".to_string(), + )]), + }) + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + provider_credentials + .snapshot() + .child_env + .get("TOKEN") + .map(String::as_str), + Some("new") + ); + handle.abort(); + } + + #[tokio::test] + async fn sidecar_control_agent_proposals_update_flips_shared_state() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let provider_credentials = + ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); + let agent_proposals = AgentProposals::new(true); + let handle = + spawn_sidecar_control_update_watcher(rx, provider_credentials, agent_proposals.clone()); + + tx.send(sidecar_control::ControlUpdate::AgentProposals { + enabled: false, + config_revision: 5, + }) + .unwrap(); + + timeout(Duration::from_secs(1), async { + loop { + if !agent_proposals.enabled() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + handle.abort(); + } + + #[test] + fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { + let agent_proposals = AgentProposals::default(); + let installs = AtomicUsize::new(0); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(!agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + } + + #[test] + fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { + let enabled = AtomicBool::new(false); + let mut settings = std::collections::HashMap::new(); + settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(enabled.load(Ordering::Relaxed)); + } + + #[test] + fn apply_ocsf_json_setting_disables_when_setting_is_unset() { + let enabled = AtomicBool::new(true); + let settings = std::collections::HashMap::new(); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(!enabled.load(Ordering::Relaxed)); + } + + #[test] + fn agent_proposals_setting_enables_from_initial_settings_snapshot() { + let mut settings = std::collections::HashMap::new(); + settings.insert( + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), + effective_bool(true), + ); + + assert!(agent_proposals_enabled_from_settings(&settings)); + } + + #[test] + fn agent_proposals_setting_defaults_false_when_unset() { + let settings = std::collections::HashMap::new(); + + assert!(!agent_proposals_enabled_from_settings(&settings)); + } + + // ---- Policy disk discovery tests ---- + + #[test] + fn discover_policy_from_nonexistent_path_returns_restrictive_default() { + let path = std::path::Path::new("/nonexistent/policy.yaml"); + let policy = discover_policy_from_path(path); + // Restrictive default has no network policies. + assert!(policy.network_policies.is_empty()); + // But does have filesystem and process policies. + assert!(policy.filesystem.is_some()); + assert!(policy.process.is_some()); + } + + #[test] + fn discover_policy_from_valid_yaml_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +filesystem_policy: + include_workdir: false + read_only: + - /usr + read_write: + - /tmp +network_policies: + test: + name: test + endpoints: + - { host: example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + assert_eq!(policy.network_policies.len(), 1); + assert!(policy.network_policies.contains_key("test")); + let fs = policy.filesystem.unwrap(); + assert!(!fs.include_workdir); + } + + #[test] + fn discover_policy_from_invalid_yaml_returns_restrictive_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default. + assert!(policy.network_policies.is_empty()); + assert!(policy.filesystem.is_some()); + } + + #[test] + fn discover_policy_from_unsafe_yaml_falls_back_to_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +process: + run_as_user: root + run_as_group: root +filesystem_policy: + include_workdir: true + read_only: + - /usr + read_write: + - /tmp +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default because of root user. + let proc = policy.process.unwrap(); + assert_eq!(proc.run_as_user, "sandbox"); + assert_eq!(proc.run_as_group, "sandbox"); + } + + #[test] + fn discover_policy_restrictive_default_blocks_network() { + // In cluster mode we keep proxy mode enabled so `inference.local` + // can always be routed through proxy/OPA controls. + let proto = openshell_policy::restrictive_default_policy(); + let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); + assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); + } + + // ---- Initial policy acknowledgement tests ---- + + fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::restrictive_default_policy() + } + + fn settings_poll_result( + policy: Option, + version: u32, + source: openshell_core::proto::PolicySource, + ) -> openshell_core::grpc_client::SettingsPollResult { + openshell_core::grpc_client::SettingsPollResult { + policy, + version, + policy_hash: format!("hash-v{version}"), + config_revision: u64::from(version) * 100, + policy_source: source, + settings: std::collections::HashMap::new(), + global_policy_version: 0, + provider_env_revision: 0, + supervisor_middleware_services: Vec::new(), + workspace: String::new(), + } + } + + #[tokio::test] + async fn failed_external_startup_registry_build_preserves_installed_builtins() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let builtins_generation = engine.current_generation(); + assert_eq!(builtins_generation, 1); + + let invalid_external = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_body_bytes: 1024, + ..Default::default() + }; + connect_middleware_registry(&[invalid_external]) + .await + .expect_err("unavailable external service must not replace built-ins"); + + assert_eq!(engine.current_generation(), builtins_generation); + } + + #[test] + fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { + let services = Vec::new(); + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + } + + #[test] + fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &no_services, + &no_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &no_services, + &desired_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + false, + "local-policy", + "hash-v2", + &no_services, + &desired_services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + } + + #[test] + fn policy_only_change_does_not_rebuild_middleware_registry() { + let services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + // The runtime must reconcile, but the registry (and therefore + // middleware reachability) is not part of that reconciliation. + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &services, + &services, + )); + } + + #[test] + fn registry_rebuild_requires_service_set_change_or_degraded_registry() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &no_services, + &desired_services, + )); + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::NeedsReconciliation, + &desired_services, + &desired_services, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &desired_services, + &desired_services, + )); + } + + #[test] + fn initial_ack_candidate_matches_sandbox_revision() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) + .expect("sandbox-sourced matching revision should be acknowledged"); + + assert_eq!(ack.version, 2); + assert_eq!(ack.policy_hash, "hash-v2"); + assert_eq!(ack.config_revision, 200); + } + + #[test] + fn initial_ack_candidate_ignores_global_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Global, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_version_zero() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 0, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_local_file_mode() { + // Local-file mode retains no proto policy, so there is nothing to + // acknowledge to the gateway. + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(None, &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_rejects_mismatched_identity() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let mut newer = proto_policy_fixture(); + newer.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule::default(), + ); + let canonical = + settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); + let canonical = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "hash-provider-change".to_string(), + config_revision: loaded.config_revision + 1, + ..canonical + }; + + assert_eq!( + initial_poll_disposition( + &LoadedPolicyOrigin::Gateway { + revision: Some(loaded), + }, + &canonical, + ), + InitialPollDisposition::Reconcile + ); + } + + #[test] + fn initial_poll_tracks_local_override_without_reconciliation() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert_eq!( + initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), + InitialPollDisposition::TrackOnly + ); + assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); + } + + #[test] + fn initial_poll_reconciles_unbound_gateway_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let origin = LoadedPolicyOrigin::Gateway { revision: None }; + + assert_eq!( + initial_poll_disposition(&origin, &canonical), + InitialPollDisposition::Reconcile + ); + assert!(origin.allows_gateway_policy_reload()); + } + + #[test] + fn policy_status_outbox_preserves_all_revision_order() { + let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); + for version in 1..=128 { + enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); + } + + for version in 1..=128 { + assert_eq!( + receiver.try_recv().unwrap(), + PolicyStatusUpdate::loaded(version) + ); + } + } + + #[test] + fn settings_snapshot_carries_workspace_for_policy_sync() { + let mut snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + snapshot.workspace = "beta".to_string(); + + let revision = LoadedPolicyRevision::from_snapshot(&snapshot); + assert_eq!(revision.version, 1); + assert_eq!( + snapshot.workspace, "beta", + "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" + ); + } +} diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 62ae37b5a1..948e0e7108 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -1,749 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Sandbox - process sandbox and monitor. - -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; - -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; -use tracing::{info, warn}; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::LevelFilter; -use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; - -use openshell_sandbox::run_sandbox; - -/// Subcommand name used to self-copy the supervisor binary into a shared volume. -/// -/// Init containers invoke the binary directly instead of relying on `sh`/`cp` -/// to copy the binary out. Invoking the binary itself with this argument -/// performs the copy in pure Rust. -const COPY_SELF_SUBCOMMAND: &str = "copy-self"; - -/// Subcommand for one-shot debug RPCs from inside a sandbox container. -/// -/// Reads the same token sources as the supervisor (env, file, K8s SA -/// bootstrap) and issues a single gRPC call against the gateway. Useful -/// for end-to-end verification: e.g. `docker exec` into a sandbox, then -/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` -/// to confirm the cross-sandbox IDOR guard fires. -const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; - -/// Default `--mode` value: run both supervisor leaves in a single binary. -const DEFAULT_MODE: &str = "network,process"; -const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; -const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; -#[cfg(target_os = "linux")] -const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; -#[cfg(target_os = "linux")] -const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; -#[cfg(target_os = "linux")] -const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; -#[cfg(target_os = "linux")] -const SIDECAR_TLS_DIR_MODE: u32 = 0o755; -#[cfg(target_os = "linux")] -const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; - -/// Which supervisor leaves are enabled in this process. -/// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. -#[derive(Clone, Copy, Debug)] -struct Mode { - network: bool, - process: bool, - network_init: bool, -} - -impl std::str::FromStr for Mode { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut mode = Self { - network: false, - process: false, - network_init: false, - }; - for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { - match part { - "network" => mode.network = true, - "process" => mode.process = true, - "network-init" => mode.network_init = true, - other => { - return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" - )); - } - } - } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); - } - if !mode.network && !mode.process && !mode.network_init { - return Err( - "--mode must enable at least one of: network, process, network-init".into(), - ); - } - Ok(mode) - } -} - -/// `OpenShell` Sandbox - process isolation and monitoring. -// CLI flags are naturally boolean switches; grouping them into structs would -// only obscure the clap definition. -#[allow(clippy::struct_excessive_bools)] -#[derive(Parser, Debug)] -#[command(name = "openshell-sandbox")] -#[command(version = openshell_core::VERSION)] -#[command(about = "Process sandbox and monitor", long_about = None)] -struct Args { - /// Command to execute in the sandbox. - /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. - /// Defaults to `/bin/bash` if neither is provided. - #[arg(trailing_var_arg = true)] - command: Vec, - - /// Working directory for the sandboxed process. - #[arg(long, short)] - workdir: Option, - - /// Timeout in seconds (0 = no timeout). - #[arg(long, short, default_value = "0")] - timeout: u64, - - /// Run in interactive mode (inherit process group for terminal control). - #[arg(long, short = 'i')] - interactive: bool, - - /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. - /// Requires --openshell-endpoint to be set. - #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] - sandbox_id: Option, - - /// Sandbox (used for policy sync when the sandbox discovers policy - /// from disk or falls back to the restrictive default). - #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] - sandbox: Option, - - /// `OpenShell` server gRPC endpoint for fetching policy. - /// Required when using --sandbox-id. - #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] - openshell_endpoint: Option, - - /// Path to Rego policy file for OPA-based network access control. - /// Requires --policy-data to also be set. - #[arg(long, env = "OPENSHELL_POLICY_RULES")] - policy_rules: Option, - - /// Path to YAML data file containing network policies and sandbox config. - /// Requires --policy-rules to also be set. - #[arg(long, env = "OPENSHELL_POLICY_DATA")] - policy_data: Option, - - /// Log level (trace, debug, info, warn, error). - #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] - log_level: String, - - /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning - /// with `@` selects an abstract socket in the network namespace. - /// The supervisor bridges `RelayStream` traffic from the gateway onto - /// this socket; nothing else should connect to it. - #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] - ssh_socket_path: Option, - - /// Path to YAML inference routes for standalone routing. - /// When set, inference routes are loaded from this file instead of - /// fetching a bundle from the gateway. - #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] - inference_routes: Option, - - /// Enable health check endpoint. - #[arg(long)] - health_check: bool, - - /// Port for health check endpoint. - #[arg(long, default_value = "8080")] - health_port: u16, - - /// Which supervisor components to run. Comma-separated list of - /// "network" and/or "process". Defaults to both (single-binary - /// topology). Use --mode=network for a network-only sidecar, or - /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. Use --mode=network-init only in - /// the Kubernetes init container that prepares sidecar nftables. - #[arg(long, default_value = DEFAULT_MODE)] - mode: Mode, - - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] - proxy_uid: u32, - - /// GID assigned to shared sidecar state directories. Defaults to - /// `--proxy-uid` when omitted. - #[arg(long, env = "OPENSHELL_PROXY_GID")] - proxy_gid: Option, - - /// Shared state directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] - sidecar_state_dir: String, - - /// Shared TLS work directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] - sidecar_tls_dir: String, - - // Corporate upstream proxy. Operator-owned egress boundary: accepted - // only as command-line arguments (no `env =`), because the driver - // controls the supervisor's argv while a sandbox image could bake - // matching `ENV` values. - /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. - #[arg(long)] - upstream_proxy: Option, - - /// Comma-separated `NO_PROXY` list for the corporate proxy. - #[arg(long)] - upstream_no_proxy: Option, - - /// Path to the root-only file holding corporate proxy credentials (`user:pass`). - #[arg(long)] - upstream_proxy_auth_file: Option, - - /// Acknowledge that proxy credentials travel as cleartext Basic auth over - /// the plain-TCP connection to the `http://` proxy. - #[arg(long)] - upstream_proxy_auth_allow_insecure: bool, - - /// Send the destination hostname in CONNECT instead of a validated IP - /// (for proxies whose ACLs filter on hostnames). - #[arg(long)] - upstream_proxy_connect_by_hostname: bool, -} - -/// Copy the running executable to `dest`, creating parent directories as -/// needed and ensuring the result is executable (mode `0755`). -/// -/// If `dest` already exists as a directory, the binary is placed inside it -/// using the source executable's file name. This mirrors `cp` semantics so -/// callers can pass either a full target path or a directory. -fn copy_self(dest: &str) -> Result<()> { - let exe = std::env::current_exe().into_diagnostic()?; - - let dest_path = Path::new(dest); - let final_path = if dest_path.is_dir() { - let file_name = exe - .file_name() - .ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?; - dest_path.join(file_name) - } else { - dest_path.to_path_buf() - }; - - if let Some(parent) = final_path.parent() - && !parent.as_os_str().is_empty() - { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - - std::fs::copy(&exe, &final_path).into_diagnostic()?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut perms = std::fs::metadata(&final_path) - .into_diagnostic()? - .permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&final_path, perms).into_diagnostic()?; - } - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {uid}:{gid}", - path.display() - ) - })?; - Ok(()) -} - -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - let uid = Uid::current(); - let gid = Gid::current(); - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - chown(path, Some(uid), Some(gid)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {}:{}", - path.display(), - uid.as_raw(), - gid.as_raw() - ) - })?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - Ok(()) +#[cfg(target_os = "windows")] +fn main() { + eprintln!("openshell-sandbox supervisor is not available on Windows"); + std::process::exit(1); } -#[cfg(target_os = "linux")] -fn copy_sidecar_client_tls_if_present( - source_dir: &Path, - sidecar_tls_dir: &Path, - uid: u32, - gid: u32, -) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - if !source_dir.exists() { - return Ok(()); - } - - let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); - prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; - for file_name in CLIENT_TLS_FILES { - let source = source_dir.join(file_name); - if !source.exists() { - return Err(miette::miette!( - "client TLS source file is missing: {}", - source.display() - )); - } - let dest = dest_dir.join(file_name); - if dest.exists() { - std::fs::remove_file(&dest) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to remove stale client TLS file {}", dest.display()) - })?; - } - std::fs::copy(&source, &dest) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to copy client TLS file {} to {}", - source.display(), - dest.display() - ) - })?; - let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); - perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); - std::fs::set_permissions(&dest, perms) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to chmod copied client TLS file {}", dest.display()) - })?; - chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown copied client TLS file {} to {uid}:{gid}", - dest.display() - ) - })?; - } - - prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn run_network_init( - proxy_user_id: u32, - proxy_primary_group_id: u32, - sidecar_state_dir: &str, - sidecar_tls_dir: &str, -) -> Result<()> { - validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; - - let sidecar_state_dir = Path::new(sidecar_state_dir); - let sidecar_tls_dir = Path::new(sidecar_tls_dir); - prepare_sidecar_directory( - sidecar_state_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_STATE_DIR_MODE, - )?; - // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the - // TLS work directory owned by the init user until the client cert copy is - // complete, then hand it to the long-running proxy UID. - prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; - copy_sidecar_client_tls_if_present( - Path::new(CLIENT_TLS_DIR), - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - )?; - prepare_sidecar_directory( - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_TLS_DIR_MODE, - )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) -} - -#[cfg(target_os = "linux")] -fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { - if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-uid must be 0 or at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-gid must be at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - Ok(()) -} - -#[cfg(not(target_os = "linux"))] -fn run_network_init( - _proxy_uid: u32, - _proxy_gid: u32, - _sidecar_state_dir: &str, - _sidecar_tls_dir: &str, -) -> Result<()> { - Err(miette::miette!( - "--mode=network-init is only supported on Linux" - )) -} - -fn main() -> Result<()> { - // Handle `copy-self ` before clap so it works without any of the - // sandbox flags. Kubernetes init containers invoke this path to seed an - // emptyDir volume that the agent container then executes from. - let raw_args: Vec = std::env::args().collect(); - if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { - let dest = raw_args.get(2).ok_or_else(|| { - miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") - })?; - return copy_self(dest); - } - - // Handle `debug-rpc [args]` before clap. Uses a small - // dedicated runtime so we don't pay the supervisor's full startup cost. - if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .into_diagnostic()?; - return runtime.block_on(async move { - let _ = rustls::crypto::ring::default_provider().install_default(); - let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; - std::process::exit(exit); - }); - } - - let args = Args::parse(); - - if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); - return run_network_init( - args.proxy_uid, - proxy_gid, - &args.sidecar_state_dir, - &args.sidecar_tls_dir, - ); - } - - // Try to open a rolling log file; fall back to stderr-only logging if it fails - // (e.g., /var/log is not writable in custom workload images). - // Rotates daily, keeps the 3 most recent files to bound disk usage. - let file_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - (writer, guard) - }); - - let console_filter = - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .into_diagnostic()?; - - let exit_code = runtime.block_on(async move { - // Install rustls crypto provider before any TLS connections (including log push). - let _ = rustls::crypto::ring::default_provider().install_default(); - - // Set up optional log push layer (gRPC mode only). - let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = - (&args.sandbox_id, &args.openshell_endpoint) - { - let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), - sandbox_id.clone(), - ); - let layer = - openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); - Some((layer, handle)) - } else { - None - }; - let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); - let _log_push_handle = log_push_state.map(|(_, handle)| handle); - - // Shared flag: the sandbox poll loop toggles this when the - // `ocsf_json_enabled` setting changes. The JSONL layer checks it - // on each event and short-circuits when false. - let ocsf_enabled = Arc::new(AtomicBool::new(false)); - - // Keep guards alive for the entire process. When a guard is dropped the - // non-blocking writer flushes remaining logs. - let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { - let file_filter = EnvFilter::new("info"); - - // OCSF JSONL file: rolling appender matching the main log file - // (daily rotation, 3 files max). Created eagerly but gated by the - // enabled flag — no JSONL is written until ocsf_json_enabled is set. - let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell-ocsf") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); - (layer, guard) - }); - let (jsonl_layer, jsonl_guard) = match jsonl_logging { - Some((layer, guard)) => (Some(layer), Some(guard)), - None => (None, None), - }; - - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with( - OcsfShorthandLayer::new(file_writer) - .with_non_ocsf(true) - .with_filter(file_filter), - ) - .with(jsonl_layer.with_filter(LevelFilter::INFO)) - .with(push_layer.clone()) - .init(); - (Some(file_guard), jsonl_guard) - } else { - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with(push_layer) - .init(); - // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stderr-only logging"); - (None, None) - }; - - // Get command - either from CLI args, environment variable, or default to /bin/bash - let command = if !args.command.is_empty() { - args.command - } else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) { - // Simple shell-like splitting on whitespace - c.split_whitespace().map(String::from).collect() - } else { - vec!["/bin/bash".to_string()] - }; - - info!(command = ?command, "Starting sandbox"); - // Note: "Starting sandbox" stays as plain info!() since the OCSF context - // is not yet initialized at this point (run_sandbox hasn't been called). - // The shorthand layer will render it in fallback format. - - let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { - https_proxy: args.upstream_proxy, - no_proxy: args.upstream_no_proxy, - proxy_auth_file: args.upstream_proxy_auth_file, - proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, - }; - - run_sandbox( - command, - args.workdir, - args.timeout, - args.interactive, - args.sandbox_id, - args.sandbox, - args.openshell_endpoint, - args.policy_rules, - args.policy_data, - args.ssh_socket_path, - args.health_check, - args.health_port, - args.inference_routes, - ocsf_enabled, - args.mode.network, - args.mode.process, - upstream_proxy_args, - ) - .await - })?; - - std::process::exit(exit_code); -} - -#[cfg(test)] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - /// Drives `copy_self`'s file-copy logic against an arbitrary source path - /// so tests don't depend on `current_exe()`. - fn copy_executable(src: &Path, dest: &Path) -> Result<()> { - let final_path = if dest.is_dir() { - dest.join(src.file_name().unwrap()) - } else { - dest.to_path_buf() - }; - if let Some(parent) = final_path.parent() - && !parent.as_os_str().is_empty() - { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - std::fs::copy(src, &final_path).into_diagnostic()?; - let mut perms = std::fs::metadata(&final_path) - .into_diagnostic()? - .permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&final_path, perms).into_diagnostic()?; - Ok(()) - } - - #[test] - fn copy_self_writes_executable_at_target_path() { - let tmp = tempfile::tempdir().unwrap(); - let src = tmp.path().join("source-bin"); - std::fs::write(&src, b"#!/bin/false\n").unwrap(); - - let dest = tmp.path().join("subdir/openshell-sandbox"); - copy_executable(&src, &dest).unwrap(); - - assert!(dest.exists(), "destination file should exist"); - let mode = std::fs::metadata(&dest).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o755, "destination must be 0755"); - let copied = std::fs::read(&dest).unwrap(); - assert_eq!(copied, b"#!/bin/false\n"); - } - - #[test] - fn copy_self_into_existing_directory_uses_source_filename() { - let tmp = tempfile::tempdir().unwrap(); - let src = tmp.path().join("openshell-sandbox"); - std::fs::write(&src, b"binary").unwrap(); - - let dest_dir = tmp.path().join("bin"); - std::fs::create_dir_all(&dest_dir).unwrap(); - - copy_executable(&src, &dest_dir).unwrap(); - - let final_path = dest_dir.join("openshell-sandbox"); - assert!(final_path.exists(), "binary should land inside dest dir"); - } - - #[test] - fn mode_parses_network_init_standalone() { - let mode = "network-init".parse::().unwrap(); - assert!(mode.network_init); - assert!(!mode.network); - assert!(!mode.process); - } - - #[test] - fn mode_rejects_combined_network_init() { - let err = "network-init,network".parse::().unwrap_err(); - assert!(err.contains("cannot be combined")); - } - - #[test] - fn mode_rejects_empty_value() { - let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); - } - - #[cfg(target_os = "linux")] - #[test] - fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { - assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); - assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); - assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); - assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { - validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_still_rejects_low_non_root_proxy_ids() { - let uid_err = - validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); - assert!(uid_err.to_string().contains("--proxy-uid")); - - let gid_err = validate_network_init_ids(0, 999).unwrap_err(); - assert!(gid_err.to_string().contains("--proxy-gid")); - } -} +#[cfg(not(target_os = "windows"))] +include!("main_unix.rs"); diff --git a/crates/openshell-sandbox/src/main_unix.rs b/crates/openshell-sandbox/src/main_unix.rs new file mode 100644 index 0000000000..62ae37b5a1 --- /dev/null +++ b/crates/openshell-sandbox/src/main_unix.rs @@ -0,0 +1,749 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `OpenShell` Sandbox - process sandbox and monitor. + +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; + +use openshell_sandbox::run_sandbox; + +/// Subcommand name used to self-copy the supervisor binary into a shared volume. +/// +/// Init containers invoke the binary directly instead of relying on `sh`/`cp` +/// to copy the binary out. Invoking the binary itself with this argument +/// performs the copy in pure Rust. +const COPY_SELF_SUBCOMMAND: &str = "copy-self"; + +/// Subcommand for one-shot debug RPCs from inside a sandbox container. +/// +/// Reads the same token sources as the supervisor (env, file, K8s SA +/// bootstrap) and issues a single gRPC call against the gateway. Useful +/// for end-to-end verification: e.g. `docker exec` into a sandbox, then +/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` +/// to confirm the cross-sandbox IDOR guard fires. +const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; + +/// Default `--mode` value: run both supervisor leaves in a single binary. +const DEFAULT_MODE: &str = "network,process"; +const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +#[cfg(target_os = "linux")] +const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; + +/// Which supervisor leaves are enabled in this process. +/// +/// Parsed from a comma-separated `--mode` value, e.g. `network`, +/// `process`, or `network,process`. `network-init` is a one-shot setup mode +/// used by the Kubernetes sidecar topology and cannot be combined with other +/// mode components. At least one must be set. +#[derive(Clone, Copy, Debug)] +struct Mode { + network: bool, + process: bool, + network_init: bool, +} + +impl std::str::FromStr for Mode { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut mode = Self { + network: false, + process: false, + network_init: false, + }; + for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { + match part { + "network" => mode.network = true, + "process" => mode.process = true, + "network-init" => mode.network_init = true, + other => { + return Err(format!( + "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" + )); + } + } + } + if mode.network_init && (mode.network || mode.process) { + return Err("--mode=network-init cannot be combined with other components".into()); + } + if !mode.network && !mode.process && !mode.network_init { + return Err( + "--mode must enable at least one of: network, process, network-init".into(), + ); + } + Ok(mode) + } +} + +/// `OpenShell` Sandbox - process isolation and monitoring. +// CLI flags are naturally boolean switches; grouping them into structs would +// only obscure the clap definition. +#[allow(clippy::struct_excessive_bools)] +#[derive(Parser, Debug)] +#[command(name = "openshell-sandbox")] +#[command(version = openshell_core::VERSION)] +#[command(about = "Process sandbox and monitor", long_about = None)] +struct Args { + /// Command to execute in the sandbox. + /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. + /// Defaults to `/bin/bash` if neither is provided. + #[arg(trailing_var_arg = true)] + command: Vec, + + /// Working directory for the sandboxed process. + #[arg(long, short)] + workdir: Option, + + /// Timeout in seconds (0 = no timeout). + #[arg(long, short, default_value = "0")] + timeout: u64, + + /// Run in interactive mode (inherit process group for terminal control). + #[arg(long, short = 'i')] + interactive: bool, + + /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. + /// Requires --openshell-endpoint to be set. + #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] + sandbox_id: Option, + + /// Sandbox (used for policy sync when the sandbox discovers policy + /// from disk or falls back to the restrictive default). + #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] + sandbox: Option, + + /// `OpenShell` server gRPC endpoint for fetching policy. + /// Required when using --sandbox-id. + #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] + openshell_endpoint: Option, + + /// Path to Rego policy file for OPA-based network access control. + /// Requires --policy-data to also be set. + #[arg(long, env = "OPENSHELL_POLICY_RULES")] + policy_rules: Option, + + /// Path to YAML data file containing network policies and sandbox config. + /// Requires --policy-rules to also be set. + #[arg(long, env = "OPENSHELL_POLICY_DATA")] + policy_data: Option, + + /// Log level (trace, debug, info, warn, error). + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, + + /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning + /// with `@` selects an abstract socket in the network namespace. + /// The supervisor bridges `RelayStream` traffic from the gateway onto + /// this socket; nothing else should connect to it. + #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] + ssh_socket_path: Option, + + /// Path to YAML inference routes for standalone routing. + /// When set, inference routes are loaded from this file instead of + /// fetching a bundle from the gateway. + #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] + inference_routes: Option, + + /// Enable health check endpoint. + #[arg(long)] + health_check: bool, + + /// Port for health check endpoint. + #[arg(long, default_value = "8080")] + health_port: u16, + + /// Which supervisor components to run. Comma-separated list of + /// "network" and/or "process". Defaults to both (single-binary + /// topology). Use --mode=network for a network-only sidecar, or + /// --mode=process for a process-only supervisor when network + /// enforcement runs in another pod. Use --mode=network-init only in + /// the Kubernetes init container that prepares sidecar nftables. + #[arg(long, default_value = DEFAULT_MODE)] + mode: Mode, + + /// UID that the long-running Kubernetes network sidecar will run as. + /// `--mode=network-init` installs nftables rules that exempt this UID. + #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] + proxy_uid: u32, + + /// GID assigned to shared sidecar state directories. Defaults to + /// `--proxy-uid` when omitted. + #[arg(long, env = "OPENSHELL_PROXY_GID")] + proxy_gid: Option, + + /// Shared state directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] + sidecar_state_dir: String, + + /// Shared TLS work directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] + sidecar_tls_dir: String, + + // Corporate upstream proxy. Operator-owned egress boundary: accepted + // only as command-line arguments (no `env =`), because the driver + // controls the supervisor's argv while a sandbox image could bake + // matching `ENV` values. + /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. + #[arg(long)] + upstream_proxy: Option, + + /// Comma-separated `NO_PROXY` list for the corporate proxy. + #[arg(long)] + upstream_no_proxy: Option, + + /// Path to the root-only file holding corporate proxy credentials (`user:pass`). + #[arg(long)] + upstream_proxy_auth_file: Option, + + /// Acknowledge that proxy credentials travel as cleartext Basic auth over + /// the plain-TCP connection to the `http://` proxy. + #[arg(long)] + upstream_proxy_auth_allow_insecure: bool, + + /// Send the destination hostname in CONNECT instead of a validated IP + /// (for proxies whose ACLs filter on hostnames). + #[arg(long)] + upstream_proxy_connect_by_hostname: bool, +} + +/// Copy the running executable to `dest`, creating parent directories as +/// needed and ensuring the result is executable (mode `0755`). +/// +/// If `dest` already exists as a directory, the binary is placed inside it +/// using the source executable's file name. This mirrors `cp` semantics so +/// callers can pass either a full target path or a directory. +fn copy_self(dest: &str) -> Result<()> { + let exe = std::env::current_exe().into_diagnostic()?; + + let dest_path = Path::new(dest); + let final_path = if dest_path.is_dir() { + let file_name = exe + .file_name() + .ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?; + dest_path.join(file_name) + } else { + dest_path.to_path_buf() + }; + + if let Some(parent) = final_path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + + std::fs::copy(&exe, &final_path).into_diagnostic()?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&final_path) + .into_diagnostic()? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&final_path, perms).into_diagnostic()?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {uid}:{gid}", + path.display() + ) + })?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + let uid = Uid::current(); + let gid = Gid::current(); + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + chown(path, Some(uid), Some(gid)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {}:{}", + path.display(), + uid.as_raw(), + gid.as_raw() + ) + })?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn copy_sidecar_client_tls_if_present( + source_dir: &Path, + sidecar_tls_dir: &Path, + uid: u32, + gid: u32, +) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + if !source_dir.exists() { + return Ok(()); + } + + let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); + prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; + for file_name in CLIENT_TLS_FILES { + let source = source_dir.join(file_name); + if !source.exists() { + return Err(miette::miette!( + "client TLS source file is missing: {}", + source.display() + )); + } + let dest = dest_dir.join(file_name); + if dest.exists() { + std::fs::remove_file(&dest) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to remove stale client TLS file {}", dest.display()) + })?; + } + std::fs::copy(&source, &dest) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to copy client TLS file {} to {}", + source.display(), + dest.display() + ) + })?; + let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); + perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); + std::fs::set_permissions(&dest, perms) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to chmod copied client TLS file {}", dest.display()) + })?; + chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown copied client TLS file {} to {uid}:{gid}", + dest.display() + ) + })?; + } + + prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn run_network_init( + proxy_user_id: u32, + proxy_primary_group_id: u32, + sidecar_state_dir: &str, + sidecar_tls_dir: &str, +) -> Result<()> { + validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; + + let sidecar_state_dir = Path::new(sidecar_state_dir); + let sidecar_tls_dir = Path::new(sidecar_tls_dir); + prepare_sidecar_directory( + sidecar_state_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_STATE_DIR_MODE, + )?; + // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the + // TLS work directory owned by the init user until the client cert copy is + // complete, then hand it to the long-running proxy UID. + prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; + copy_sidecar_client_tls_if_present( + Path::new(CLIENT_TLS_DIR), + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + )?; + prepare_sidecar_directory( + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_TLS_DIR_MODE, + )?; + openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) +} + +#[cfg(target_os = "linux")] +fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { + if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-uid must be 0 or at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn run_network_init( + _proxy_uid: u32, + _proxy_gid: u32, + _sidecar_state_dir: &str, + _sidecar_tls_dir: &str, +) -> Result<()> { + Err(miette::miette!( + "--mode=network-init is only supported on Linux" + )) +} + +fn main() -> Result<()> { + // Handle `copy-self ` before clap so it works without any of the + // sandbox flags. Kubernetes init containers invoke this path to seed an + // emptyDir volume that the agent container then executes from. + let raw_args: Vec = std::env::args().collect(); + if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { + let dest = raw_args.get(2).ok_or_else(|| { + miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") + })?; + return copy_self(dest); + } + + // Handle `debug-rpc [args]` before clap. Uses a small + // dedicated runtime so we don't pay the supervisor's full startup cost. + if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .into_diagnostic()?; + return runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; + std::process::exit(exit); + }); + } + + let args = Args::parse(); + + if args.mode.network_init { + let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + return run_network_init( + args.proxy_uid, + proxy_gid, + &args.sidecar_state_dir, + &args.sidecar_tls_dir, + ); + } + + // Try to open a rolling log file; fall back to stderr-only logging if it fails + // (e.g., /var/log is not writable in custom workload images). + // Rotates daily, keeps the 3 most recent files to bound disk usage. + let file_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + (writer, guard) + }); + + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .into_diagnostic()?; + + let exit_code = runtime.block_on(async move { + // Install rustls crypto provider before any TLS connections (including log push). + let _ = rustls::crypto::ring::default_provider().install_default(); + + // Set up optional log push layer (gRPC mode only). + let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = + (&args.sandbox_id, &args.openshell_endpoint) + { + let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( + endpoint.clone(), + sandbox_id.clone(), + ); + let layer = + openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); + Some((layer, handle)) + } else { + None + }; + let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); + let _log_push_handle = log_push_state.map(|(_, handle)| handle); + + // Shared flag: the sandbox poll loop toggles this when the + // `ocsf_json_enabled` setting changes. The JSONL layer checks it + // on each event and short-circuits when false. + let ocsf_enabled = Arc::new(AtomicBool::new(false)); + + // Keep guards alive for the entire process. When a guard is dropped the + // non-blocking writer flushes remaining logs. + let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { + let file_filter = EnvFilter::new("info"); + + // OCSF JSONL file: rolling appender matching the main log file + // (daily rotation, 3 files max). Created eagerly but gated by the + // enabled flag — no JSONL is written until ocsf_json_enabled is set. + let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell-ocsf") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + (layer, guard) + }); + let (jsonl_layer, jsonl_guard) = match jsonl_logging { + Some((layer, guard)) => (Some(layer), Some(guard)), + None => (None, None), + }; + + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with( + OcsfShorthandLayer::new(file_writer) + .with_non_ocsf(true) + .with_filter(file_filter), + ) + .with(jsonl_layer.with_filter(LevelFilter::INFO)) + .with(push_layer.clone()) + .init(); + (Some(file_guard), jsonl_guard) + } else { + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with(push_layer) + .init(); + // Log the warning after the subscriber is initialized + warn!("Could not open /var/log for log rotation; using stderr-only logging"); + (None, None) + }; + + // Get command - either from CLI args, environment variable, or default to /bin/bash + let command = if !args.command.is_empty() { + args.command + } else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) { + // Simple shell-like splitting on whitespace + c.split_whitespace().map(String::from).collect() + } else { + vec!["/bin/bash".to_string()] + }; + + info!(command = ?command, "Starting sandbox"); + // Note: "Starting sandbox" stays as plain info!() since the OCSF context + // is not yet initialized at this point (run_sandbox hasn't been called). + // The shorthand layer will render it in fallback format. + + let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { + https_proxy: args.upstream_proxy, + no_proxy: args.upstream_no_proxy, + proxy_auth_file: args.upstream_proxy_auth_file, + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, + }; + + run_sandbox( + command, + args.workdir, + args.timeout, + args.interactive, + args.sandbox_id, + args.sandbox, + args.openshell_endpoint, + args.policy_rules, + args.policy_data, + args.ssh_socket_path, + args.health_check, + args.health_port, + args.inference_routes, + ocsf_enabled, + args.mode.network, + args.mode.process, + upstream_proxy_args, + ) + .await + })?; + + std::process::exit(exit_code); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + /// Drives `copy_self`'s file-copy logic against an arbitrary source path + /// so tests don't depend on `current_exe()`. + fn copy_executable(src: &Path, dest: &Path) -> Result<()> { + let final_path = if dest.is_dir() { + dest.join(src.file_name().unwrap()) + } else { + dest.to_path_buf() + }; + if let Some(parent) = final_path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + std::fs::copy(src, &final_path).into_diagnostic()?; + let mut perms = std::fs::metadata(&final_path) + .into_diagnostic()? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&final_path, perms).into_diagnostic()?; + Ok(()) + } + + #[test] + fn copy_self_writes_executable_at_target_path() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("source-bin"); + std::fs::write(&src, b"#!/bin/false\n").unwrap(); + + let dest = tmp.path().join("subdir/openshell-sandbox"); + copy_executable(&src, &dest).unwrap(); + + assert!(dest.exists(), "destination file should exist"); + let mode = std::fs::metadata(&dest).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o755, "destination must be 0755"); + let copied = std::fs::read(&dest).unwrap(); + assert_eq!(copied, b"#!/bin/false\n"); + } + + #[test] + fn copy_self_into_existing_directory_uses_source_filename() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("openshell-sandbox"); + std::fs::write(&src, b"binary").unwrap(); + + let dest_dir = tmp.path().join("bin"); + std::fs::create_dir_all(&dest_dir).unwrap(); + + copy_executable(&src, &dest_dir).unwrap(); + + let final_path = dest_dir.join("openshell-sandbox"); + assert!(final_path.exists(), "binary should land inside dest dir"); + } + + #[test] + fn mode_parses_network_init_standalone() { + let mode = "network-init".parse::().unwrap(); + assert!(mode.network_init); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_combined_network_init() { + let err = "network-init,network".parse::().unwrap_err(); + assert!(err.contains("cannot be combined")); + } + + #[test] + fn mode_rejects_empty_value() { + let err = "".parse::().unwrap_err(); + assert!(err.contains("at least one")); + } + + #[cfg(target_os = "linux")] + #[test] + fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { + assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); + assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); + assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); + assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { + validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_still_rejects_low_non_root_proxy_ids() { + let uid_err = + validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); + assert!(uid_err.to_string().contains("--proxy-uid")); + + let gid_err = validate_network_init_ids(0, 999).unwrap_err(); + assert!(gid_err.to_string().contains("--proxy-gid")); + } +} diff --git a/crates/openshell-sandbox/tests/stdout_logging.rs b/crates/openshell-sandbox/tests/stdout_logging.rs index c4f5213de8..6feb91f869 100644 --- a/crates/openshell-sandbox/tests/stdout_logging.rs +++ b/crates/openshell-sandbox/tests/stdout_logging.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![cfg(not(target_os = "windows"))] + use std::process::Command; #[test] diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 53124261e6..5cb816ee62 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -53,10 +53,12 @@ use openshell_core::{ use openshell_ocsf::{ ConfigStateChangeBuilder, OCSF_TARGET, OcsfEvent, SandboxContext, SeverityId, StateId, StatusId, }; +#[cfg(not(target_os = "windows"))] +use openshell_policy::serialize_sandbox_policy; use openshell_policy::{ PolicyMergeOp, ProviderPolicyLayer, compose_effective_policy, merge_policy, - serialize_sandbox_policy, }; +#[cfg(not(target_os = "windows"))] use openshell_prover::{ credentials::{Credential, CredentialSet}, finding::{Finding, FindingPath}, @@ -69,7 +71,9 @@ use openshell_prover::{ use openshell_providers::normalize_provider_type; use prost::Message; use sha2::{Digest, Sha256}; -use std::collections::{BTreeMap, HashMap, HashSet}; +#[cfg(not(target_os = "windows"))] +use std::collections::HashSet; +use std::collections::{BTreeMap, HashMap}; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use tonic::{Request, Response, Status}; @@ -440,6 +444,7 @@ fn summarize_draft_chunk_rule(chunk: &DraftChunkRecord) -> Result String { + let merge_op = PolicyMergeOp::AddRule { + rule_name: rule_name.to_string(), + rule: proposed_rule.clone(), + }; + let merged = match merge_policy(current_policy, &[merge_op]) { + Ok(result) => result.policy, + Err(error) => return format!("merge failed: {}", one_line(&error.to_string())), + }; + if let Err(error) = validate_policy_safety(&merged) { + return format!("policy invalid: {}", one_line(&error.to_string())); + } + "validation unavailable".to_string() +} + /// Run the prover end-to-end against a single policy with the given /// credential set. Returns the raw finding list, or a short error string /// identifying which infrastructure step failed. /// /// The credential set is passed in because it's stable across all chunks in /// one `SubmitPolicyAnalysis` batch — the caller builds it once and shares. +#[cfg(not(target_os = "windows"))] fn run_prover_findings( policy: &ProtoSandboxPolicy, credentials: &CredentialSet, @@ -522,6 +548,7 @@ fn run_prover_findings( /// a `warn!` — the merged policy already excludes them at compose time, so /// silently treating them as absent here keeps the credential set consistent /// with the merged policy the prover validates against. +#[cfg(not(target_os = "windows"))] async fn build_credential_set_for_sandbox_with_catalog( store: &Store, catalog: &EffectiveProviderProfileCatalog, @@ -586,6 +613,7 @@ async fn build_credential_set_for_sandbox_with_catalog( /// the same security gap whether they live in rule `foo` or rule `bar`. This /// keeps the delta from spuriously surfacing baseline gaps just because the /// proposal added a new rule name that produces the same gap shape. +#[cfg(not(target_os = "windows"))] fn finding_path_key(path: &FindingPath) -> String { let FindingPath::Exfil(p) = path; // Include the category and (for capability_expansion) the method so @@ -607,6 +635,7 @@ fn finding_path_key(path: &FindingPath) -> String { /// are suppressed. A brand-new credentialed reach is described by the /// reach-expansion finding alone; we don't double-report by also /// flagging every method as a separate `capability_expansion`. +#[cfg(not(target_os = "windows"))] fn finding_delta(base: &[Finding], merged: &[Finding]) -> Vec { use openshell_prover::finding::category; @@ -1097,7 +1126,7 @@ fn truncate_for_log(input: &str, max_chars: usize) -> String { } } -#[cfg(test)] +#[cfg(all(test, not(target_os = "windows")))] fn is_sandbox_caller(request: &Request) -> bool { matches!( request.extensions().get::(), @@ -2657,18 +2686,21 @@ pub(super) async fn handle_submit_policy_analysis( // it once. v1 captures presence only — no scope modeling — so the prover // can answer "is there a credential in scope for this host?" but not // "what action class does that credential authorize?" - let provider_names_for_creds: Vec = sandbox - .spec - .as_ref() - .map(|spec| spec.providers.clone()) - .unwrap_or_default(); - let credential_set = build_credential_set_for_sandbox_with_catalog( - state.store.as_ref(), - &provider_profile_catalog, - &workspace, - &provider_names_for_creds, - ) - .await?; + #[cfg(not(target_os = "windows"))] + let credential_set = { + let provider_names_for_creds: Vec = sandbox + .spec + .as_ref() + .map(|spec| spec.providers.clone()) + .unwrap_or_default(); + build_credential_set_for_sandbox_with_catalog( + state.store.as_ref(), + &provider_profile_catalog, + &workspace, + &provider_names_for_creds, + ) + .await? + }; let current_version = state .store @@ -2732,12 +2764,19 @@ pub(super) async fn handle_submit_policy_analysis( // Source provenance (mechanistic vs agent_authored) is preserved in // OCSF audit fields, but the safety decision is grounded in the // merged-policy consequence, not the author — proposer-agnostic. + #[cfg(not(target_os = "windows"))] let validation_result = validation_result_for_agent_proposal( current_policy.clone(), &chunk.rule_name, rule_ref, &credential_set, ); + #[cfg(target_os = "windows")] + let validation_result = validation_result_for_agent_proposal( + current_policy.clone(), + &chunk.rule_name, + chunk.proposed_rule.as_ref().expect("checked above"), + ); let record = DraftChunkRecord { // The handler proposes an id; the store may swap it for an @@ -4504,7 +4543,7 @@ fn materialize_global_settings( // Tests // --------------------------------------------------------------------------- -#[cfg(test)] +#[cfg(all(test, not(target_os = "windows")))] mod tests { use super::*; use crate::auth::identity::{Identity, IdentityProvider}; diff --git a/crates/openshell-supervisor-network/tests/system_inference.rs b/crates/openshell-supervisor-network/tests/system_inference.rs index 3c8e6ee8fc..189cbb6d53 100644 --- a/crates/openshell-supervisor-network/tests/system_inference.rs +++ b/crates/openshell-supervisor-network/tests/system_inference.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![cfg(not(target_os = "windows"))] + //! Integration test for the in-process system inference API. //! //! Uses the router's built-in `mock://` route support to verify the full diff --git a/crates/openshell-supervisor-network/tests/websocket_upgrade.rs b/crates/openshell-supervisor-network/tests/websocket_upgrade.rs index 322d6709cd..f9586b10e0 100644 --- a/crates/openshell-supervisor-network/tests/websocket_upgrade.rs +++ b/crates/openshell-supervisor-network/tests/websocket_upgrade.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![cfg(not(target_os = "windows"))] #![allow( clippy::match_same_arms, clippy::cast_possible_truncation, diff --git a/crates/openshell-vfio/src/bind.rs b/crates/openshell-vfio/src/bind.rs index 606f56c978..8aa44ffd16 100644 --- a/crates/openshell-vfio/src/bind.rs +++ b/crates/openshell-vfio/src/bind.rs @@ -310,13 +310,13 @@ impl VfioIdRegistry { self.refcounts.clear(); } - #[cfg(test)] + #[cfg(all(test, unix))] fn remove(&mut self, id: &str) { self.refcounts.remove(id); } } -#[cfg(test)] +#[cfg(all(test, unix))] pub(crate) mod test_refcounts { use super::VFIO_ID_REGISTRY; use std::sync::{Mutex, MutexGuard, PoisonError}; @@ -337,7 +337,7 @@ pub(crate) mod test_refcounts { } } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use super::*; use crate::test_support::{ diff --git a/crates/openshell-vfio/src/gpu.rs b/crates/openshell-vfio/src/gpu.rs index 6c421fdeeb..eb356f046b 100644 --- a/crates/openshell-vfio/src/gpu.rs +++ b/crates/openshell-vfio/src/gpu.rs @@ -162,7 +162,7 @@ pub fn prepare_gpu_for_passthrough( )) } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use super::*; use crate::bind::test_refcounts; diff --git a/crates/openshell-vfio/src/lib.rs b/crates/openshell-vfio/src/lib.rs index 9174c78932..c773f18d3b 100644 --- a/crates/openshell-vfio/src/lib.rs +++ b/crates/openshell-vfio/src/lib.rs @@ -22,7 +22,7 @@ pub mod pci; pub mod reconcile; pub mod sysfs; -#[cfg(test)] +#[cfg(all(test, unix))] mod test_support; pub use error::VfioError; diff --git a/crates/openshell-vfio/src/pci.rs b/crates/openshell-vfio/src/pci.rs index 2db6dd0ece..83e3eba54e 100644 --- a/crates/openshell-vfio/src/pci.rs +++ b/crates/openshell-vfio/src/pci.rs @@ -523,7 +523,7 @@ pub fn release_pci_group_from_passthrough( first_err.map_or(Ok(()), Err) } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use super::*; use crate::bind::test_refcounts; diff --git a/crates/openshell-vfio/src/reconcile.rs b/crates/openshell-vfio/src/reconcile.rs index 6ab2d8b3bb..be0b959bfa 100644 --- a/crates/openshell-vfio/src/reconcile.rs +++ b/crates/openshell-vfio/src/reconcile.rs @@ -100,7 +100,7 @@ pub fn reconcile_stale_bindings(sysfs: &SysfsRoot, state_path: &Path) -> Vec &Path { &self.base } @@ -223,7 +223,7 @@ pub(crate) fn write_sysfs(path: &Path, value: &str) -> Result<(), VfioError> { }) } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use super::*; use crate::test_support::{create_pci_device, setup_mock_sysfs}; From e4b776854a9f8a923d8762f195bb0a3830d139aa Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Mon, 1 Jun 2026 11:13:22 -0700 Subject: [PATCH 02/30] feat(windows): stub unsupported compute drivers Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- crates/openshell-driver-docker/Cargo.toml | 3 +- crates/openshell-driver-docker/src/lib.rs | 3523 +---------------- .../openshell-driver-docker/src/lib_unix.rs | 3522 ++++++++++++++++ crates/openshell-driver-docker/src/windows.rs | 77 + crates/openshell-driver-kubernetes/Cargo.toml | 4 +- crates/openshell-driver-kubernetes/src/lib.rs | 6 + .../openshell-driver-kubernetes/src/main.rs | 182 +- .../src/main_unix.rs | 181 + crates/openshell-driver-podman/Cargo.toml | 5 +- crates/openshell-driver-podman/src/lib.rs | 18 +- .../openshell-driver-podman/src/lib_unix.rs | 15 + crates/openshell-driver-podman/src/main.rs | 186 +- .../openshell-driver-podman/src/main_unix.rs | 185 + crates/openshell-driver-podman/src/windows.rs | 66 + crates/openshell-driver-vm/Cargo.toml | 2 +- crates/openshell-driver-vm/src/lib.rs | 25 +- crates/openshell-driver-vm/src/lib_unix.rs | 23 + crates/openshell-driver-vm/src/main.rs | 729 +--- crates/openshell-driver-vm/src/main_unix.rs | 728 ++++ crates/openshell-server/Cargo.toml | 2 +- crates/openshell-server/src/cli.rs | 8 +- crates/openshell-server/src/compute/mod.rs | 74 +- crates/openshell-server/src/compute/vm.rs | 32 +- .../src/supervisor_session.rs | 1 + 24 files changed, 4961 insertions(+), 4636 deletions(-) create mode 100644 crates/openshell-driver-docker/src/lib_unix.rs create mode 100644 crates/openshell-driver-docker/src/windows.rs create mode 100644 crates/openshell-driver-kubernetes/src/main_unix.rs create mode 100644 crates/openshell-driver-podman/src/lib_unix.rs create mode 100644 crates/openshell-driver-podman/src/main_unix.rs create mode 100644 crates/openshell-driver-podman/src/windows.rs create mode 100644 crates/openshell-driver-vm/src/lib_unix.rs create mode 100644 crates/openshell-driver-vm/src/main_unix.rs diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7e1bc069cb..296274bf87 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -12,14 +12,15 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +serde = { workspace = true } +[target.'cfg(not(target_os = "windows"))'.dependencies] tokio = { workspace = true } tonic = { workspace = true } futures = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } bytes = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } prost-types = { workspace = true } bollard = { version = "0.20" } diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 2f89c2229e..1db6e9f00f 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1,3522 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Docker compute driver. +#[cfg(target_os = "windows")] +mod windows; -#![allow(clippy::result_large_err)] +#[cfg(target_os = "windows")] +pub use windows::{DockerComputeConfig, default_docker_supervisor_image}; -use bollard::Docker; -use bollard::errors::Error as BollardError; -use bollard::models::{ - ContainerCreateBody, ContainerSummary, ContainerSummaryStateEnum, CreateImageInfo, - DeviceRequest, EndpointSettings, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, - MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, ProgressDetail, RestartPolicy, - RestartPolicyNameEnum, SystemInfo, -}; -use bollard::query_parameters::{ - CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, - ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, -}; -use bytes::Bytes; -use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; -use openshell_core::driver_mounts; -use openshell_core::driver_utils::{ - LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - supervisor_image_should_refresh, -}; -use openshell_core::gpu::{ - CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, - effective_driver_gpu_count, validate_specific_gpu_device_request, -}; -use openshell_core::progress::{ - PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX, - format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail, -}; -use openshell_core::proto::compute::v1::{ - CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, watch_sandboxes_event, -}; -use openshell_core::proto_struct::{ - deserialize_optional_non_empty_string_list, struct_to_json_value, -}; -use openshell_core::{Config, Error, Result as CoreResult}; -use std::collections::{HashMap, HashSet}; -use std::io::Read; -use std::net::{IpAddr, SocketAddr}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; -use tokio::task::JoinHandle; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status}; -use tracing::{info, warn}; -use url::Url; - -const WATCH_BUFFER: usize = 128; -const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); -const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); - -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const SANDBOX_COMMAND: &str = "sleep infinity"; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; -const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; -const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; -const DOCKER_NETWORK_DRIVER: &str = "bridge"; - -/// Queried by the Docker driver to decide when a sandbox's supervisor -/// relay is live. Implementations return `true` once a sandbox has an -/// active `ConnectSupervisor` session registered. -/// -/// The driver cannot observe the supervisor's SSH socket directly (it -/// lives inside the container), so it leans on this signal to flip the -/// Ready condition from `DependenciesNotReady` to `True`. -pub trait SupervisorReadiness: Send + Sync + 'static { - fn is_supervisor_connected(&self, sandbox_id: &str) -> bool; -} - -/// Gateway-local configuration for the Docker compute driver. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct DockerComputeConfig { - /// Docker API Unix socket. When unset, use the socket selected by gateway - /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly - /// configured Docker driver. - pub socket_path: Option, - - /// Default OCI image for sandboxes. - pub default_image: String, - - /// Image pull policy for sandbox images. - pub image_pull_policy: String, - - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, - - /// Gateway gRPC endpoint the sandbox connects back to. - pub grpc_endpoint: String, - - /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. - pub supervisor_bin: Option, - - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. - pub supervisor_image: Option, - - /// Host-side CA certificate for Docker sandbox mTLS. - pub guest_tls_ca: Option, - - /// Host-side client certificate for Docker sandbox mTLS. - pub guest_tls_cert: Option, - - /// Host-side private key for Docker sandbox mTLS. - pub guest_tls_key: Option, - - /// Docker bridge network that sandbox containers join. - pub network_name: String, - - /// Host gateway IP used for sandbox host aliases. - pub host_gateway_ip: String, - - /// Unix socket path the in-container supervisor bridges relay traffic to. - pub ssh_socket_path: String, - - /// Container cgroup PID limit for Docker-managed sandboxes. - /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, - - /// Allow sandbox requests to attach host bind mounts through - /// `template.driver_config`. - #[serde(default)] - pub enable_bind_mounts: bool, -} - -impl Default for DockerComputeConfig { - fn default() -> Self { - Self { - socket_path: None, - default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), - grpc_endpoint: String::new(), - supervisor_bin: None, - supervisor_image: None, - guest_tls_ca: None, - guest_tls_cert: None, - guest_tls_key: None, - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - host_gateway_ip: String::new(), - ssh_socket_path: "/run/openshell/ssh.sock".to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, - enable_bind_mounts: false, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct DockerGuestTlsPaths { - pub(crate) ca: PathBuf, - pub(crate) cert: PathBuf, - pub(crate) key: PathBuf, -} - -#[derive(Debug, Clone)] -struct DockerDriverRuntimeConfig { - default_image: String, - image_pull_policy: String, - sandbox_namespace: String, - grpc_endpoint: String, - network_name: String, - gateway_route: DockerGatewayRoute, - ssh_socket_path: String, - stop_timeout_secs: u32, - log_level: String, - supervisor_bin: PathBuf, - guest_tls: Option, - daemon_version: String, - supports_gpu: bool, - allow_all_default_gpu: bool, - sandbox_pids_limit: i64, - enable_bind_mounts: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, - HostGateway, -} - -#[derive(Clone)] -pub struct DockerComputeDriver { - docker: Arc, - config: DockerDriverRuntimeConfig, - events: broadcast::Sender, - pending: Arc>>, - supervisor_readiness: Arc, - gpu_selector: Arc, -} - -struct PendingSandboxRecord { - sandbox: DriverSandbox, - task: Option>, -} - -#[derive(Debug, Clone)] -struct DockerProvisioningFailure { - reason: &'static str, - message: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -struct DockerResourceLimits { - nano_cpus: Option, - memory_bytes: Option, -} - -#[derive(Debug, Clone, Default, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -struct DockerSandboxDriverConfig { - #[serde( - default, - deserialize_with = "deserialize_optional_non_empty_string_list" - )] - cdi_devices: Option>, - mounts: Vec, -} - -struct ValidatedDockerSandbox<'a> { - template: &'a DriverSandboxTemplate, - driver_config: DockerSandboxDriverConfig, - gpu_requirements: Option<&'a GpuResourceRequirements>, -} - -impl DockerSandboxDriverConfig { - fn from_template(template: &DriverSandboxTemplate) -> Result { - let Some(config) = template.driver_config.as_ref() else { - return Ok(Self::default()); - }; - - serde_json::from_value(struct_to_json_value(config)) - .map_err(|err| format!("invalid docker driver_config: {err}")) - } -} - -use openshell_core::driver_mounts::SelinuxLabel; - -#[derive(Debug, Clone, serde::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum DockerDriverMountConfig { - Bind { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - selinux_label: Option, - }, - Volume { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, - Tmpfs { - target: String, - #[serde(default)] - options: Vec, - #[serde(default)] - size_bytes: Option, - #[serde(default)] - mode: Option, - }, - Image { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, -} - -fn default_true() -> bool { - true -} - -type WatchStream = - Pin> + Send + 'static>>; - -impl DockerComputeDriver { - pub async fn new( - config: &Config, - docker_config: &DockerComputeConfig, - supervisor_readiness: Arc, - ) -> CoreResult { - let socket_path = docker_config - .socket_path - .clone() - .or_else(openshell_core::config::detect_docker_socket) - .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); - let socket_path_str = socket_path.to_str().ok_or_else(|| { - Error::config(format!( - "Docker socket path is not valid UTF-8: {}", - socket_path.display() - )) - })?; - let docker = - Docker::connect_with_socket(socket_path_str, 120, bollard::API_DEFAULT_VERSION) - .map_err(|err| { - Error::execution(format!("failed to create Docker client: {err}")) - })?; - let version = docker.version().await.map_err(|err| { - Error::execution(format!("failed to query Docker daemon version: {err}")) - })?; - let info = docker.info().await.map_err(|err| { - Error::execution(format!("failed to query Docker daemon info: {err}")) - })?; - let supports_gpu = info - .cdi_spec_dirs - .as_ref() - .is_some_and(|dirs| !dirs.is_empty()); - let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); - let allow_all_default_gpu = docker_info_reports_wsl2(&info); - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - let gateway_port = config.bind_address.port(); - if gateway_port == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } - let network_name = docker_network_name(docker_config); - let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; - let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; - let gateway_route = - docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); - let mut docker_config = docker_config.clone(); - if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); - } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, - ); - let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); - let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; - let guest_tls = docker_guest_tls_paths(&docker_config)?; - - let driver = Self { - docker: Arc::new(docker), - config: DockerDriverRuntimeConfig { - default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), - grpc_endpoint, - network_name, - gateway_route, - ssh_socket_path: docker_config.ssh_socket_path.clone(), - stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, - log_level: config.log_level.clone(), - supervisor_bin, - guest_tls, - daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), - supports_gpu, - allow_all_default_gpu, - sandbox_pids_limit: docker_config.sandbox_pids_limit, - enable_bind_mounts: docker_config.enable_bind_mounts, - }, - events: broadcast::channel(WATCH_BUFFER).0, - pending: Arc::new(Mutex::new(HashMap::new())), - supervisor_readiness, - gpu_selector: Arc::new(CdiGpuDefaultSelector::new( - cdi_gpu_inventory, - allow_all_default_gpu, - )), - }; - - let poll_driver = driver.clone(); - tokio::spawn(async move { - poll_driver.poll_loop().await; - }); - - Ok(driver) - } - - #[must_use] - pub fn gateway_bind_addresses(&self) -> Vec { - match self.config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], - DockerGatewayRoute::HostGateway => Vec::new(), - } - } - - fn capabilities(&self) -> GetCapabilitiesResponse { - openshell_core::driver_utils::build_capabilities_response( - "docker", - &self.config.daemon_version, - &self.config.default_image, - ) - } - - #[cfg(test)] - fn validate_sandbox( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - ) -> Result<(), Status> { - let _ = Self::validated_sandbox(sandbox, config)?; - Ok(()) - } - - fn validated_sandbox<'a>( - sandbox: &'a DriverSandbox, - config: &DockerDriverRuntimeConfig, - ) -> Result, Status> { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - - Self::validate_sandbox_template_base(template)?; - let _ = docker_resource_limits(template)?; - let driver_config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; - let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); - Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?; - Ok(ValidatedDockerSandbox { - template, - driver_config, - gpu_requirements, - }) - } - - fn validate_sandbox_template_base(template: &DriverSandboxTemplate) -> Result<(), Status> { - if template.image.trim().is_empty() { - return Err(Status::failed_precondition( - "docker sandboxes require a template image", - )); - } - if !template.agent_socket_path.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support template.agent_socket_path", - )); - } - if template - .platform_config - .as_ref() - .is_some_and(|config| !config.fields.is_empty()) - { - return Err(Status::failed_precondition( - "docker compute driver does not support template.platform_config", - )); - } - - Ok(()) - } - - fn validate_sandbox_auth(sandbox: &DriverSandbox) -> Result<(), Status> { - let token_present = sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.trim().is_empty()); - if token_present { - return Ok(()); - } - - Err(Status::failed_precondition( - "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]", - )) - } - - fn validate_gpu_request( - gpu_requirements: Option<&GpuResourceRequirements>, - supports_gpu: bool, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - let requested_count = - effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)?; - if requested_count.is_some() && !supports_gpu { - return Err(Status::failed_precondition( - "docker GPU sandboxes require Docker CDI support. Enable CDI on the Docker daemon, then restart the OpenShell gateway/server so GPU capability is detected.", - )); - } - - if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - } - - Ok(()) - } - - async fn validate_user_volume_mounts_available( - &self, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - for mount in &driver_config.mounts { - if let DockerDriverMountConfig::Volume { source, .. } = mount { - match self.docker.inspect_volume(source).await { - Ok(volume) => { - if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) - { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]" - ))); - } - } - Err(err) if is_not_found_error(&err) => { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' does not exist" - ))); - } - Err(err) => { - return Err(internal_status("inspect docker volume", err)); - } - } - } - } - Ok(()) - } - - async fn refresh_gpu_inventory(&self) -> Result<(), Status> { - let info = self - .docker - .info() - .await - .map_err(|err| internal_status("query Docker daemon info", err))?; - self.gpu_selector.refresh( - docker_cdi_gpu_inventory(&info), - self.config.allow_all_default_gpu, - ); - Ok(()) - } - - async fn resolve_gpu_cdi_devices( - &self, - gpu_requirements: Option<&GpuResourceRequirements>, - driver_config: &DockerSandboxDriverConfig, - select_default_devices: fn( - &CdiGpuDefaultSelector, - u32, - ) -> Result, CdiGpuSelectionError>, - ) -> Result>, Status> { - if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - return Ok(Some(cdi_devices.to_vec())); - } - - let Some(count) = - effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)? - else { - return Ok(None); - }; - - self.refresh_gpu_inventory().await?; - select_default_devices(&self.gpu_selector, count) - .map(Some) - .map_err(docker_gpu_selection_status) - } - - async fn get_sandbox_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result, Status> { - let container = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await?; - if let Some(sandbox) = container.and_then(|summary| { - sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) - }) { - return Ok(Some(sandbox)); - } - - Ok(self.pending_snapshot(sandbox_id, sandbox_name).await) - } - - async fn current_snapshots(&self) -> Result, Status> { - let containers = self.list_managed_container_summaries().await?; - let container_sandboxes = containers - .iter() - .filter_map(|summary| { - sandbox_from_container_summary(summary, self.supervisor_readiness.as_ref()) - }) - .collect::>(); - let mut by_id = self.pending_snapshot_map().await; - for sandbox in container_sandboxes { - by_id.insert(sandbox.id.clone(), sandbox); - } - let mut sandboxes = by_id.into_values().collect::>(); - sandboxes.sort_by(|left, right| left.id.cmp(&right.id)); - Ok(sandboxes) - } - - async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { - let validated = Self::validated_sandbox(sandbox, &self.config)?; - Self::validate_sandbox_auth(sandbox)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; - let gpu_devices = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::peek_device_ids, - ) - .await?; - let _ = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - )?; - - if self - .find_managed_container_summary(&sandbox.id, &sandbox.name) - .await? - .is_some() - { - return Err(Status::already_exists("sandbox already exists")); - } - - self.reserve_pending_sandbox(sandbox).await?; - let image = sandbox_image(sandbox).unwrap_or_default(); - self.publish_docker_progress( - &sandbox.id, - "Scheduled", - format!("Docker sandbox accepted for image \"{image}\""), - HashMap::from([("image_ref".to_string(), image)]), - ); - self.publish_sandbox_snapshot(pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - provisioning_condition(), - false, - )); - - let driver = self.clone(); - let sandbox_for_task = sandbox.clone(); - let sandbox_id = sandbox.id.clone(); - let task = tokio::spawn(async move { - driver.provision_sandbox(sandbox_for_task).await; - }); - - let mut pending = self.pending.lock().await; - if let Some(record) = pending.get_mut(&sandbox_id) { - record.task = Some(task); - } else { - task.abort(); - } - - Ok(()) - } - - async fn provision_sandbox(&self, sandbox: DriverSandbox) { - match self.provision_sandbox_inner(&sandbox).await { - Ok(()) => { - self.clear_pending_sandbox(&sandbox.id).await; - } - Err(failure) => { - self.fail_pending_sandbox(&sandbox, &failure).await; - } - } - } - - async fn provision_sandbox_inner( - &self, - sandbox: &DriverSandbox, - ) -> Result<(), DockerProvisioningFailure> { - let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let template = validated.template; - self.ensure_image_available(&sandbox.id, &template.image) - .await - .map_err(|status| { - DockerProvisioningFailure::new("ImagePullFailed", status.message()) - })?; - let token_file_created = write_sandbox_token_file(sandbox, &self.config) - .await - .map_err(|status| { - DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) - })?; - - let container_name = container_name_for_sandbox(sandbox); - let gpu_devices = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::next_device_ids, - ) - .await - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let create_body = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - ) - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, - ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - })?; - self.publish_docker_progress( - &sandbox.id, - "Created", - format!("Created Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name.clone())]), - ); - - if let Err(err) = self.docker.start_container(&container_name, None).await { - let cleanup = self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await; - if let Err(cleanup_err) = cleanup { - warn!( - sandbox_id = %sandbox.id, - container_name, - error = %cleanup_err, - "Failed to clean up Docker container after start failure" - ); - } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - return Err(DockerProvisioningFailure::from_status( - "ContainerStartFailed", - create_status_from_docker_error("start docker sandbox container", err), - )); - } - self.publish_docker_progress( - &sandbox.id, - "Started", - format!("Started Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name)]), - ); - if let Err(err) = self - .publish_container_snapshot(&sandbox.id, &sandbox.name) - .await - { - warn!( - sandbox_id = %sandbox.id, - error = %err, - "Failed to publish Docker sandbox snapshot after start" - ); - } - - Ok(()) - } - - async fn delete_sandbox_inner( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; - if let Some(record) = pending.as_ref() - && let Some(task) = record.task.as_ref() - { - task.abort(); - } - - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = pending { - let container_name = container_name_for_sandbox(&record.sandbox); - match self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) => { - return Err(internal_status("delete docker sandbox container", err)); - } - } - } - return Ok(false); - }; - let Some(target) = summary_container_target(&container) else { - return Ok(pending.is_some()); - }; - - match self - .docker - .remove_container( - &target, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); - Ok(true) - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); - Ok(pending.is_some()) - } - Err(err) => Err(internal_status("delete docker sandbox container", err)), - } - } - - async fn stop_sandbox_inner(&self, sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { - if let Some(task) = record.task { - task.abort(); - } - cleanup_sandbox_token_file(&record.sandbox, &self.config); - self.publish_deleted(record.sandbox.id); - return Ok(()); - } - return Err(Status::not_found("sandbox not found")); - }; - let Some(target) = summary_container_target(&container) else { - return Err(Status::not_found("sandbox container has no id or name")); - }; - - match self - .docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) - .build(), - ), - ) - .await - { - Ok(()) => Ok(()), - Err(err) if is_not_modified_error(&err) => Ok(()), - Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), - Err(err) => Err(internal_status("stop docker sandbox container", err)), - } - } - - /// Start a managed sandbox container that was previously stopped. Used - /// by the gateway to resume sandboxes after a restart so that running - /// state in the gateway store is matched by an actually-running - /// container. - /// - /// Returns `Ok(true)` when a container existed and was started (or was - /// already running), `Ok(false)` when no managed container is found for - /// the sandbox, and `Err(...)` for any Docker failure. - pub async fn resume_sandbox( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - return Ok(false); - }; - let Some(target) = summary_container_target(&container) else { - return Ok(false); - }; - let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if !container_state_needs_resume(state) { - return Ok(true); - } - - match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), - // Already running — race with another resume path or the - // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), - } - } - - pub async fn stop_managed_containers_on_shutdown(&self) -> Result { - let containers = self.list_managed_container_summaries().await?; - let targets = containers - .into_iter() - .filter_map(|container| { - let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if container_state_needs_shutdown_stop(state) { - summary_container_target(&container) - } else { - None - } - }) - .collect::>(); - let target_count = targets.len(); - let mut stopped = 0usize; - let mut failures = Vec::new(); - let stop_timeout_secs = self.config.stop_timeout_secs; - - let mut stop_results = futures::stream::iter(targets.into_iter().map(|target| { - let docker = self.docker.clone(); - async move { - let result = docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(stop_timeout_secs)) - .build(), - ), - ) - .await; - (target, result) - } - })) - .buffer_unordered(16); - - while let Some((target, result)) = stop_results.next().await { - match result { - Ok(()) => { - stopped += 1; - } - Err(err) if is_not_found_error(&err) || is_not_modified_error(&err) => {} - Err(err) => { - warn!( - container = %target, - error = %err, - "Failed to stop Docker sandbox container during shutdown" - ); - failures.push(target); - } - } - } - - if !failures.is_empty() { - return Err(Status::internal(format!( - "failed to stop {} of {target_count} Docker sandbox containers during shutdown", - failures.len() - ))); - } - - Ok(stopped) - } - - async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { - let mut pending = self.pending.lock().await; - if pending - .values() - .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) - { - return Err(Status::already_exists("sandbox already exists")); - } - - pending.insert( - sandbox.id.clone(), - PendingSandboxRecord { - sandbox: pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - provisioning_condition(), - false, - ), - task: None, - }, - ); - Ok(()) - } - - async fn pending_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Option { - let pending = self.pending.lock().await; - pending - .values() - .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) - .map(|record| record.sandbox.clone()) - } - - async fn pending_snapshot_map(&self) -> HashMap { - let pending = self.pending.lock().await; - pending - .iter() - .map(|(sandbox_id, record)| (sandbox_id.clone(), record.sandbox.clone())) - .collect() - } - - async fn clear_pending_sandbox(&self, sandbox_id: &str) { - let mut pending = self.pending.lock().await; - pending.remove(sandbox_id); - } - - async fn remove_pending_sandbox( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Option { - let mut pending = self.pending.lock().await; - let id = pending.iter().find_map(|(id, record)| { - pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) - })?; - pending.remove(&id) - } - - async fn fail_pending_sandbox( - &self, - sandbox: &DriverSandbox, - failure: &DockerProvisioningFailure, - ) { - cleanup_sandbox_token_file(sandbox, &self.config); - let snapshot = pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - error_condition(failure.reason, &failure.message), - false, - ); - { - let mut pending = self.pending.lock().await; - if let Some(record) = pending.get_mut(&sandbox.id) { - record.sandbox = snapshot.clone(); - record.task = None; - } else { - return; - } - } - - self.publish_platform_event( - sandbox.id.clone(), - platform_event( - "docker", - "Warning", - failure.reason, - format!("Docker sandbox provisioning failed: {}", failure.message), - ), - ); - self.publish_sandbox_snapshot(snapshot); - } - - async fn publish_container_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result<(), Status> { - if let Some(summary) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - && let Some(sandbox) = - sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) - { - self.publish_sandbox_snapshot(sandbox); - } - Ok(()) - } - - fn publish_sandbox_snapshot(&self, sandbox: DriverSandbox) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox), - }, - )), - }); - } - - fn publish_deleted(&self, sandbox_id: String) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { sandbox_id }, - )), - }); - } - - fn publish_platform_event(&self, sandbox_id: String, event: DriverPlatformEvent) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::PlatformEvent( - WatchSandboxesPlatformEvent { - sandbox_id, - event: Some(event), - }, - )), - }); - } - - fn publish_docker_progress( - &self, - sandbox_id: &str, - reason: &str, - message: String, - mut metadata: HashMap, - ) { - attach_docker_progress_metadata(&mut metadata, reason, &message); - self.publish_platform_event( - sandbox_id.to_string(), - DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: "docker".to_string(), - r#type: "Normal".to_string(), - reason: reason.to_string(), - message, - metadata, - }, - ); - } - - async fn poll_loop(self) { - let mut previous = match self.current_snapshot_map().await { - Ok(snapshots) => snapshots, - Err(err) => { - warn!(error = %err, "Failed to seed Docker sandbox watch state"); - HashMap::new() - } - }; - - // Exponential backoff on consecutive Docker failures to avoid a 2s - // warn-log flood when the daemon is unreachable for an extended - // period (e.g. restart, socket removed). - let mut backoff = WATCH_POLL_INTERVAL; - loop { - tokio::time::sleep(backoff).await; - match self.current_snapshot_map().await { - Ok(current) => { - emit_snapshot_diff(&self.events, &previous, ¤t); - previous = current; - backoff = WATCH_POLL_INTERVAL; - } - Err(err) => { - warn!( - error = %err, - backoff_secs = backoff.as_secs(), - "Failed to poll Docker sandboxes" - ); - backoff = (backoff * 2).min(WATCH_POLL_MAX_BACKOFF); - } - } - } - } - - async fn current_snapshot_map(&self) -> Result, Status> { - self.current_snapshots().await.map(|snapshots| { - snapshots - .into_iter() - .map(|sandbox| (sandbox.id.clone(), sandbox)) - .collect() - }) - } - - async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); - self.docker - .list_containers(Some( - ListContainersOptionsBuilder::default() - .all(true) - .filters(&filters) - .build(), - )) - .await - .map_err(|err| internal_status("list Docker sandbox containers", err)) - } - - async fn find_managed_container_summary( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result, Status> { - let mut label_filter_values = Vec::new(); - if !sandbox_id.is_empty() { - label_filter_values.push(format!("{LABEL_SANDBOX_ID}={sandbox_id}")); - } else if !sandbox_name.is_empty() { - label_filter_values.push(format!("{LABEL_SANDBOX_NAME}={sandbox_name}")); - } - - let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); - let containers = self - .docker - .list_containers(Some( - ListContainersOptionsBuilder::default() - .all(true) - .filters(&filters) - .build(), - )) - .await - .map_err(|err| internal_status("find Docker sandbox container", err))?; - - Ok(containers.into_iter().find(|summary| { - let Some(labels) = summary.labels.as_ref() else { - return false; - }; - let namespace_matches = labels - .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); - let id_matches = sandbox_id.is_empty() - || labels - .get(LABEL_SANDBOX_ID) - .is_some_and(|value| value == sandbox_id); - let name_matches = sandbox_name.is_empty() - || labels - .get(LABEL_SANDBOX_NAME) - .is_some_and(|value| value == sandbox_name); - namespace_matches && id_matches && name_matches - })) - } - - async fn ensure_image_available(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - match policy.as_str() { - "" | "ifnotpresent" => { - if self.docker.inspect_image(image).await.is_ok() { - self.publish_docker_progress( - sandbox_id, - "ImagePresent", - format!("Docker image \"{image}\" is already present"), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - return Ok(()); - } - self.pull_image(sandbox_id, image).await - } - "always" => self.pull_image(sandbox_id, image).await, - "never" => match self.docker.inspect_image(image).await { - Ok(_) => { - self.publish_docker_progress( - sandbox_id, - "ImagePresent", - format!("Docker image \"{image}\" is already present"), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - Ok(()) - } - Err(err) if is_not_found_error(&err) => Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" - ))), - Err(err) => Err(internal_status("inspect Docker image", err)), - }, - other => Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))), - } - } - - async fn pull_image(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { - self.publish_docker_progress( - sandbox_id, - "Pulling", - format!("Pulling Docker image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - let mut stream = self.docker.create_image( - Some(CreateImageOptions { - from_image: Some(image.to_string()), - ..Default::default() - }), - None, - None, - ); - while let Some(result) = stream.next().await { - let info = result.map_err(|err| internal_status("pull Docker image", err))?; - if let Some(message) = info - .error_detail - .as_ref() - .and_then(|detail| detail.message.as_ref()) - { - return Err(Status::failed_precondition(format!( - "pull Docker image '{image}' failed: {message}" - ))); - } - if let Some(event) = docker_pull_progress_event(image, &info) { - self.publish_platform_event(sandbox_id.to_string(), event); - } - } - self.publish_docker_progress( - sandbox_id, - "Pulled", - format!("Pulled Docker image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - Ok(()) - } -} - -#[tonic::async_trait] -impl ComputeDriver for DockerComputeDriver { - type WatchSandboxesStream = WatchStream; - - async fn get_capabilities( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(self.capabilities())) - } - - async fn validate_sandbox_create( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - let validated = Self::validated_sandbox(&sandbox, &self.config)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; - let _ = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::peek_device_ids, - ) - .await?; - Ok(Response::new(ValidateSandboxCreateResponse {})) - } - - async fn get_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - let sandbox = self - .get_sandbox_snapshot(&request.sandbox_id, &request.sandbox_name) - .await? - .ok_or_else(|| Status::not_found("sandbox not found"))?; - - if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { - return Err(Status::failed_precondition( - "sandbox_id did not match the fetched sandbox", - )); - } - - Ok(Response::new(GetSandboxResponse { - sandbox: Some(sandbox), - })) - } - - async fn list_sandboxes( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(ListSandboxesResponse { - sandboxes: self.current_snapshots().await?, - })) - } - - async fn create_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.create_sandbox_inner(&sandbox).await?; - Ok(Response::new(CreateSandboxResponse {})) - } - - async fn stop_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - self.stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - Ok(Response::new(StopSandboxResponse {})) - } - - async fn delete_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - let event_sandbox_id = request.sandbox_id.clone(); - let deleted = self - .delete_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - if deleted && !event_sandbox_id.is_empty() { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { - sandbox_id: event_sandbox_id, - }, - )), - }); - } - - Ok(Response::new(DeleteSandboxResponse { deleted })) - } - - async fn watch_sandboxes( - &self, - _request: Request, - ) -> Result, Status> { - // Subscribe before taking the initial snapshot so any event emitted - // between the snapshot and this subscriber becoming active is still - // delivered. Downstream consumers treat sandbox events as - // idempotent (keyed by sandbox id), so a duplicate event is benign - // while a missed one leaks state. - let mut rx = self.events.subscribe(); - let initial = self.current_snapshots().await?; - let (tx, out_rx) = mpsc::channel(WATCH_BUFFER); - tokio::spawn(async move { - for sandbox in initial { - if tx - .send(Ok(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox), - }, - )), - })) - .await - .is_err() - { - return; - } - } - - loop { - match rx.recv().await { - Ok(event) => { - if tx.send(Ok(event)).await.is_err() { - return; - } - } - Err(broadcast::error::RecvError::Lagged(_)) => {} - Err(broadcast::error::RecvError::Closed) => return, - } - } - }); - - Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) - } -} - -impl DockerProvisioningFailure { - fn new(reason: &'static str, message: impl Into) -> Self { - Self { - reason, - message: message.into(), - } - } - - fn from_status(reason: &'static str, status: Status) -> Self { - Self::new(reason, status.message()) - } -} - -fn sandbox_image(sandbox: &DriverSandbox) -> Option { - sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .map(|template| template.image.clone()) - .filter(|image| !image.trim().is_empty()) -} - -fn pending_sandbox_snapshot( - sandbox: &DriverSandbox, - namespace: &str, - condition: DriverCondition, - deleting: bool, -) -> DriverSandbox { - DriverSandbox { - id: sandbox.id.clone(), - name: sandbox.name.clone(), - namespace: namespace.to_string(), - spec: None, - status: Some(DriverSandboxStatus { - sandbox_name: sandbox.name.clone(), - instance_id: String::new(), - agent_fd: String::new(), - sandbox_fd: String::new(), - conditions: vec![condition], - deleting, - }), - workspace: sandbox.workspace.clone(), - } -} - -fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { - (!sandbox_id.is_empty() && sandbox.id == sandbox_id) - || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) -} - -fn provisioning_condition() -> DriverCondition { - DriverCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: "Starting".to_string(), - message: "Docker container is starting".to_string(), - last_transition_time: String::new(), - } -} - -fn error_condition(reason: &str, message: &str) -> DriverCondition { - DriverCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: reason.to_string(), - message: message.to_string(), - last_transition_time: String::new(), - } -} - -fn platform_event( - source: &str, - event_type: &str, - reason: &str, - message: String, -) -> DriverPlatformEvent { - DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: source.to_string(), - r#type: event_type.to_string(), - reason: reason.to_string(), - message, - metadata: HashMap::new(), - } -} - -fn docker_pull_progress_event(image: &str, info: &CreateImageInfo) -> Option { - let status = info.status.as_deref().map(str::trim)?; - if status.is_empty() { - return None; - } - - let mut metadata = HashMap::from([ - ("image_ref".to_string(), image.to_string()), - ("docker_status".to_string(), status.to_string()), - ]); - if let Some(layer_id) = info.id.as_deref().filter(|id| !id.is_empty()) { - metadata.insert("layer_id".to_string(), layer_id.to_string()); - } - if let Some(detail) = docker_pull_progress_detail(info) { - metadata.insert("detail".to_string(), detail); - } - attach_docker_progress_metadata(&mut metadata, "PullingLayer", status); - - Some(DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: "docker".to_string(), - r#type: "Normal".to_string(), - reason: "PullingLayer".to_string(), - message: docker_pull_message(info, status), - metadata, - }) -} - -fn docker_pull_message(info: &CreateImageInfo, status: &str) -> String { - info.id.as_deref().filter(|id| !id.is_empty()).map_or_else( - || format!("Docker image pull: {status}"), - |layer_id| format!("Docker image pull {layer_id}: {status}"), - ) -} - -fn docker_pull_progress_detail(info: &CreateImageInfo) -> Option { - let status = info.status.as_deref().unwrap_or("Pulling"); - let layer_id = info.id.as_deref().filter(|id| !id.is_empty()); - let progress = info - .progress_detail - .as_ref() - .and_then(format_progress_detail); - - match (layer_id, progress) { - (Some(layer_id), Some(progress)) => Some(format!("{status} {layer_id} ({progress})")), - (Some(layer_id), None) => Some(format!("{status} {layer_id}")), - (None, Some(progress)) => Some(format!("{status} ({progress})")), - (None, None) => (!status.is_empty()).then(|| status.to_string()), - } -} - -fn format_progress_detail(progress: &ProgressDetail) -> Option { - let current = progress.current.and_then(|value| u64::try_from(value).ok()); - let total = progress - .total - .and_then(|value| u64::try_from(value).ok()) - .filter(|value| *value > 0); - - match (current, total) { - (Some(current), Some(total)) => { - Some(format!("{}/{}", format_bytes(current), format_bytes(total))) - } - (Some(current), _) if current > 0 => Some(format_bytes(current)), - _ => None, - } -} - -fn attach_docker_progress_metadata( - metadata: &mut HashMap, - reason: &str, - message: &str, -) { - match reason { - "Scheduled" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_REQUESTING_SANDBOX, - "Sandbox allocated", - ); - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = metadata.get("image_ref").cloned() { - mark_progress_detail(metadata, image); - } - } - "Pulling" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = metadata.get("image_ref").cloned() { - mark_progress_detail(metadata, image); - } - } - "PullingLayer" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(detail) = metadata - .get("detail") - .cloned() - .filter(|detail| !detail.is_empty()) - { - mark_progress_detail(metadata, detail); - } else if !message.is_empty() { - mark_progress_detail(metadata, message); - } - } - "ImagePresent" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_PULLING_IMAGE, - "Image already present", - ); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - "Pulled" => { - mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, "Image pulled"); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - "Created" => { - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - mark_progress_detail(metadata, "Container created"); - } - "Started" => { - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - mark_progress_detail(metadata, "Waiting for supervisor relay"); - } - _ => {} - } -} - -#[cfg(test)] -fn docker_driver_config( - template: &DriverSandboxTemplate, - enable_bind_mounts: bool, -) -> Result { - let config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?; - Ok(config) -} - -/// Collect user-supplied bind mounts as string-format binds. -/// -/// Bind mounts use the legacy `Binds` field (`-v` syntax) rather than the -/// structured `Mount` API because the Docker Engine Mount object does not -/// support `SELinux` relabelling (`:z` / `:Z`). The string format does. -fn docker_driver_bind_strings(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| match m { - DockerDriverMountConfig::Bind { - source, - target, - read_only, - selinux_label, - } => Some(docker_bind_string( - source, - target, - *read_only, - *selinux_label, - )), - _ => None, - }) - .collect() -} - -fn docker_bind_string( - source: &str, - target: &str, - read_only: bool, - selinux_label: Option, -) -> Result { - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - // Legacy `-v` binds silently create missing source directories as empty, - // root-owned paths. The structured `--mount` API that was used before this - // change rejected missing sources at container-create time. Preserve that - // fail-fast behaviour with an explicit existence check. - if !Path::new(source).exists() { - return Err(Status::failed_precondition(format!( - "bind source path does not exist: {source}" - ))); - } - driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - - let mut opts = Vec::new(); - if read_only { - opts.push("ro"); - } - match selinux_label { - Some(SelinuxLabel::Shared) => opts.push("z"), - Some(SelinuxLabel::Private) => opts.push("Z"), - None => {} - } - - if opts.is_empty() { - Ok(format!("{source}:{normalized_target}")) - } else { - Ok(format!("{source}:{normalized_target}:{}", opts.join(","))) - } -} - -/// Collect user-supplied non-bind mounts as structured `Mount` objects. -fn docker_driver_mounts(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| docker_mount_from_config(m).transpose()) - .collect() -} - -fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result, Status> { - match config { - DockerDriverMountConfig::Bind { .. } => { - // Bind mounts are handled via docker_driver_bind_strings. - Ok(None) - } - DockerDriverMountConfig::Volume { - source, - target, - read_only, - subpath, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::VOLUME), - source: Some(source.clone()), - target: Some(target.clone()), - read_only: Some(*read_only), - volume_options: subpath.as_ref().map(|subpath| MountVolumeOptions { - subpath: Some(subpath.clone()), - ..Default::default() - }), - ..Default::default() - })), - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::TMPFS), - target: Some(target.clone()), - tmpfs_options: Some(MountTmpfsOptions { - size_bytes: validate_optional_positive_integral_i64( - *size_bytes, - "tmpfs size_bytes", - )?, - mode: validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?, - options: (!options.is_empty()) - .then(|| { - options - .iter() - .map(|option| docker_tmpfs_option(option)) - .collect::, _>>() - }) - .transpose()?, - }), - ..Default::default() - })), - DockerDriverMountConfig::Image { .. } => Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )), - } -} - -fn validate_docker_driver_mounts( - mounts: &[DockerDriverMountConfig], - enable_bind_mounts: bool, -) -> Result<(), Status> { - let mut targets = HashSet::new(); - for mount in mounts { - let target = match mount { - DockerDriverMountConfig::Bind { source, target, .. } => { - if !enable_bind_mounts { - return Err(Status::failed_precondition( - "docker bind mounts require enable_bind_mounts = true in [openshell.drivers.docker]", - )); - } - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - target - } - DockerDriverMountConfig::Volume { - source, - target, - subpath, - .. - } => { - driver_mounts::validate_mount_source(source, "volume source") - .map_err(Status::failed_precondition)?; - if let Some(subpath) = subpath { - driver_mounts::validate_mount_subpath(subpath) - .map_err(Status::failed_precondition)?; - } - target - } - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => { - validate_optional_positive_integral_i64(*size_bytes, "tmpfs size_bytes")?; - validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?; - for option in options { - docker_tmpfs_option(option)?; - } - target - } - DockerDriverMountConfig::Image { - source, - target, - read_only, - subpath, - } => { - let _ = (source, target, read_only, subpath); - return Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )); - } - }; - driver_mounts::validate_container_mount_target(target) - .map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - if !targets.insert(normalized_target.clone()) { - return Err(Status::failed_precondition(format!( - "duplicate docker driver_config mount target '{normalized_target}'" - ))); - } - } - Ok(()) -} - -fn validate_optional_positive_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value <= 0 { - return Err(Status::failed_precondition(format!( - "{field} must be positive" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_nonnegative_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value < 0 { - return Err(Status::failed_precondition(format!( - "{field} must be zero or greater" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_integral_i64(value: Option, field: &str) -> Result, Status> { - let Some(value) = value else { - return Ok(None); - }; - if !value.is_finite() || value.fract() != 0.0 { - return Err(Status::failed_precondition(format!( - "{field} must be an integer" - ))); - } - value.to_string().parse::().map(Some).map_err(|_| { - Status::failed_precondition(format!("{field} must be representable as an i64")) - }) -} - -fn docker_tmpfs_option(option: &str) -> Result, Status> { - let option = option.trim(); - if option.is_empty() { - return Err(Status::failed_precondition( - "tmpfs options must not contain empty values", - )); - } - if let Some((key, value)) = option.split_once('=') { - let key = key.trim(); - let value = value.trim(); - if key.is_empty() || value.is_empty() { - return Err(Status::failed_precondition( - "tmpfs key=value options must include both key and value", - )); - } - Ok(vec![key.to_string(), value.to_string()]) - } else { - Ok(vec![option.to_string()]) - } -} - -fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { - volume.driver == "local" - && volume.options.get("o").is_some_and(|options| { - options.split(',').any(|option| { - let option = option.trim(); - option.eq_ignore_ascii_case("bind") || option.eq_ignore_ascii_case("rbind") - }) - }) -} - -fn build_binds( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - Ok(binds) -} - -fn sandbox_token_host_path( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - sandbox_token_host_path_by_id(&sandbox.id, config) -} - -fn sandbox_token_host_path_by_id( - sandbox_id: &str, - config: &DockerDriverRuntimeConfig, -) -> Result { - openshell_core::driver_utils::sandbox_token_path( - "docker-sandbox-tokens", - Some(&config.sandbox_namespace), - sandbox_id, - ) - .map_err(|err| { - Status::internal(format!( - "resolve sandbox token state directory failed: {err}" - )) - }) -} - -async fn write_sandbox_token_file( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let Some(spec) = sandbox.spec.as_ref() else { - return Ok(false); - }; - if spec.sandbox_token.is_empty() { - return Ok(false); - } - let path = sandbox_token_host_path(sandbox, config)?; - if let Some(parent) = path.parent() { - openshell_core::paths::create_dir_restricted(parent).map_err(|err| { - Status::internal(format!( - "create sandbox token directory {} failed: {err}", - parent.display() - )) - })?; - } - tokio::fs::write(&path, format!("{}\n", spec.sandbox_token)) - .await - .map_err(|err| { - Status::internal(format!( - "write sandbox token file {} failed: {err}", - path.display() - )) - })?; - openshell_core::paths::set_file_owner_only(&path).map_err(|err| { - Status::internal(format!( - "restrict sandbox token file {} failed: {err}", - path.display() - )) - })?; - Ok(true) -} - -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); -} - -fn cleanup_sandbox_token_file_for_delete( - sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, - config: &DockerDriverRuntimeConfig, -) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); - } -} - -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { - return; - }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" - ); - } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); - } -} - -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), - ), - ]); - - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - } - - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_COMMAND.to_string(), - SANDBOX_COMMAND.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); - } - - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), - ); - } - - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() -} - -fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { - CdiGpuInventory::new( - info.discovered_devices - .as_deref() - .unwrap_or_default() - .iter() - .filter(|device| device.source.as_deref() == Some("cdi")) - .filter_map(|device| device.id.as_deref()), - ) -} - -fn docker_info_reports_wsl2(info: &SystemInfo) -> bool { - [ - info.kernel_version.as_deref(), - info.operating_system.as_deref(), - ] - .into_iter() - .flatten() - .any(os_or_kernel_reports_wsl2) -} - -fn os_or_kernel_reports_wsl2(value: &str) -> bool { - let value = value.to_ascii_lowercase(); - value.contains("wsl2") || value.contains("microsoft-standard") -} - -fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status { - Status::failed_precondition(err.to_string()) -} - -#[cfg(test)] -fn build_container_create_body( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let template = sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let driver_config = docker_driver_config(template, config.enable_bind_mounts)?; - let gpu_requirements = sandbox - .spec - .as_ref() - .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); - let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - Some(cdi_devices.as_slice()) - } else { - None - }; - build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) -} - -fn build_container_create_body_with_gpu_devices( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - driver_config: &DockerSandboxDriverConfig, - gpu_device_ids: Option<&[String]>, -) -> Result { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let resource_limits = docker_resource_limits(template)?; - let user_mounts = docker_driver_mounts(driver_config)?; - let user_bind_strings = docker_driver_bind_strings(driver_config)?; - let device_requests = gpu_device_ids.map(|device_ids| { - vec![DeviceRequest { - driver: Some("cdi".to_string()), - device_ids: Some(device_ids.to_vec()), - ..Default::default() - }] - }); - let mut labels = template.labels.clone(); - labels.insert( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ); - labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); - labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); - labels.insert( - LABEL_SANDBOX_WORKSPACE.to_string(), - sandbox.workspace.clone(), - ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use - // the same value here. `DriverSandbox.namespace` is unset on the request - // path (the gateway elides it), and using it would produce containers - // that the driver itself cannot find afterwards. - labels.insert( - LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), - ); - - Ok(ContainerCreateBody { - image: Some(template.image.clone()), - user: Some("0".to_string()), - env: Some(build_environment(sandbox, config)), - entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Clear the image CMD so Docker does not append inherited args to the - // supervisor entrypoint. - cmd: Some(Vec::new()), - labels: Some(labels), - host_config: Some(HostConfig { - nano_cpus: resource_limits.nano_cpus, - memory: resource_limits.memory_bytes, - pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, - device_requests, - binds: { - let mut binds = build_binds(sandbox, config)?; - binds.extend(user_bind_strings); - Some(binds) - }, - mounts: Some(user_mounts), - restart_policy: Some(RestartPolicy { - name: Some(RestartPolicyNameEnum::UNLESS_STOPPED), - maximum_retry_count: None, - }), - cap_add: Some(vec![ - "SYS_ADMIN".to_string(), - "NET_ADMIN".to_string(), - "SYS_PTRACE".to_string(), - "SYSLOG".to_string(), - ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), - )])), - }), - ..Default::default() - }) -} - -/// Reject driver requests that arrive with neither a sandbox id nor a -/// sandbox name. Without this guard, downstream label filters degenerate -/// to "match every managed container in the namespace", which would let -/// `delete_sandbox`/`stop_sandbox`/`get_sandbox` pick an arbitrary -/// sandbox out of the set the driver manages. -fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { - if sandbox_id.is_empty() && sandbox_name.is_empty() { - return Err(Status::invalid_argument( - "sandbox_id or sandbox_name is required", - )); - } - Ok(()) -} - -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); - } - - endpoint.to_string() -} - -fn docker_network_name(config: &DockerComputeConfig) -> String { - let name = config.network_name.trim(); - if name.is_empty() { - return DEFAULT_DOCKER_NETWORK_NAME.to_string(); - } - name.to_string() -} - -fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { - let trimmed = value.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - trimmed - .parse() - .map(Some) - .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) -} - -fn docker_gateway_route( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, -) -> DockerGatewayRoute { - docker_gateway_route_for_host( - info, - bridge_gateway_ip, - port, - host_gateway_ip, - host_runtime_requires_host_gateway_alias(), - ) -} - -fn docker_gateway_route_for_host( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, - host_requires_host_gateway_alias: bool, -) -> DockerGatewayRoute { - if let Some(host_alias_ip) = host_gateway_ip { - return DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, - }; - } - - if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { - DockerGatewayRoute::HostGateway - } else { - DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, - } - } -} - -fn host_runtime_requires_host_gateway_alias() -> bool { - cfg!(target_os = "macos") -} - -/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, -/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing -/// constraint: the bridge gateway IP is reachable from inside containers but -/// not from the `OpenShell` server process running on the host, so callbacks -/// must traverse `host-gateway`. -/// -/// Each runtime is detected via the daemon's reported OS string or hostname, -/// supplemented by labels where the runtime publishes them. -fn uses_host_gateway_alias(info: &SystemInfo) -> bool { - let operating_system = info - .operating_system - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if operating_system.contains("docker desktop") { - return true; - } - - let name = info - .name - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if name.starts_with("colima") - || name.starts_with("lima-") - || name.starts_with("rancher-desktop") - || name.starts_with("orbstack") - { - return true; - } - - info.labels.as_ref().is_some_and(|labels| { - labels.iter().any(|label| { - label.starts_with("com.docker.desktop.") - || label.starts_with("dev.rancherdesktop.") - || label.starts_with("dev.orbstack.") - }) - }) -} - -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - -async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { - match docker.inspect_network(network_name, None).await { - Ok(network) => return validate_bridge_network(network_name, &network), - Err(err) if !is_not_found_error(&err) => { - return Err(Error::execution(format!( - "failed to inspect Docker network '{network_name}': {err}" - ))); - } - Err(_) => {} - } - - docker - .create_network(NetworkCreateRequest { - name: network_name.to_string(), - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - attachable: Some(true), - labels: Some(HashMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }) - .await - .map(|_| ()) - .or_else(|err| { - if is_conflict_error(&err) { - Ok(()) - } else { - Err(Error::execution(format!( - "failed to create Docker network '{network_name}': {err}" - ))) - } - })?; - - let network = docker - .inspect_network(network_name, None) - .await - .map_err(|err| { - Error::execution(format!( - "failed to inspect Docker network '{network_name}' after create: {err}" - )) - })?; - validate_bridge_network(network_name, &network) -} - -fn validate_bridge_network( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { - return Err(Error::config(format!( - "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", - network.driver.as_deref().unwrap_or("unknown") - ))); - } - - docker_bridge_gateway_ip(network_name, network) -} - -fn docker_bridge_gateway_ip( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { - return Err(Error::config(format!( - "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" - ))); - }; - - for config in configs { - let Some(gateway) = config.gateway.as_deref() else { - continue; - }; - let ip = gateway.parse::().map_err(|err| { - Error::config(format!( - "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" - )) - })?; - if matches!(ip, IpAddr::V4(_)) { - return Ok(ip); - } - } - - Err(Error::config(format!( - "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" - ))) -} - -fn docker_resource_limits( - template: &DriverSandboxTemplate, -) -> Result { - let Some(resources) = template.resources.as_ref() else { - return Ok(DockerResourceLimits::default()); - }; - - if !resources.cpu_request.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support resources.requests.cpu", - )); - } - if !resources.memory_request.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support resources.requests.memory", - )); - } - - Ok(DockerResourceLimits { - nano_cpus: parse_cpu_limit(&resources.cpu_limit)?, - memory_bytes: parse_memory_limit(&resources.memory_limit)?, - }) -} - -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { - return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", - )); - } - Ok(()) -} - -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", - )); - } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) - } -} - -#[allow(clippy::cast_possible_truncation)] -fn parse_cpu_limit(value: &str) -> Result, Status> { - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - if let Some(millicores) = value.strip_suffix('m') { - let millicores = millicores.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", - )) - })?; - if millicores <= 0 { - return Err(Status::failed_precondition( - "docker cpu_limit must be greater than zero", - )); - } - return Ok(Some(millicores.saturating_mul(1_000_000))); - } - - let cores = value.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", - )) - })?; - if !cores.is_finite() || cores <= 0.0 { - return Err(Status::failed_precondition( - "docker cpu_limit must be greater than zero", - )); - } - - Ok(Some((cores * 1_000_000_000.0).round() as i64)) -} - -#[allow(clippy::cast_possible_truncation)] -fn parse_memory_limit(value: &str) -> Result, Status> { - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - - let number_end = value - .find(|ch: char| !(ch.is_ascii_digit() || ch == '.')) - .unwrap_or(value.len()); - let (number, suffix) = value.split_at(number_end); - let amount = number.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker memory_limit '{value}'; expected a Kubernetes-style quantity", - )) - })?; - if !amount.is_finite() || amount <= 0.0 { - return Err(Status::failed_precondition( - "docker memory_limit must be greater than zero", - )); - } - - let multiplier = match suffix { - "" => 1_f64, - "Ki" => 1024_f64, - "Mi" => 1024_f64.powi(2), - "Gi" => 1024_f64.powi(3), - "Ti" => 1024_f64.powi(4), - "Pi" => 1024_f64.powi(5), - "Ei" => 1024_f64.powi(6), - "K" => 1000_f64, - "M" => 1000_f64.powi(2), - "G" => 1000_f64.powi(3), - "T" => 1000_f64.powi(4), - "P" => 1000_f64.powi(5), - "E" => 1000_f64.powi(6), - _ => { - return Err(Status::failed_precondition(format!( - "invalid docker memory_limit suffix '{suffix}'", - ))); - } - }; - - Ok(Some((amount * multiplier).round() as i64)) -} - -fn sandbox_from_container_summary( - summary: &ContainerSummary, - readiness: &dyn SupervisorReadiness, -) -> Option { - let labels = summary.labels.as_ref()?; - let id = labels.get(LABEL_SANDBOX_ID)?.clone(); - let name = labels.get(LABEL_SANDBOX_NAME)?.clone(); - let namespace = labels - .get(LABEL_SANDBOX_NAMESPACE) - .cloned() - .unwrap_or_default(); - let workspace = labels - .get(LABEL_SANDBOX_WORKSPACE) - .cloned() - .unwrap_or_default(); - - let supervisor_connected = readiness.is_supervisor_connected(&id); - Some(DriverSandbox { - id, - name: name.clone(), - namespace, - spec: None, - status: Some(driver_status_from_summary( - summary, - &name, - supervisor_connected, - )), - workspace, - }) -} - -fn driver_status_from_summary( - summary: &ContainerSummary, - sandbox_name: &str, - supervisor_connected: bool, -) -> DriverSandboxStatus { - let state = summary.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - let (ready, reason, message, deleting) = container_ready_condition(state, supervisor_connected); - - DriverSandboxStatus { - sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), - instance_id: summary.id.clone().unwrap_or_default(), - agent_fd: String::new(), - sandbox_fd: String::new(), - conditions: vec![DriverCondition { - r#type: "Ready".to_string(), - status: ready.to_string(), - reason: reason.to_string(), - message: message.to_string(), - last_transition_time: String::new(), - }], - deleting, - } -} - -fn container_ready_condition( - state: ContainerSummaryStateEnum, - supervisor_connected: bool, -) -> (&'static str, &'static str, &'static str, bool) { - match state { - ContainerSummaryStateEnum::RUNNING => { - if supervisor_connected { - ( - "True", - "SupervisorConnected", - "Supervisor relay is live", - false, - ) - } else { - ( - "False", - "DependenciesNotReady", - "Container is running; waiting for supervisor relay", - false, - ) - } - } - ContainerSummaryStateEnum::CREATED => ("False", "Starting", "Container created", false), - ContainerSummaryStateEnum::RESTARTING => ( - "False", - "ContainerRestarting", - "Container is restarting after a failure", - false, - ), - ContainerSummaryStateEnum::EMPTY => { - ("False", "Starting", "Container state is unknown", false) - } - ContainerSummaryStateEnum::REMOVING => { - ("False", "Deleting", "Container is being removed", true) - } - ContainerSummaryStateEnum::PAUSED => { - ("False", "ContainerPaused", "Container is paused", false) - } - ContainerSummaryStateEnum::EXITED => { - ("False", "ContainerExited", "Container exited", false) - } - ContainerSummaryStateEnum::DEAD => ("False", "ContainerDead", "Container is dead", false), - } -} - -fn summary_container_name(summary: &ContainerSummary) -> Option { - summary - .names - .as_ref() - .and_then(|names| names.first()) - .map(|name| name.trim_start_matches('/').to_string()) - .filter(|name| !name.is_empty()) -} - -fn summary_container_target(summary: &ContainerSummary) -> Option { - // Prefer the container ID: it's stable while the container exists and is - // accepted by Docker APIs just like a name. Fall back to the parsed name - // for transient summaries that do not include an ID. - summary - .id - .as_deref() - .filter(|id| !id.is_empty()) - .map(str::to_string) - .or_else(|| summary_container_name(summary)) -} - -fn container_state_needs_shutdown_stop(state: ContainerSummaryStateEnum) -> bool { - matches!( - state, - ContainerSummaryStateEnum::RUNNING - | ContainerSummaryStateEnum::RESTARTING - | ContainerSummaryStateEnum::PAUSED - ) -} - -/// States from which a managed container can be brought back to running by -/// `start_container`. Skip `Restarting` (already coming up), `Removing`, -/// `Dead` (terminal), `Paused` (needs `unpause`, not `start`), and -/// `Running` (nothing to do). -fn container_state_needs_resume(state: ContainerSummaryStateEnum) -> bool { - matches!( - state, - ContainerSummaryStateEnum::EXITED | ContainerSummaryStateEnum::CREATED - ) -} - -fn docker_stop_timeout_secs(timeout_secs: u32) -> i32 { - i32::try_from(timeout_secs).unwrap_or(i32::MAX) -} - -fn emit_snapshot_diff( - events: &broadcast::Sender, - previous: &HashMap, - current: &HashMap, -) { - for (sandbox_id, sandbox) in current { - if previous.get(sandbox_id) == Some(sandbox) { - continue; - } - let _ = events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox.clone()), - }, - )), - }); - } - - for sandbox_id in previous.keys() { - if current.contains_key(sandbox_id) { - continue; - } - let _ = events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { - sandbox_id: sandbox_id.clone(), - }, - )), - }); - } -} - -fn label_filters(values: impl IntoIterator) -> HashMap> { - HashMap::from([("label".to_string(), values.into_iter().collect())]) -} - -fn managed_container_label_filters( - sandbox_namespace: &str, - extra_values: impl IntoIterator, -) -> HashMap> { - let mut values = vec![ - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), - ]; - values.extend(extra_values); - label_filters(values) -} - -/// Maximum Docker container name length. Docker's own limit is 253 bytes, but -/// we cap at a conservative 200 to leave headroom for tooling that truncates -/// names further. -const MAX_CONTAINER_NAME_LEN: usize = 200; -const CONTAINER_NAME_PREFIX: &str = "openshell-"; - -fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String { - let id_suffix = sanitize_docker_name(&sandbox.id); - let workspace = sanitize_docker_name(&sandbox.workspace); - let name = sanitize_docker_name(&sandbox.name); - - // Format: openshell-{workspace}--{name}-{id} - // The workspace and id are never truncated — they ensure uniqueness. - // Only the sandbox name portion is truncated when the total exceeds - // MAX_CONTAINER_NAME_LEN. - - if name.is_empty() { - let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); - if base.len() > MAX_CONTAINER_NAME_LEN { - base.truncate(MAX_CONTAINER_NAME_LEN); - } - return trim_container_name_tail(base); - } - - // Reserve space for fixed parts: prefix + workspace + "--" + "-" + id - let reserved = CONTAINER_NAME_PREFIX.len() + workspace.len() + 2 + 1 + id_suffix.len(); - if reserved >= MAX_CONTAINER_NAME_LEN { - let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); - base.truncate(MAX_CONTAINER_NAME_LEN); - return trim_container_name_tail(base); - } - - let name_budget = MAX_CONTAINER_NAME_LEN - reserved; - let truncated_name = if name.len() > name_budget { - trim_container_name_tail(name[..name_budget].to_string()) - } else { - name - }; - format!("{CONTAINER_NAME_PREFIX}{workspace}--{truncated_name}-{id_suffix}") -} - -/// Docker container names may not end with `-`, `.`, or `_`. Truncation can -/// leave one of those trailing, so strip them before returning. -fn trim_container_name_tail(mut value: String) -> String { - while value - .chars() - .last() - .is_some_and(|ch| matches!(ch, '-' | '.' | '_')) - { - value.pop(); - } - value -} - -fn sanitize_docker_name(value: &str) -> String { - value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '-') { - ch.to_ascii_lowercase() - } else { - '-' - } - }) - .collect::() - .trim_matches('-') - .to_string() -} - -fn normalize_docker_arch(arch: &str) -> String { - match arch { - "x86_64" => "amd64".to_string(), - "aarch64" => "arm64".to_string(), - other => other.to_ascii_lowercase(), - } -} - -#[derive(Debug, Eq, PartialEq)] -enum SupervisorBinSource { - Binary(PathBuf), - Image(String), -} - -fn resolve_supervisor_bin_source( - docker_config: &DockerComputeConfig, - current_exe: Option<&Path>, - target_candidates: &[PathBuf], -) -> CoreResult { - // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. - if let Some(path) = docker_config.supervisor_bin.clone() { - let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path)?; - return Ok(SupervisorBinSource::Binary(path)); - } - - // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. - // A configured image should be the source of truth even when a local - // developer build is present under target/. - if let Some(image) = docker_config.supervisor_image.clone() { - return Ok(SupervisorBinSource::Image(image)); - } - - // Tier 3: sibling `openshell-sandbox` next to the running gateway - // (release artifact layout). Linux-only because the sibling must be a - // Linux ELF to bind-mount into a Linux container. - if cfg!(target_os = "linux") - && let Some(current_exe) = current_exe - && let Some(parent) = current_exe.parent() - { - let sibling = parent.join("openshell-sandbox"); - if sibling.is_file() { - let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 4: local cargo target build (developer workflow). Preferred - // over the default registry image when available because it matches - // whatever the developer just built. - for candidate in target_candidates { - if candidate.is_file() { - let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 5: pull the release-matched default supervisor image and extract - // the binary to a host-side cache keyed by image content digest. - Ok(SupervisorBinSource::Image( - openshell_core::config::default_supervisor_image(), - )) -} - -pub(crate) async fn resolve_supervisor_bin( - docker: &Docker, - docker_config: &DockerComputeConfig, - daemon_arch: &str, -) -> CoreResult { - let current_exe = - if cfg!(target_os = "linux") - && docker_config.supervisor_bin.is_none() - && docker_config.supervisor_image.is_none() - { - Some(std::env::current_exe().map_err(|err| { - Error::config(format!("failed to resolve current executable: {err}")) - })?) - } else { - None - }; - let target_candidates = linux_supervisor_candidates(daemon_arch); - - match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? - { - SupervisorBinSource::Binary(path) => Ok(path), - SupervisorBinSource::Image(image) => { - extract_supervisor_bin_from_image(docker, &image).await - } - } -} - -fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { - match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], - _ => Vec::new(), - } -} - -/// Pull the supervisor image (if not already local), extract -/// `/openshell-sandbox` to a host cache keyed by the image's content -/// digest, and return the cache path. -/// -/// The extraction is atomic: the binary is written to a sibling temp file -/// inside the digest-keyed directory and renamed into place, so concurrent -/// gateway starts don't observe a partial file. -async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { - let refresh_attempted = if supervisor_image_should_refresh(image) { - info!(image = image, "Refreshing mutable docker supervisor image"); - match pull_supervisor_image(docker, image).await { - Ok(()) => true, - Err(err) => { - warn!( - image = image, - error = %err, - "failed to refresh mutable docker supervisor image; falling back to local image if present", - ); - true - } - } - } else { - false - }; - - // Inspect first to see if the image is already present; only pull on miss. - let inspect = match docker.inspect_image(image).await { - Ok(inspect) => inspect, - Err(err) if is_not_found_error(&err) && !refresh_attempted => { - info!(image = image, "Pulling docker supervisor image"); - pull_supervisor_image(docker, image).await?; - docker.inspect_image(image).await.map_err(|err| { - Error::config(format!( - "failed to inspect docker supervisor image '{image}' after pull: {err}", - )) - })? - } - Err(err) if is_not_found_error(&err) => { - return Err(Error::config(format!( - "docker supervisor image '{image}' is not present locally after refresh attempt", - ))); - } - Err(err) => { - return Err(Error::config(format!( - "failed to inspect docker supervisor image '{image}': {err}", - ))); - } - }; - - let digest = inspect.id.clone().ok_or_else(|| { - Error::config(format!( - "docker supervisor image '{image}' inspect response has no Id", - )) - })?; - - let cache_path = supervisor_cache_path(&digest)?; - if cache_path.is_file() { - validate_linux_elf_binary(&cache_path)?; - return Ok(cache_path); - } - - let cache_dir = cache_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - cache_path.display(), - )) - })?; - std::fs::create_dir_all(cache_dir).map_err(|err| { - Error::config(format!( - "failed to create docker supervisor cache dir '{}': {err}", - cache_dir.display(), - )) - })?; - - info!( - image = image, - digest = digest, - cache_path = %cache_path.display(), - "Extracting supervisor binary from image to host cache", - ); - - let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes)?; - validate_linux_elf_binary(&cache_path)?; - Ok(cache_path) -} - -async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { - let mut stream = docker.create_image( - Some(CreateImageOptions { - from_image: Some(image.to_string()), - ..Default::default() - }), - None, - None, - ); - while let Some(result) = stream.next().await { - result.map_err(|err| { - Error::config(format!( - "failed to pull docker supervisor image '{image}': {err}", - )) - })?; - } - Ok(()) -} - -/// Create a short-lived container from `image`, stream out the supervisor -/// binary as a tar archive, and return the untarred file bytes. The -/// container is always removed, even on error paths. -async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { - let container_name = temp_extract_container_name(); - docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - ContainerCreateBody { - image: Some(image.to_string()), - entrypoint: Some(vec![SUPERVISOR_IMAGE_BINARY_PATH.to_string()]), - cmd: Some(Vec::new()), - ..Default::default() - }, - ) - .await - .map_err(|err| { - Error::config(format!( - "failed to create extractor container from '{image}': {err}", - )) - })?; - - // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; - if let Err(remove_err) = docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - warn!( - container = container_name, - error = %remove_err, - "Failed to remove supervisor extractor container", - ); - } - result -} - -async fn download_binary_from_container( - docker: &Docker, - container_name: &str, -) -> CoreResult> { - let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) - .build(); - let mut stream = docker.download_from_container(container_name, Some(options)); - - let mut tar_bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk: Bytes = chunk.map_err(|err| { - Error::config(format!( - "failed to read supervisor binary stream from '{container_name}': {err}", - )) - })?; - tar_bytes.extend_from_slice(&chunk); - } - - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) -} - -/// Extract the payload of the first regular-file entry in a tar archive. -/// Docker's `/containers//archive` endpoint returns a single-file tar -/// when `path` points to a file, so we only need the first entry. -fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { - let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let mut entries = archive - .entries() - .map_err(|err| format!("open tar archive: {err}"))?; - let mut entry = entries - .next() - .ok_or_else(|| "tar archive was empty".to_string())? - .map_err(|err| format!("read tar entry: {err}"))?; - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|err| format!("read tar entry payload: {err}"))?; - Ok(bytes) -} - -fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { - let dir = final_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - final_path.display(), - )) - })?; - let mut temp = tempfile::Builder::new() - .prefix(".openshell-sandbox-") - .tempfile_in(dir) - .map_err(|err| { - Error::config(format!( - "failed to create temp file for supervisor binary in '{}': {err}", - dir.display(), - )) - })?; - std::io::Write::write_all(&mut temp, bytes).map_err(|err| { - Error::config(format!( - "failed to write supervisor binary to temp file: {err}", - )) - })?; - temp.as_file().sync_all().map_err(|err| { - Error::config(format!("failed to sync supervisor binary temp file: {err}")) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( - |err| { - Error::config(format!( - "failed to chmod supervisor binary temp file: {err}", - )) - }, - )?; - } - - temp.persist(final_path).map_err(|err| { - Error::config(format!( - "failed to rename supervisor binary into '{}': {}", - final_path.display(), - err.error, - )) - })?; - Ok(()) -} - -/// Cache path for an extracted supervisor binary, keyed by the image's -/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed -/// directory keeps stale extractions from earlier releases isolated so they -/// can be GC'd without affecting the active binary. -fn supervisor_cache_path(digest: &str) -> CoreResult { - let base = openshell_core::paths::xdg_data_dir() - .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; - Ok(supervisor_cache_path_with_base(&base, digest)) -} - -fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { - let sanitized = digest.replace(':', "-"); - base.join("openshell") - .join("docker-supervisor") - .join(sanitized) - .join("openshell-sandbox") -} - -fn temp_extract_container_name() -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static SEQ: AtomicU64 = AtomicU64::new(0); - let pid = std::process::id(); - let seq = SEQ.fetch_add(1, Ordering::Relaxed); - format!("openshell-supervisor-extract-{pid}-{seq}") -} - -fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { - if !path.is_file() { - return Err(Error::config(format!( - "{description} '{}' does not exist or is not a file", - path.display() - ))); - } - std::fs::canonicalize(path).map_err(|err| { - Error::config(format!( - "failed to resolve {description} '{}': {err}", - path.display() - )) - }) -} - -pub(crate) fn validate_linux_elf_binary(path: &Path) -> CoreResult<()> { - let mut file = std::fs::File::open(path).map_err(|err| { - Error::config(format!( - "failed to open docker supervisor binary '{}': {err}", - path.display() - )) - })?; - let mut magic = [0_u8; 4]; - file.read_exact(&mut magic).map_err(|err| { - Error::config(format!( - "failed to read docker supervisor binary '{}': {err}", - path.display() - )) - })?; - if magic != [0x7f, b'E', b'L', b'F'] { - return Err(Error::config(format!( - "docker supervisor binary '{}' must be a Linux ELF executable", - path.display() - ))); - } - Ok(()) -} - -fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { - docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() -} - -pub(crate) fn docker_guest_tls_paths( - docker_config: &DockerComputeConfig, -) -> CoreResult> { - let tls_flags_provided = docker_config.guest_tls_ca.is_some() - || docker_config.guest_tls_cert.is_some() - || docker_config.guest_tls_key.is_some(); - - if !docker_config.grpc_endpoint.starts_with("https://") { - if tls_flags_provided { - return Err(Error::config(format!( - "guest_tls_ca/guest_tls_cert/guest_tls_key were provided but grpc_endpoint is '{}'; TLS materials require an https:// endpoint", - docker_config.grpc_endpoint, - ))); - } - return Ok(None); - } - - let provided = [ - docker_config.guest_tls_ca.as_ref(), - docker_config.guest_tls_cert.as_ref(), - docker_config.guest_tls_key.as_ref(), - ]; - if provided.iter().all(Option::is_none) { - return Err(Error::config( - "docker compute driver requires guest_tls_ca, guest_tls_cert, and guest_tls_key when grpc_endpoint uses https://", - )); - } - - let Some(ca) = docker_config.guest_tls_ca.clone() else { - return Err(Error::config( - "guest_tls_ca is required when Docker sandbox TLS materials are configured", - )); - }; - let Some(cert) = docker_config.guest_tls_cert.clone() else { - return Err(Error::config( - "guest_tls_cert is required when Docker sandbox TLS materials are configured", - )); - }; - let Some(key) = docker_config.guest_tls_key.clone() else { - return Err(Error::config( - "guest_tls_key is required when Docker sandbox TLS materials are configured", - )); - }; - - Ok(Some(DockerGuestTlsPaths { - ca: canonicalize_existing_file(&ca, "docker TLS CA certificate")?, - cert: canonicalize_existing_file(&cert, "docker TLS client certificate")?, - key: canonicalize_existing_file(&key, "docker TLS client private key")?, - })) -} - -fn is_not_found_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 404, - .. - } - ) -} - -fn is_conflict_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) -} - -fn is_not_modified_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 304, - .. - } - ) -} - -fn create_status_from_docker_error(operation: &str, err: BollardError) -> Status { - if matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) { - Status::already_exists("sandbox already exists") - } else { - internal_status(operation, err) - } -} - -fn internal_status(operation: &str, err: BollardError) -> Status { - Status::internal(format!("{operation} failed: {err}")) -} - -#[cfg(test)] -mod tests; +#[cfg(not(target_os = "windows"))] +include!("lib_unix.rs"); diff --git a/crates/openshell-driver-docker/src/lib_unix.rs b/crates/openshell-driver-docker/src/lib_unix.rs new file mode 100644 index 0000000000..2f89c2229e --- /dev/null +++ b/crates/openshell-driver-docker/src/lib_unix.rs @@ -0,0 +1,3522 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Docker compute driver. + +#![allow(clippy::result_large_err)] + +use bollard::Docker; +use bollard::errors::Error as BollardError; +use bollard::models::{ + ContainerCreateBody, ContainerSummary, ContainerSummaryStateEnum, CreateImageInfo, + DeviceRequest, EndpointSettings, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, + MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, ProgressDetail, RestartPolicy, + RestartPolicyNameEnum, SystemInfo, +}; +use bollard::query_parameters::{ + CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, + ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, +}; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use openshell_core::config::{ + DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, +}; +use openshell_core::driver_mounts; +use openshell_core::driver_utils::{ + LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, + LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, + supervisor_image_should_refresh, +}; +use openshell_core::gpu::{ + CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, + effective_driver_gpu_count, validate_specific_gpu_device_request, +}; +use openshell_core::progress::{ + PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX, + format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail, +}; +use openshell_core::proto::compute::v1::{ + CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, + DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, + DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, + GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, + compute_driver_server::ComputeDriver, watch_sandboxes_event, +}; +use openshell_core::proto_struct::{ + deserialize_optional_non_empty_string_list, struct_to_json_value, +}; +use openshell_core::{Config, Error, Result as CoreResult}; +use std::collections::{HashMap, HashSet}; +use std::io::Read; +use std::net::{IpAddr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio::task::JoinHandle; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use tracing::{info, warn}; +use url::Url; + +const WATCH_BUFFER: usize = 128; +const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); +const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); + +const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; +const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; +const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; +const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +const SANDBOX_COMMAND: &str = "sleep infinity"; +const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; +const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; +const DOCKER_NETWORK_DRIVER: &str = "bridge"; + +/// Queried by the Docker driver to decide when a sandbox's supervisor +/// relay is live. Implementations return `true` once a sandbox has an +/// active `ConnectSupervisor` session registered. +/// +/// The driver cannot observe the supervisor's SSH socket directly (it +/// lives inside the container), so it leans on this signal to flip the +/// Ready condition from `DependenciesNotReady` to `True`. +pub trait SupervisorReadiness: Send + Sync + 'static { + fn is_supervisor_connected(&self, sandbox_id: &str) -> bool; +} + +/// Gateway-local configuration for the Docker compute driver. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DockerComputeConfig { + /// Docker API Unix socket. When unset, use the socket selected by gateway + /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly + /// configured Docker driver. + pub socket_path: Option, + + /// Default OCI image for sandboxes. + pub default_image: String, + + /// Image pull policy for sandbox images. + pub image_pull_policy: String, + + /// Namespace label applied to Docker sandboxes. + pub sandbox_namespace: String, + + /// Gateway gRPC endpoint the sandbox connects back to. + pub grpc_endpoint: String, + + /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. + pub supervisor_bin: Option, + + /// Optional image used to extract the Linux `openshell-sandbox` binary. + /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for + /// the full resolution order. + pub supervisor_image: Option, + + /// Host-side CA certificate for Docker sandbox mTLS. + pub guest_tls_ca: Option, + + /// Host-side client certificate for Docker sandbox mTLS. + pub guest_tls_cert: Option, + + /// Host-side private key for Docker sandbox mTLS. + pub guest_tls_key: Option, + + /// Docker bridge network that sandbox containers join. + pub network_name: String, + + /// Host gateway IP used for sandbox host aliases. + pub host_gateway_ip: String, + + /// Unix socket path the in-container supervisor bridges relay traffic to. + pub ssh_socket_path: String, + + /// Container cgroup PID limit for Docker-managed sandboxes. + /// + /// Set to `0` to leave Docker's runtime/default PID limit unchanged. + pub sandbox_pids_limit: i64, + + /// Allow sandbox requests to attach host bind mounts through + /// `template.driver_config`. + #[serde(default)] + pub enable_bind_mounts: bool, +} + +impl Default for DockerComputeConfig { + fn default() -> Self { + Self { + socket_path: None, + default_image: openshell_core::image::default_sandbox_image(), + image_pull_policy: String::new(), + sandbox_namespace: "default".to_string(), + grpc_endpoint: String::new(), + supervisor_bin: None, + supervisor_image: None, + guest_tls_ca: None, + guest_tls_cert: None, + guest_tls_key: None, + network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), + host_gateway_ip: String::new(), + ssh_socket_path: "/run/openshell/ssh.sock".to_string(), + sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + enable_bind_mounts: false, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DockerGuestTlsPaths { + pub(crate) ca: PathBuf, + pub(crate) cert: PathBuf, + pub(crate) key: PathBuf, +} + +#[derive(Debug, Clone)] +struct DockerDriverRuntimeConfig { + default_image: String, + image_pull_policy: String, + sandbox_namespace: String, + grpc_endpoint: String, + network_name: String, + gateway_route: DockerGatewayRoute, + ssh_socket_path: String, + stop_timeout_secs: u32, + log_level: String, + supervisor_bin: PathBuf, + guest_tls: Option, + daemon_version: String, + supports_gpu: bool, + allow_all_default_gpu: bool, + sandbox_pids_limit: i64, + enable_bind_mounts: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum DockerGatewayRoute { + Bridge { + bind_address: SocketAddr, + host_alias_ip: IpAddr, + }, + HostGateway, +} + +#[derive(Clone)] +pub struct DockerComputeDriver { + docker: Arc, + config: DockerDriverRuntimeConfig, + events: broadcast::Sender, + pending: Arc>>, + supervisor_readiness: Arc, + gpu_selector: Arc, +} + +struct PendingSandboxRecord { + sandbox: DriverSandbox, + task: Option>, +} + +#[derive(Debug, Clone)] +struct DockerProvisioningFailure { + reason: &'static str, + message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct DockerResourceLimits { + nano_cpus: Option, + memory_bytes: Option, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +struct DockerSandboxDriverConfig { + #[serde( + default, + deserialize_with = "deserialize_optional_non_empty_string_list" + )] + cdi_devices: Option>, + mounts: Vec, +} + +struct ValidatedDockerSandbox<'a> { + template: &'a DriverSandboxTemplate, + driver_config: DockerSandboxDriverConfig, + gpu_requirements: Option<&'a GpuResourceRequirements>, +} + +impl DockerSandboxDriverConfig { + fn from_template(template: &DriverSandboxTemplate) -> Result { + let Some(config) = template.driver_config.as_ref() else { + return Ok(Self::default()); + }; + + serde_json::from_value(struct_to_json_value(config)) + .map_err(|err| format!("invalid docker driver_config: {err}")) + } +} + +use openshell_core::driver_mounts::SelinuxLabel; + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +enum DockerDriverMountConfig { + Bind { + source: String, + target: String, + #[serde(default = "default_true")] + read_only: bool, + #[serde(default)] + selinux_label: Option, + }, + Volume { + source: String, + target: String, + #[serde(default = "default_true")] + read_only: bool, + #[serde(default)] + subpath: Option, + }, + Tmpfs { + target: String, + #[serde(default)] + options: Vec, + #[serde(default)] + size_bytes: Option, + #[serde(default)] + mode: Option, + }, + Image { + source: String, + target: String, + #[serde(default = "default_true")] + read_only: bool, + #[serde(default)] + subpath: Option, + }, +} + +fn default_true() -> bool { + true +} + +type WatchStream = + Pin> + Send + 'static>>; + +impl DockerComputeDriver { + pub async fn new( + config: &Config, + docker_config: &DockerComputeConfig, + supervisor_readiness: Arc, + ) -> CoreResult { + let socket_path = docker_config + .socket_path + .clone() + .or_else(openshell_core::config::detect_docker_socket) + .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); + let socket_path_str = socket_path.to_str().ok_or_else(|| { + Error::config(format!( + "Docker socket path is not valid UTF-8: {}", + socket_path.display() + )) + })?; + let docker = + Docker::connect_with_socket(socket_path_str, 120, bollard::API_DEFAULT_VERSION) + .map_err(|err| { + Error::execution(format!("failed to create Docker client: {err}")) + })?; + let version = docker.version().await.map_err(|err| { + Error::execution(format!("failed to query Docker daemon version: {err}")) + })?; + let info = docker.info().await.map_err(|err| { + Error::execution(format!("failed to query Docker daemon info: {err}")) + })?; + let supports_gpu = info + .cdi_spec_dirs + .as_ref() + .is_some_and(|dirs| !dirs.is_empty()); + let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); + let allow_all_default_gpu = docker_info_reports_wsl2(&info); + validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + let gateway_port = config.bind_address.port(); + if gateway_port == 0 { + return Err(Error::config( + "docker compute driver requires a fixed non-zero gateway bind port", + )); + } + let network_name = docker_network_name(docker_config); + let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; + let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; + let gateway_route = + docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); + let mut docker_config = docker_config.clone(); + if docker_config.grpc_endpoint.trim().is_empty() { + let scheme = if docker_guest_tls_configured(&docker_config) { + "https" + } else { + "http" + }; + docker_config.grpc_endpoint = + format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); + } + let grpc_endpoint = docker_container_openshell_endpoint( + &docker_config.grpc_endpoint, + HOST_OPENSHELL_INTERNAL, + gateway_port, + ); + let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); + let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; + let guest_tls = docker_guest_tls_paths(&docker_config)?; + + let driver = Self { + docker: Arc::new(docker), + config: DockerDriverRuntimeConfig { + default_image: docker_config.default_image.clone(), + image_pull_policy: docker_config.image_pull_policy.clone(), + sandbox_namespace: docker_config.sandbox_namespace.clone(), + grpc_endpoint, + network_name, + gateway_route, + ssh_socket_path: docker_config.ssh_socket_path.clone(), + stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, + log_level: config.log_level.clone(), + supervisor_bin, + guest_tls, + daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), + supports_gpu, + allow_all_default_gpu, + sandbox_pids_limit: docker_config.sandbox_pids_limit, + enable_bind_mounts: docker_config.enable_bind_mounts, + }, + events: broadcast::channel(WATCH_BUFFER).0, + pending: Arc::new(Mutex::new(HashMap::new())), + supervisor_readiness, + gpu_selector: Arc::new(CdiGpuDefaultSelector::new( + cdi_gpu_inventory, + allow_all_default_gpu, + )), + }; + + let poll_driver = driver.clone(); + tokio::spawn(async move { + poll_driver.poll_loop().await; + }); + + Ok(driver) + } + + #[must_use] + pub fn gateway_bind_addresses(&self) -> Vec { + match self.config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], + DockerGatewayRoute::HostGateway => Vec::new(), + } + } + + fn capabilities(&self) -> GetCapabilitiesResponse { + openshell_core::driver_utils::build_capabilities_response( + "docker", + &self.config.daemon_version, + &self.config.default_image, + ) + } + + #[cfg(test)] + fn validate_sandbox( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + ) -> Result<(), Status> { + let _ = Self::validated_sandbox(sandbox, config)?; + Ok(()) + } + + fn validated_sandbox<'a>( + sandbox: &'a DriverSandbox, + config: &DockerDriverRuntimeConfig, + ) -> Result, Status> { + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; + let template = spec + .template + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + + Self::validate_sandbox_template_base(template)?; + let _ = docker_resource_limits(template)?; + let driver_config = + DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; + validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; + let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); + Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?; + Ok(ValidatedDockerSandbox { + template, + driver_config, + gpu_requirements, + }) + } + + fn validate_sandbox_template_base(template: &DriverSandboxTemplate) -> Result<(), Status> { + if template.image.trim().is_empty() { + return Err(Status::failed_precondition( + "docker sandboxes require a template image", + )); + } + if !template.agent_socket_path.trim().is_empty() { + return Err(Status::failed_precondition( + "docker compute driver does not support template.agent_socket_path", + )); + } + if template + .platform_config + .as_ref() + .is_some_and(|config| !config.fields.is_empty()) + { + return Err(Status::failed_precondition( + "docker compute driver does not support template.platform_config", + )); + } + + Ok(()) + } + + fn validate_sandbox_auth(sandbox: &DriverSandbox) -> Result<(), Status> { + let token_present = sandbox + .spec + .as_ref() + .is_some_and(|spec| !spec.sandbox_token.trim().is_empty()); + if token_present { + return Ok(()); + } + + Err(Status::failed_precondition( + "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]", + )) + } + + fn validate_gpu_request( + gpu_requirements: Option<&GpuResourceRequirements>, + supports_gpu: bool, + driver_config: &DockerSandboxDriverConfig, + ) -> Result<(), Status> { + let requested_count = + effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)?; + if requested_count.is_some() && !supports_gpu { + return Err(Status::failed_precondition( + "docker GPU sandboxes require Docker CDI support. Enable CDI on the Docker daemon, then restart the OpenShell gateway/server so GPU capability is detected.", + )); + } + + if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { + validate_specific_gpu_device_request( + gpu_requirements, + cdi_devices, + "driver_config.cdi_devices", + ) + .map_err(Status::invalid_argument)?; + } + + Ok(()) + } + + async fn validate_user_volume_mounts_available( + &self, + driver_config: &DockerSandboxDriverConfig, + ) -> Result<(), Status> { + for mount in &driver_config.mounts { + if let DockerDriverMountConfig::Volume { source, .. } = mount { + match self.docker.inspect_volume(source).await { + Ok(volume) => { + if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) + { + return Err(Status::failed_precondition(format!( + "docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]" + ))); + } + } + Err(err) if is_not_found_error(&err) => { + return Err(Status::failed_precondition(format!( + "docker volume '{source}' does not exist" + ))); + } + Err(err) => { + return Err(internal_status("inspect docker volume", err)); + } + } + } + } + Ok(()) + } + + async fn refresh_gpu_inventory(&self) -> Result<(), Status> { + let info = self + .docker + .info() + .await + .map_err(|err| internal_status("query Docker daemon info", err))?; + self.gpu_selector.refresh( + docker_cdi_gpu_inventory(&info), + self.config.allow_all_default_gpu, + ); + Ok(()) + } + + async fn resolve_gpu_cdi_devices( + &self, + gpu_requirements: Option<&GpuResourceRequirements>, + driver_config: &DockerSandboxDriverConfig, + select_default_devices: fn( + &CdiGpuDefaultSelector, + u32, + ) -> Result, CdiGpuSelectionError>, + ) -> Result>, Status> { + if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { + validate_specific_gpu_device_request( + gpu_requirements, + cdi_devices, + "driver_config.cdi_devices", + ) + .map_err(Status::invalid_argument)?; + return Ok(Some(cdi_devices.to_vec())); + } + + let Some(count) = + effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)? + else { + return Ok(None); + }; + + self.refresh_gpu_inventory().await?; + select_default_devices(&self.gpu_selector, count) + .map(Some) + .map_err(docker_gpu_selection_status) + } + + async fn get_sandbox_snapshot( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result, Status> { + let container = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await?; + if let Some(sandbox) = container.and_then(|summary| { + sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) + }) { + return Ok(Some(sandbox)); + } + + Ok(self.pending_snapshot(sandbox_id, sandbox_name).await) + } + + async fn current_snapshots(&self) -> Result, Status> { + let containers = self.list_managed_container_summaries().await?; + let container_sandboxes = containers + .iter() + .filter_map(|summary| { + sandbox_from_container_summary(summary, self.supervisor_readiness.as_ref()) + }) + .collect::>(); + let mut by_id = self.pending_snapshot_map().await; + for sandbox in container_sandboxes { + by_id.insert(sandbox.id.clone(), sandbox); + } + let mut sandboxes = by_id.into_values().collect::>(); + sandboxes.sort_by(|left, right| left.id.cmp(&right.id)); + Ok(sandboxes) + } + + async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { + let validated = Self::validated_sandbox(sandbox, &self.config)?; + Self::validate_sandbox_auth(sandbox)?; + self.validate_user_volume_mounts_available(&validated.driver_config) + .await?; + let gpu_devices = self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await?; + let _ = build_container_create_body_with_gpu_devices( + sandbox, + &self.config, + &validated.driver_config, + gpu_devices.as_deref(), + )?; + + if self + .find_managed_container_summary(&sandbox.id, &sandbox.name) + .await? + .is_some() + { + return Err(Status::already_exists("sandbox already exists")); + } + + self.reserve_pending_sandbox(sandbox).await?; + let image = sandbox_image(sandbox).unwrap_or_default(); + self.publish_docker_progress( + &sandbox.id, + "Scheduled", + format!("Docker sandbox accepted for image \"{image}\""), + HashMap::from([("image_ref".to_string(), image)]), + ); + self.publish_sandbox_snapshot(pending_sandbox_snapshot( + sandbox, + &self.config.sandbox_namespace, + provisioning_condition(), + false, + )); + + let driver = self.clone(); + let sandbox_for_task = sandbox.clone(); + let sandbox_id = sandbox.id.clone(); + let task = tokio::spawn(async move { + driver.provision_sandbox(sandbox_for_task).await; + }); + + let mut pending = self.pending.lock().await; + if let Some(record) = pending.get_mut(&sandbox_id) { + record.task = Some(task); + } else { + task.abort(); + } + + Ok(()) + } + + async fn provision_sandbox(&self, sandbox: DriverSandbox) { + match self.provision_sandbox_inner(&sandbox).await { + Ok(()) => { + self.clear_pending_sandbox(&sandbox.id).await; + } + Err(failure) => { + self.fail_pending_sandbox(&sandbox, &failure).await; + } + } + } + + async fn provision_sandbox_inner( + &self, + sandbox: &DriverSandbox, + ) -> Result<(), DockerProvisioningFailure> { + let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { + DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) + })?; + let template = validated.template; + self.ensure_image_available(&sandbox.id, &template.image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("ImagePullFailed", status.message()) + })?; + let token_file_created = write_sandbox_token_file(sandbox, &self.config) + .await + .map_err(|status| { + DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) + })?; + + let container_name = container_name_for_sandbox(sandbox); + let gpu_devices = self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::next_device_ids, + ) + .await + .map_err(|status| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) + })?; + let create_body = build_container_create_body_with_gpu_devices( + sandbox, + &self.config, + &validated.driver_config, + gpu_devices.as_deref(), + ) + .map_err(|status| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) + })?; + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + create_body, + ) + .await + .map_err(|err| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", err), + ) + })?; + self.publish_docker_progress( + &sandbox.id, + "Created", + format!("Created Docker container \"{container_name}\""), + HashMap::from([("container_name".to_string(), container_name.clone())]), + ); + + if let Err(err) = self.docker.start_container(&container_name, None).await { + let cleanup = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + if let Err(cleanup_err) = cleanup { + warn!( + sandbox_id = %sandbox.id, + container_name, + error = %cleanup_err, + "Failed to clean up Docker container after start failure" + ); + } + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + return Err(DockerProvisioningFailure::from_status( + "ContainerStartFailed", + create_status_from_docker_error("start docker sandbox container", err), + )); + } + self.publish_docker_progress( + &sandbox.id, + "Started", + format!("Started Docker container \"{container_name}\""), + HashMap::from([("container_name".to_string(), container_name)]), + ); + if let Err(err) = self + .publish_container_snapshot(&sandbox.id, &sandbox.name) + .await + { + warn!( + sandbox_id = %sandbox.id, + error = %err, + "Failed to publish Docker sandbox snapshot after start" + ); + } + + Ok(()) + } + + async fn delete_sandbox_inner( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; + if let Some(record) = pending.as_ref() + && let Some(task) = record.task.as_ref() + { + task.abort(); + } + + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + if let Some(record) = pending { + let container_name = container_name_for_sandbox(&record.sandbox); + match self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + Ok(()) => { + cleanup_sandbox_token_file(&record.sandbox, &self.config); + return Ok(true); + } + Err(err) if is_not_found_error(&err) => { + cleanup_sandbox_token_file(&record.sandbox, &self.config); + return Ok(true); + } + Err(err) => { + return Err(internal_status("delete docker sandbox container", err)); + } + } + } + return Ok(false); + }; + let Some(target) = summary_container_target(&container) else { + return Ok(pending.is_some()); + }; + + match self + .docker + .remove_container( + &target, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + Ok(()) => { + cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + Ok(true) + } + Err(err) if is_not_found_error(&err) => { + cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + Ok(pending.is_some()) + } + Err(err) => Err(internal_status("delete docker sandbox container", err)), + } + } + + async fn stop_sandbox_inner(&self, sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + if let Some(task) = record.task { + task.abort(); + } + cleanup_sandbox_token_file(&record.sandbox, &self.config); + self.publish_deleted(record.sandbox.id); + return Ok(()); + } + return Err(Status::not_found("sandbox not found")); + }; + let Some(target) = summary_container_target(&container) else { + return Err(Status::not_found("sandbox container has no id or name")); + }; + + match self + .docker + .stop_container( + &target, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) + .build(), + ), + ) + .await + { + Ok(()) => Ok(()), + Err(err) if is_not_modified_error(&err) => Ok(()), + Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), + Err(err) => Err(internal_status("stop docker sandbox container", err)), + } + } + + /// Start a managed sandbox container that was previously stopped. Used + /// by the gateway to resume sandboxes after a restart so that running + /// state in the gateway store is matched by an actually-running + /// container. + /// + /// Returns `Ok(true)` when a container existed and was started (or was + /// already running), `Ok(false)` when no managed container is found for + /// the sandbox, and `Err(...)` for any Docker failure. + pub async fn resume_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + return Ok(false); + }; + let Some(target) = summary_container_target(&container) else { + return Ok(false); + }; + let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); + if !container_state_needs_resume(state) { + return Ok(true); + } + + match self.docker.start_container(&target, None).await { + Ok(()) => Ok(true), + // Already running — race with another resume path or the + // restart policy. Treat as success. + Err(err) if is_not_modified_error(&err) => Ok(true), + Err(err) if is_not_found_error(&err) => Ok(false), + Err(err) => Err(internal_status("start docker sandbox container", err)), + } + } + + pub async fn stop_managed_containers_on_shutdown(&self) -> Result { + let containers = self.list_managed_container_summaries().await?; + let targets = containers + .into_iter() + .filter_map(|container| { + let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); + if container_state_needs_shutdown_stop(state) { + summary_container_target(&container) + } else { + None + } + }) + .collect::>(); + let target_count = targets.len(); + let mut stopped = 0usize; + let mut failures = Vec::new(); + let stop_timeout_secs = self.config.stop_timeout_secs; + + let mut stop_results = futures::stream::iter(targets.into_iter().map(|target| { + let docker = self.docker.clone(); + async move { + let result = docker + .stop_container( + &target, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(stop_timeout_secs)) + .build(), + ), + ) + .await; + (target, result) + } + })) + .buffer_unordered(16); + + while let Some((target, result)) = stop_results.next().await { + match result { + Ok(()) => { + stopped += 1; + } + Err(err) if is_not_found_error(&err) || is_not_modified_error(&err) => {} + Err(err) => { + warn!( + container = %target, + error = %err, + "Failed to stop Docker sandbox container during shutdown" + ); + failures.push(target); + } + } + } + + if !failures.is_empty() { + return Err(Status::internal(format!( + "failed to stop {} of {target_count} Docker sandbox containers during shutdown", + failures.len() + ))); + } + + Ok(stopped) + } + + async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { + let mut pending = self.pending.lock().await; + if pending + .values() + .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) + { + return Err(Status::already_exists("sandbox already exists")); + } + + pending.insert( + sandbox.id.clone(), + PendingSandboxRecord { + sandbox: pending_sandbox_snapshot( + sandbox, + &self.config.sandbox_namespace, + provisioning_condition(), + false, + ), + task: None, + }, + ); + Ok(()) + } + + async fn pending_snapshot( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Option { + let pending = self.pending.lock().await; + pending + .values() + .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) + .map(|record| record.sandbox.clone()) + } + + async fn pending_snapshot_map(&self) -> HashMap { + let pending = self.pending.lock().await; + pending + .iter() + .map(|(sandbox_id, record)| (sandbox_id.clone(), record.sandbox.clone())) + .collect() + } + + async fn clear_pending_sandbox(&self, sandbox_id: &str) { + let mut pending = self.pending.lock().await; + pending.remove(sandbox_id); + } + + async fn remove_pending_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Option { + let mut pending = self.pending.lock().await; + let id = pending.iter().find_map(|(id, record)| { + pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) + })?; + pending.remove(&id) + } + + async fn fail_pending_sandbox( + &self, + sandbox: &DriverSandbox, + failure: &DockerProvisioningFailure, + ) { + cleanup_sandbox_token_file(sandbox, &self.config); + let snapshot = pending_sandbox_snapshot( + sandbox, + &self.config.sandbox_namespace, + error_condition(failure.reason, &failure.message), + false, + ); + { + let mut pending = self.pending.lock().await; + if let Some(record) = pending.get_mut(&sandbox.id) { + record.sandbox = snapshot.clone(); + record.task = None; + } else { + return; + } + } + + self.publish_platform_event( + sandbox.id.clone(), + platform_event( + "docker", + "Warning", + failure.reason, + format!("Docker sandbox provisioning failed: {}", failure.message), + ), + ); + self.publish_sandbox_snapshot(snapshot); + } + + async fn publish_container_snapshot( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result<(), Status> { + if let Some(summary) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + && let Some(sandbox) = + sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) + { + self.publish_sandbox_snapshot(sandbox); + } + Ok(()) + } + + fn publish_sandbox_snapshot(&self, sandbox: DriverSandbox) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + }); + } + + fn publish_deleted(&self, sandbox_id: String) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id }, + )), + }); + } + + fn publish_platform_event(&self, sandbox_id: String, event: DriverPlatformEvent) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id, + event: Some(event), + }, + )), + }); + } + + fn publish_docker_progress( + &self, + sandbox_id: &str, + reason: &str, + message: String, + mut metadata: HashMap, + ) { + attach_docker_progress_metadata(&mut metadata, reason, &message); + self.publish_platform_event( + sandbox_id.to_string(), + DriverPlatformEvent { + timestamp_ms: openshell_core::time::now_ms(), + source: "docker".to_string(), + r#type: "Normal".to_string(), + reason: reason.to_string(), + message, + metadata, + }, + ); + } + + async fn poll_loop(self) { + let mut previous = match self.current_snapshot_map().await { + Ok(snapshots) => snapshots, + Err(err) => { + warn!(error = %err, "Failed to seed Docker sandbox watch state"); + HashMap::new() + } + }; + + // Exponential backoff on consecutive Docker failures to avoid a 2s + // warn-log flood when the daemon is unreachable for an extended + // period (e.g. restart, socket removed). + let mut backoff = WATCH_POLL_INTERVAL; + loop { + tokio::time::sleep(backoff).await; + match self.current_snapshot_map().await { + Ok(current) => { + emit_snapshot_diff(&self.events, &previous, ¤t); + previous = current; + backoff = WATCH_POLL_INTERVAL; + } + Err(err) => { + warn!( + error = %err, + backoff_secs = backoff.as_secs(), + "Failed to poll Docker sandboxes" + ); + backoff = (backoff * 2).min(WATCH_POLL_MAX_BACKOFF); + } + } + } + } + + async fn current_snapshot_map(&self) -> Result, Status> { + self.current_snapshots().await.map(|snapshots| { + snapshots + .into_iter() + .map(|sandbox| (sandbox.id.clone(), sandbox)) + .collect() + }) + } + + async fn list_managed_container_summaries(&self) -> Result, Status> { + let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); + self.docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|err| internal_status("list Docker sandbox containers", err)) + } + + async fn find_managed_container_summary( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result, Status> { + let mut label_filter_values = Vec::new(); + if !sandbox_id.is_empty() { + label_filter_values.push(format!("{LABEL_SANDBOX_ID}={sandbox_id}")); + } else if !sandbox_name.is_empty() { + label_filter_values.push(format!("{LABEL_SANDBOX_NAME}={sandbox_name}")); + } + + let filters = + managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); + let containers = self + .docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|err| internal_status("find Docker sandbox container", err))?; + + Ok(containers.into_iter().find(|summary| { + let Some(labels) = summary.labels.as_ref() else { + return false; + }; + let namespace_matches = labels + .get(LABEL_SANDBOX_NAMESPACE) + .is_some_and(|value| value == &self.config.sandbox_namespace); + let id_matches = sandbox_id.is_empty() + || labels + .get(LABEL_SANDBOX_ID) + .is_some_and(|value| value == sandbox_id); + let name_matches = sandbox_name.is_empty() + || labels + .get(LABEL_SANDBOX_NAME) + .is_some_and(|value| value == sandbox_name); + namespace_matches && id_matches && name_matches + })) + } + + async fn ensure_image_available(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { + let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); + match policy.as_str() { + "" | "ifnotpresent" => { + if self.docker.inspect_image(image).await.is_ok() { + self.publish_docker_progress( + sandbox_id, + "ImagePresent", + format!("Docker image \"{image}\" is already present"), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + return Ok(()); + } + self.pull_image(sandbox_id, image).await + } + "always" => self.pull_image(sandbox_id, image).await, + "never" => match self.docker.inspect_image(image).await { + Ok(_) => { + self.publish_docker_progress( + sandbox_id, + "ImagePresent", + format!("Docker image \"{image}\" is already present"), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + Ok(()) + } + Err(err) if is_not_found_error(&err) => Err(Status::failed_precondition(format!( + "docker image '{image}' is not present locally and image_pull_policy=Never" + ))), + Err(err) => Err(internal_status("inspect Docker image", err)), + }, + other => Err(Status::failed_precondition(format!( + "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", + ))), + } + } + + async fn pull_image(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { + self.publish_docker_progress( + sandbox_id, + "Pulling", + format!("Pulling Docker image \"{image}\""), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + let mut stream = self.docker.create_image( + Some(CreateImageOptions { + from_image: Some(image.to_string()), + ..Default::default() + }), + None, + None, + ); + while let Some(result) = stream.next().await { + let info = result.map_err(|err| internal_status("pull Docker image", err))?; + if let Some(message) = info + .error_detail + .as_ref() + .and_then(|detail| detail.message.as_ref()) + { + return Err(Status::failed_precondition(format!( + "pull Docker image '{image}' failed: {message}" + ))); + } + if let Some(event) = docker_pull_progress_event(image, &info) { + self.publish_platform_event(sandbox_id.to_string(), event); + } + } + self.publish_docker_progress( + sandbox_id, + "Pulled", + format!("Pulled Docker image \"{image}\""), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + Ok(()) + } +} + +#[tonic::async_trait] +impl ComputeDriver for DockerComputeDriver { + type WatchSandboxesStream = WatchStream; + + async fn get_capabilities( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(self.capabilities())) + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + let validated = Self::validated_sandbox(&sandbox, &self.config)?; + self.validate_user_volume_mounts_available(&validated.driver_config) + .await?; + let _ = self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await?; + Ok(Response::new(ValidateSandboxCreateResponse {})) + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; + + let sandbox = self + .get_sandbox_snapshot(&request.sandbox_id, &request.sandbox_name) + .await? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + + if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { + return Err(Status::failed_precondition( + "sandbox_id did not match the fetched sandbox", + )); + } + + Ok(Response::new(GetSandboxResponse { + sandbox: Some(sandbox), + })) + } + + async fn list_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(ListSandboxesResponse { + sandboxes: self.current_snapshots().await?, + })) + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.create_sandbox_inner(&sandbox).await?; + Ok(Response::new(CreateSandboxResponse {})) + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; + + self.stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) + .await?; + Ok(Response::new(StopSandboxResponse {})) + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; + + let event_sandbox_id = request.sandbox_id.clone(); + let deleted = self + .delete_sandbox_inner(&request.sandbox_id, &request.sandbox_name) + .await?; + if deleted && !event_sandbox_id.is_empty() { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { + sandbox_id: event_sandbox_id, + }, + )), + }); + } + + Ok(Response::new(DeleteSandboxResponse { deleted })) + } + + async fn watch_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + // Subscribe before taking the initial snapshot so any event emitted + // between the snapshot and this subscriber becoming active is still + // delivered. Downstream consumers treat sandbox events as + // idempotent (keyed by sandbox id), so a duplicate event is benign + // while a missed one leaks state. + let mut rx = self.events.subscribe(); + let initial = self.current_snapshots().await?; + let (tx, out_rx) = mpsc::channel(WATCH_BUFFER); + tokio::spawn(async move { + for sandbox in initial { + if tx + .send(Ok(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + })) + .await + .is_err() + { + return; + } + } + + loop { + match rx.recv().await { + Ok(event) => { + if tx.send(Ok(event)).await.is_err() { + return; + } + } + Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => return, + } + } + }); + + Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) + } +} + +impl DockerProvisioningFailure { + fn new(reason: &'static str, message: impl Into) -> Self { + Self { + reason, + message: message.into(), + } + } + + fn from_status(reason: &'static str, status: Status) -> Self { + Self::new(reason, status.message()) + } +} + +fn sandbox_image(sandbox: &DriverSandbox) -> Option { + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map(|template| template.image.clone()) + .filter(|image| !image.trim().is_empty()) +} + +fn pending_sandbox_snapshot( + sandbox: &DriverSandbox, + namespace: &str, + condition: DriverCondition, + deleting: bool, +) -> DriverSandbox { + DriverSandbox { + id: sandbox.id.clone(), + name: sandbox.name.clone(), + namespace: namespace.to_string(), + spec: None, + status: Some(DriverSandboxStatus { + sandbox_name: sandbox.name.clone(), + instance_id: String::new(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![condition], + deleting, + }), + workspace: sandbox.workspace.clone(), + } +} + +fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { + (!sandbox_id.is_empty() && sandbox.id == sandbox_id) + || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) +} + +fn provisioning_condition() -> DriverCondition { + DriverCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "Starting".to_string(), + message: "Docker container is starting".to_string(), + last_transition_time: String::new(), + } +} + +fn error_condition(reason: &str, message: &str) -> DriverCondition { + DriverCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + message: message.to_string(), + last_transition_time: String::new(), + } +} + +fn platform_event( + source: &str, + event_type: &str, + reason: &str, + message: String, +) -> DriverPlatformEvent { + DriverPlatformEvent { + timestamp_ms: openshell_core::time::now_ms(), + source: source.to_string(), + r#type: event_type.to_string(), + reason: reason.to_string(), + message, + metadata: HashMap::new(), + } +} + +fn docker_pull_progress_event(image: &str, info: &CreateImageInfo) -> Option { + let status = info.status.as_deref().map(str::trim)?; + if status.is_empty() { + return None; + } + + let mut metadata = HashMap::from([ + ("image_ref".to_string(), image.to_string()), + ("docker_status".to_string(), status.to_string()), + ]); + if let Some(layer_id) = info.id.as_deref().filter(|id| !id.is_empty()) { + metadata.insert("layer_id".to_string(), layer_id.to_string()); + } + if let Some(detail) = docker_pull_progress_detail(info) { + metadata.insert("detail".to_string(), detail); + } + attach_docker_progress_metadata(&mut metadata, "PullingLayer", status); + + Some(DriverPlatformEvent { + timestamp_ms: openshell_core::time::now_ms(), + source: "docker".to_string(), + r#type: "Normal".to_string(), + reason: "PullingLayer".to_string(), + message: docker_pull_message(info, status), + metadata, + }) +} + +fn docker_pull_message(info: &CreateImageInfo, status: &str) -> String { + info.id.as_deref().filter(|id| !id.is_empty()).map_or_else( + || format!("Docker image pull: {status}"), + |layer_id| format!("Docker image pull {layer_id}: {status}"), + ) +} + +fn docker_pull_progress_detail(info: &CreateImageInfo) -> Option { + let status = info.status.as_deref().unwrap_or("Pulling"); + let layer_id = info.id.as_deref().filter(|id| !id.is_empty()); + let progress = info + .progress_detail + .as_ref() + .and_then(format_progress_detail); + + match (layer_id, progress) { + (Some(layer_id), Some(progress)) => Some(format!("{status} {layer_id} ({progress})")), + (Some(layer_id), None) => Some(format!("{status} {layer_id}")), + (None, Some(progress)) => Some(format!("{status} ({progress})")), + (None, None) => (!status.is_empty()).then(|| status.to_string()), + } +} + +fn format_progress_detail(progress: &ProgressDetail) -> Option { + let current = progress.current.and_then(|value| u64::try_from(value).ok()); + let total = progress + .total + .and_then(|value| u64::try_from(value).ok()) + .filter(|value| *value > 0); + + match (current, total) { + (Some(current), Some(total)) => { + Some(format!("{}/{}", format_bytes(current), format_bytes(total))) + } + (Some(current), _) if current > 0 => Some(format_bytes(current)), + _ => None, + } +} + +fn attach_docker_progress_metadata( + metadata: &mut HashMap, + reason: &str, + message: &str, +) { + match reason { + "Scheduled" => { + mark_progress_complete( + metadata, + PROGRESS_STEP_REQUESTING_SANDBOX, + "Sandbox allocated", + ); + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(image) = metadata.get("image_ref").cloned() { + mark_progress_detail(metadata, image); + } + } + "Pulling" => { + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(image) = metadata.get("image_ref").cloned() { + mark_progress_detail(metadata, image); + } + } + "PullingLayer" => { + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(detail) = metadata + .get("detail") + .cloned() + .filter(|detail| !detail.is_empty()) + { + mark_progress_detail(metadata, detail); + } else if !message.is_empty() { + mark_progress_detail(metadata, message); + } + } + "ImagePresent" => { + mark_progress_complete( + metadata, + PROGRESS_STEP_PULLING_IMAGE, + "Image already present", + ); + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + } + "Pulled" => { + mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, "Image pulled"); + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + } + "Created" => { + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + mark_progress_detail(metadata, "Container created"); + } + "Started" => { + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + mark_progress_detail(metadata, "Waiting for supervisor relay"); + } + _ => {} + } +} + +#[cfg(test)] +fn docker_driver_config( + template: &DriverSandboxTemplate, + enable_bind_mounts: bool, +) -> Result { + let config = + DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; + validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?; + Ok(config) +} + +/// Collect user-supplied bind mounts as string-format binds. +/// +/// Bind mounts use the legacy `Binds` field (`-v` syntax) rather than the +/// structured `Mount` API because the Docker Engine Mount object does not +/// support `SELinux` relabelling (`:z` / `:Z`). The string format does. +fn docker_driver_bind_strings(config: &DockerSandboxDriverConfig) -> Result, Status> { + config + .mounts + .iter() + .filter_map(|m| match m { + DockerDriverMountConfig::Bind { + source, + target, + read_only, + selinux_label, + } => Some(docker_bind_string( + source, + target, + *read_only, + *selinux_label, + )), + _ => None, + }) + .collect() +} + +fn docker_bind_string( + source: &str, + target: &str, + read_only: bool, + selinux_label: Option, +) -> Result { + driver_mounts::validate_absolute_mount_source(source, "bind source") + .map_err(Status::failed_precondition)?; + // Legacy `-v` binds silently create missing source directories as empty, + // root-owned paths. The structured `--mount` API that was used before this + // change rejected missing sources at container-create time. Preserve that + // fail-fast behaviour with an explicit existence check. + if !Path::new(source).exists() { + return Err(Status::failed_precondition(format!( + "bind source path does not exist: {source}" + ))); + } + driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; + let normalized_target = driver_mounts::normalize_mount_target(target); + + let mut opts = Vec::new(); + if read_only { + opts.push("ro"); + } + match selinux_label { + Some(SelinuxLabel::Shared) => opts.push("z"), + Some(SelinuxLabel::Private) => opts.push("Z"), + None => {} + } + + if opts.is_empty() { + Ok(format!("{source}:{normalized_target}")) + } else { + Ok(format!("{source}:{normalized_target}:{}", opts.join(","))) + } +} + +/// Collect user-supplied non-bind mounts as structured `Mount` objects. +fn docker_driver_mounts(config: &DockerSandboxDriverConfig) -> Result, Status> { + config + .mounts + .iter() + .filter_map(|m| docker_mount_from_config(m).transpose()) + .collect() +} + +fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result, Status> { + match config { + DockerDriverMountConfig::Bind { .. } => { + // Bind mounts are handled via docker_driver_bind_strings. + Ok(None) + } + DockerDriverMountConfig::Volume { + source, + target, + read_only, + subpath, + } => Ok(Some(Mount { + typ: Some(MountTypeEnum::VOLUME), + source: Some(source.clone()), + target: Some(target.clone()), + read_only: Some(*read_only), + volume_options: subpath.as_ref().map(|subpath| MountVolumeOptions { + subpath: Some(subpath.clone()), + ..Default::default() + }), + ..Default::default() + })), + DockerDriverMountConfig::Tmpfs { + target, + options, + size_bytes, + mode, + } => Ok(Some(Mount { + typ: Some(MountTypeEnum::TMPFS), + target: Some(target.clone()), + tmpfs_options: Some(MountTmpfsOptions { + size_bytes: validate_optional_positive_integral_i64( + *size_bytes, + "tmpfs size_bytes", + )?, + mode: validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?, + options: (!options.is_empty()) + .then(|| { + options + .iter() + .map(|option| docker_tmpfs_option(option)) + .collect::, _>>() + }) + .transpose()?, + }), + ..Default::default() + })), + DockerDriverMountConfig::Image { .. } => Err(Status::failed_precondition( + "invalid docker driver_config: docker image mounts are not supported", + )), + } +} + +fn validate_docker_driver_mounts( + mounts: &[DockerDriverMountConfig], + enable_bind_mounts: bool, +) -> Result<(), Status> { + let mut targets = HashSet::new(); + for mount in mounts { + let target = match mount { + DockerDriverMountConfig::Bind { source, target, .. } => { + if !enable_bind_mounts { + return Err(Status::failed_precondition( + "docker bind mounts require enable_bind_mounts = true in [openshell.drivers.docker]", + )); + } + driver_mounts::validate_absolute_mount_source(source, "bind source") + .map_err(Status::failed_precondition)?; + target + } + DockerDriverMountConfig::Volume { + source, + target, + subpath, + .. + } => { + driver_mounts::validate_mount_source(source, "volume source") + .map_err(Status::failed_precondition)?; + if let Some(subpath) = subpath { + driver_mounts::validate_mount_subpath(subpath) + .map_err(Status::failed_precondition)?; + } + target + } + DockerDriverMountConfig::Tmpfs { + target, + options, + size_bytes, + mode, + } => { + validate_optional_positive_integral_i64(*size_bytes, "tmpfs size_bytes")?; + validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?; + for option in options { + docker_tmpfs_option(option)?; + } + target + } + DockerDriverMountConfig::Image { + source, + target, + read_only, + subpath, + } => { + let _ = (source, target, read_only, subpath); + return Err(Status::failed_precondition( + "invalid docker driver_config: docker image mounts are not supported", + )); + } + }; + driver_mounts::validate_container_mount_target(target) + .map_err(Status::failed_precondition)?; + let normalized_target = driver_mounts::normalize_mount_target(target); + if !targets.insert(normalized_target.clone()) { + return Err(Status::failed_precondition(format!( + "duplicate docker driver_config mount target '{normalized_target}'" + ))); + } + } + Ok(()) +} + +fn validate_optional_positive_integral_i64( + value: Option, + field: &str, +) -> Result, Status> { + let Some(value) = validate_optional_integral_i64(value, field)? else { + return Ok(None); + }; + if value <= 0 { + return Err(Status::failed_precondition(format!( + "{field} must be positive" + ))); + } + Ok(Some(value)) +} + +fn validate_optional_nonnegative_integral_i64( + value: Option, + field: &str, +) -> Result, Status> { + let Some(value) = validate_optional_integral_i64(value, field)? else { + return Ok(None); + }; + if value < 0 { + return Err(Status::failed_precondition(format!( + "{field} must be zero or greater" + ))); + } + Ok(Some(value)) +} + +fn validate_optional_integral_i64(value: Option, field: &str) -> Result, Status> { + let Some(value) = value else { + return Ok(None); + }; + if !value.is_finite() || value.fract() != 0.0 { + return Err(Status::failed_precondition(format!( + "{field} must be an integer" + ))); + } + value.to_string().parse::().map(Some).map_err(|_| { + Status::failed_precondition(format!("{field} must be representable as an i64")) + }) +} + +fn docker_tmpfs_option(option: &str) -> Result, Status> { + let option = option.trim(); + if option.is_empty() { + return Err(Status::failed_precondition( + "tmpfs options must not contain empty values", + )); + } + if let Some((key, value)) = option.split_once('=') { + let key = key.trim(); + let value = value.trim(); + if key.is_empty() || value.is_empty() { + return Err(Status::failed_precondition( + "tmpfs key=value options must include both key and value", + )); + } + Ok(vec![key.to_string(), value.to_string()]) + } else { + Ok(vec![option.to_string()]) + } +} + +fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { + volume.driver == "local" + && volume.options.get("o").is_some_and(|options| { + options.split(',').any(|option| { + let option = option.trim(); + option.eq_ignore_ascii_case("bind") || option.eq_ignore_ascii_case("rbind") + }) + }) +} + +fn build_binds( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let mut binds = vec![format!( + "{}:{}:ro,z", + config.supervisor_bin.display(), + SUPERVISOR_MOUNT_PATH + )]; + if let Some(tls) = &config.guest_tls { + binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); + binds.push(format!( + "{}:{}:ro,z", + tls.cert.display(), + TLS_CERT_MOUNT_PATH + )); + binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); + } + if sandbox + .spec + .as_ref() + .is_some_and(|spec| !spec.sandbox_token.is_empty()) + { + binds.push(format!( + "{}:{}:ro,z", + sandbox_token_host_path(sandbox, config)?.display(), + SANDBOX_TOKEN_MOUNT_PATH + )); + } + Ok(binds) +} + +fn sandbox_token_host_path( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + sandbox_token_host_path_by_id(&sandbox.id, config) +} + +fn sandbox_token_host_path_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result { + openshell_core::driver_utils::sandbox_token_path( + "docker-sandbox-tokens", + Some(&config.sandbox_namespace), + sandbox_id, + ) + .map_err(|err| { + Status::internal(format!( + "resolve sandbox token state directory failed: {err}" + )) + }) +} + +async fn write_sandbox_token_file( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let Some(spec) = sandbox.spec.as_ref() else { + return Ok(false); + }; + if spec.sandbox_token.is_empty() { + return Ok(false); + } + let path = sandbox_token_host_path(sandbox, config)?; + if let Some(parent) = path.parent() { + openshell_core::paths::create_dir_restricted(parent).map_err(|err| { + Status::internal(format!( + "create sandbox token directory {} failed: {err}", + parent.display() + )) + })?; + } + tokio::fs::write(&path, format!("{}\n", spec.sandbox_token)) + .await + .map_err(|err| { + Status::internal(format!( + "write sandbox token file {} failed: {err}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(&path).map_err(|err| { + Status::internal(format!( + "restrict sandbox token file {} failed: {err}", + path.display() + )) + })?; + Ok(true) +} + +fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { + cleanup_sandbox_token_file_by_id(&sandbox.id, config); +} + +fn cleanup_sandbox_token_file_for_delete( + sandbox_id: &str, + pending: Option<&PendingSandboxRecord>, + config: &DockerDriverRuntimeConfig, +) { + if !sandbox_id.is_empty() { + cleanup_sandbox_token_file_by_id(sandbox_id, config); + } else if let Some(record) = pending { + cleanup_sandbox_token_file(&record.sandbox, config); + } +} + +fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { + return; + }; + if let Err(err) = std::fs::remove_file(&path) + && err.kind() != std::io::ErrorKind::NotFound + { + warn!( + sandbox_id = %sandbox_id, + path = %path.display(), + error = %err, + "Failed to remove Docker sandbox token file" + ); + } + if let Some(dir) = path.parent() { + let _ = std::fs::remove_dir(dir); + } +} + +fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { + let mut environment = HashMap::from([ + ("HOME".to_string(), "/root".to_string()), + ("PATH".to_string(), SUPERVISOR_PATH.to_string()), + ("TERM".to_string(), "xterm".to_string()), + ( + "OPENSHELL_LOG_LEVEL".to_string(), + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), + ), + ]); + + if let Some(spec) = sandbox.spec.as_ref() { + let mut user_env = HashMap::new(); + if let Some(template) = spec.template.as_ref() { + user_env.extend(template.environment.clone()); + } + user_env.extend(spec.environment.clone()); + environment.extend(user_env.clone()); + if !user_env.is_empty() + && let Ok(json) = serde_json::to_string(&user_env) + { + environment.insert( + openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), + json, + ); + } + } + + environment.insert( + openshell_core::sandbox_env::ENDPOINT.to_string(), + config.grpc_endpoint.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SANDBOX_ID.to_string(), + sandbox.id.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SANDBOX.to_string(), + sandbox.name.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), + config.ssh_socket_path.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SANDBOX_COMMAND.to_string(), + SANDBOX_COMMAND.to_string(), + ); + environment.insert( + openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), + openshell_core::telemetry::enabled_env_value().to_string(), + ); + // The root supervisor executes namespace helpers during bootstrap; keep + // their search path driver-owned even when the template/spec set PATH. + environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); + if config.guest_tls.is_some() { + environment.insert( + openshell_core::sandbox_env::TLS_CA.to_string(), + TLS_CA_MOUNT_PATH.to_string(), + ); + environment.insert( + openshell_core::sandbox_env::TLS_CERT.to_string(), + TLS_CERT_MOUNT_PATH.to_string(), + ); + environment.insert( + openshell_core::sandbox_env::TLS_KEY.to_string(), + TLS_KEY_MOUNT_PATH.to_string(), + ); + } + + environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); + environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + + // Gateway-minted sandbox JWT. Keep the raw bearer out of container + // metadata; the supervisor reads it from this driver-owned bind mount. + if let Some(spec) = sandbox.spec.as_ref() + && !spec.sandbox_token.is_empty() + { + environment.insert( + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), + SANDBOX_TOKEN_MOUNT_PATH.to_string(), + ); + } + + let mut pairs = environment.into_iter().collect::>(); + pairs.sort_by(|left, right| left.0.cmp(&right.0)); + pairs + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect() +} + +fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { + CdiGpuInventory::new( + info.discovered_devices + .as_deref() + .unwrap_or_default() + .iter() + .filter(|device| device.source.as_deref() == Some("cdi")) + .filter_map(|device| device.id.as_deref()), + ) +} + +fn docker_info_reports_wsl2(info: &SystemInfo) -> bool { + [ + info.kernel_version.as_deref(), + info.operating_system.as_deref(), + ] + .into_iter() + .flatten() + .any(os_or_kernel_reports_wsl2) +} + +fn os_or_kernel_reports_wsl2(value: &str) -> bool { + let value = value.to_ascii_lowercase(); + value.contains("wsl2") || value.contains("microsoft-standard") +} + +fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status { + Status::failed_precondition(err.to_string()) +} + +#[cfg(test)] +fn build_container_create_body( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let template = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let driver_config = docker_driver_config(template, config.enable_bind_mounts)?; + let gpu_requirements = sandbox + .spec + .as_ref() + .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); + let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() { + validate_specific_gpu_device_request( + gpu_requirements, + cdi_devices, + "driver_config.cdi_devices", + ) + .map_err(Status::invalid_argument)?; + Some(cdi_devices.as_slice()) + } else { + None + }; + build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) +} + +fn build_container_create_body_with_gpu_devices( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + driver_config: &DockerSandboxDriverConfig, + gpu_device_ids: Option<&[String]>, +) -> Result { + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; + let template = spec + .template + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let resource_limits = docker_resource_limits(template)?; + let user_mounts = docker_driver_mounts(driver_config)?; + let user_bind_strings = docker_driver_bind_strings(driver_config)?; + let device_requests = gpu_device_ids.map(|device_ids| { + vec![DeviceRequest { + driver: Some("cdi".to_string()), + device_ids: Some(device_ids.to_vec()), + ..Default::default() + }] + }); + let mut labels = template.labels.clone(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); + labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); + labels.insert( + LABEL_SANDBOX_WORKSPACE.to_string(), + sandbox.workspace.clone(), + ); + // The list/get/find paths filter by `config.sandbox_namespace`, so use + // the same value here. `DriverSandbox.namespace` is unset on the request + // path (the gateway elides it), and using it would produce containers + // that the driver itself cannot find afterwards. + labels.insert( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ); + + Ok(ContainerCreateBody { + image: Some(template.image.clone()), + user: Some("0".to_string()), + env: Some(build_environment(sandbox, config)), + entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), + // Clear the image CMD so Docker does not append inherited args to the + // supervisor entrypoint. + cmd: Some(Vec::new()), + labels: Some(labels), + host_config: Some(HostConfig { + nano_cpus: resource_limits.nano_cpus, + memory: resource_limits.memory_bytes, + pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, + device_requests, + binds: { + let mut binds = build_binds(sandbox, config)?; + binds.extend(user_bind_strings); + Some(binds) + }, + mounts: Some(user_mounts), + restart_policy: Some(RestartPolicy { + name: Some(RestartPolicyNameEnum::UNLESS_STOPPED), + maximum_retry_count: None, + }), + cap_add: Some(vec![ + "SYS_ADMIN".to_string(), + "NET_ADMIN".to_string(), + "SYS_PTRACE".to_string(), + "SYSLOG".to_string(), + ]), + // The sandbox supervisor needs to bind-mount `/run/netns`, + // mark it shared, and create per-process network namespaces. + // Docker's default AppArmor profile (`docker-default`) denies + // these mount operations even with CAP_SYS_ADMIN, so we opt + // out of AppArmor confinement for sandbox containers. The + // sandbox enforces its own security boundary via Landlock, + // seccomp, OPA policy evaluation, and the dedicated network + // namespace it sets up for the agent — AppArmor at the + // container layer is redundant relative to those controls + // and conflicts with them in this case. + security_opt: Some(vec!["apparmor=unconfined".to_string()]), + network_mode: Some(config.network_name.clone()), + extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), + ..Default::default() + }), + networking_config: Some(NetworkingConfig { + endpoints_config: Some(HashMap::from([( + config.network_name.clone(), + EndpointSettings::default(), + )])), + }), + ..Default::default() + }) +} + +/// Reject driver requests that arrive with neither a sandbox id nor a +/// sandbox name. Without this guard, downstream label filters degenerate +/// to "match every managed container in the namespace", which would let +/// `delete_sandbox`/`stop_sandbox`/`get_sandbox` pick an arbitrary +/// sandbox out of the set the driver manages. +fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { + if sandbox_id.is_empty() && sandbox_name.is_empty() { + return Err(Status::invalid_argument( + "sandbox_id or sandbox_name is required", + )); + } + Ok(()) +} + +fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { + let Ok(mut url) = Url::parse(endpoint) else { + return endpoint.to_string(); + }; + + if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { + return url.to_string(); + } + + endpoint.to_string() +} + +fn docker_network_name(config: &DockerComputeConfig) -> String { + let name = config.network_name.trim(); + if name.is_empty() { + return DEFAULT_DOCKER_NETWORK_NAME.to_string(); + } + name.to_string() +} + +fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Ok(None); + } + + trimmed + .parse() + .map(Some) + .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) +} + +fn docker_gateway_route( + info: &SystemInfo, + bridge_gateway_ip: IpAddr, + port: u16, + host_gateway_ip: Option, +) -> DockerGatewayRoute { + docker_gateway_route_for_host( + info, + bridge_gateway_ip, + port, + host_gateway_ip, + host_runtime_requires_host_gateway_alias(), + ) +} + +fn docker_gateway_route_for_host( + info: &SystemInfo, + bridge_gateway_ip: IpAddr, + port: u16, + host_gateway_ip: Option, + host_requires_host_gateway_alias: bool, +) -> DockerGatewayRoute { + if let Some(host_alias_ip) = host_gateway_ip { + return DockerGatewayRoute::Bridge { + bind_address: SocketAddr::new(host_alias_ip, port), + host_alias_ip, + }; + } + + if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { + DockerGatewayRoute::HostGateway + } else { + DockerGatewayRoute::Bridge { + bind_address: SocketAddr::new(bridge_gateway_ip, port), + host_alias_ip: bridge_gateway_ip, + } + } +} + +fn host_runtime_requires_host_gateway_alias() -> bool { + cfg!(target_os = "macos") +} + +/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, +/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing +/// constraint: the bridge gateway IP is reachable from inside containers but +/// not from the `OpenShell` server process running on the host, so callbacks +/// must traverse `host-gateway`. +/// +/// Each runtime is detected via the daemon's reported OS string or hostname, +/// supplemented by labels where the runtime publishes them. +fn uses_host_gateway_alias(info: &SystemInfo) -> bool { + let operating_system = info + .operating_system + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase(); + if operating_system.contains("docker desktop") { + return true; + } + + let name = info + .name + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase(); + if name.starts_with("colima") + || name.starts_with("lima-") + || name.starts_with("rancher-desktop") + || name.starts_with("orbstack") + { + return true; + } + + info.labels.as_ref().is_some_and(|labels| { + labels.iter().any(|label| { + label.starts_with("com.docker.desktop.") + || label.starts_with("dev.rancherdesktop.") + || label.starts_with("dev.orbstack.") + }) + }) +} + +fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { + match route { + DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ + format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), + format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), + ], + DockerGatewayRoute::HostGateway => vec![ + format!("{HOST_DOCKER_INTERNAL}:host-gateway"), + format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), + ], + } +} + +async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { + match docker.inspect_network(network_name, None).await { + Ok(network) => return validate_bridge_network(network_name, &network), + Err(err) if !is_not_found_error(&err) => { + return Err(Error::execution(format!( + "failed to inspect Docker network '{network_name}': {err}" + ))); + } + Err(_) => {} + } + + docker + .create_network(NetworkCreateRequest { + name: network_name.to_string(), + driver: Some(DOCKER_NETWORK_DRIVER.to_string()), + attachable: Some(true), + labels: Some(HashMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }) + .await + .map(|_| ()) + .or_else(|err| { + if is_conflict_error(&err) { + Ok(()) + } else { + Err(Error::execution(format!( + "failed to create Docker network '{network_name}': {err}" + ))) + } + })?; + + let network = docker + .inspect_network(network_name, None) + .await + .map_err(|err| { + Error::execution(format!( + "failed to inspect Docker network '{network_name}' after create: {err}" + )) + })?; + validate_bridge_network(network_name, &network) +} + +fn validate_bridge_network( + network_name: &str, + network: &bollard::models::NetworkInspect, +) -> CoreResult { + if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { + return Err(Error::config(format!( + "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", + network.driver.as_deref().unwrap_or("unknown") + ))); + } + + docker_bridge_gateway_ip(network_name, network) +} + +fn docker_bridge_gateway_ip( + network_name: &str, + network: &bollard::models::NetworkInspect, +) -> CoreResult { + let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { + return Err(Error::config(format!( + "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" + ))); + }; + + for config in configs { + let Some(gateway) = config.gateway.as_deref() else { + continue; + }; + let ip = gateway.parse::().map_err(|err| { + Error::config(format!( + "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" + )) + })?; + if matches!(ip, IpAddr::V4(_)) { + return Ok(ip); + } + } + + Err(Error::config(format!( + "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" + ))) +} + +fn docker_resource_limits( + template: &DriverSandboxTemplate, +) -> Result { + let Some(resources) = template.resources.as_ref() else { + return Ok(DockerResourceLimits::default()); + }; + + if !resources.cpu_request.trim().is_empty() { + return Err(Status::failed_precondition( + "docker compute driver does not support resources.requests.cpu", + )); + } + if !resources.memory_request.trim().is_empty() { + return Err(Status::failed_precondition( + "docker compute driver does not support resources.requests.memory", + )); + } + + Ok(DockerResourceLimits { + nano_cpus: parse_cpu_limit(&resources.cpu_limit)?, + memory_bytes: parse_memory_limit(&resources.memory_limit)?, + }) +} + +fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { + if value < 0 { + return Err(Error::config( + "docker sandbox_pids_limit must be zero or greater", + )); + } + Ok(()) +} + +fn docker_pids_limit(value: i64) -> Result, Status> { + if value < 0 { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be zero or greater", + )); + } + if value == 0 { + Ok(None) + } else { + Ok(Some(value)) + } +} + +#[allow(clippy::cast_possible_truncation)] +fn parse_cpu_limit(value: &str) -> Result, Status> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + if let Some(millicores) = value.strip_suffix('m') { + let millicores = millicores.parse::().map_err(|_| { + Status::failed_precondition(format!( + "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", + )) + })?; + if millicores <= 0 { + return Err(Status::failed_precondition( + "docker cpu_limit must be greater than zero", + )); + } + return Ok(Some(millicores.saturating_mul(1_000_000))); + } + + let cores = value.parse::().map_err(|_| { + Status::failed_precondition(format!( + "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", + )) + })?; + if !cores.is_finite() || cores <= 0.0 { + return Err(Status::failed_precondition( + "docker cpu_limit must be greater than zero", + )); + } + + Ok(Some((cores * 1_000_000_000.0).round() as i64)) +} + +#[allow(clippy::cast_possible_truncation)] +fn parse_memory_limit(value: &str) -> Result, Status> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + + let number_end = value + .find(|ch: char| !(ch.is_ascii_digit() || ch == '.')) + .unwrap_or(value.len()); + let (number, suffix) = value.split_at(number_end); + let amount = number.parse::().map_err(|_| { + Status::failed_precondition(format!( + "invalid docker memory_limit '{value}'; expected a Kubernetes-style quantity", + )) + })?; + if !amount.is_finite() || amount <= 0.0 { + return Err(Status::failed_precondition( + "docker memory_limit must be greater than zero", + )); + } + + let multiplier = match suffix { + "" => 1_f64, + "Ki" => 1024_f64, + "Mi" => 1024_f64.powi(2), + "Gi" => 1024_f64.powi(3), + "Ti" => 1024_f64.powi(4), + "Pi" => 1024_f64.powi(5), + "Ei" => 1024_f64.powi(6), + "K" => 1000_f64, + "M" => 1000_f64.powi(2), + "G" => 1000_f64.powi(3), + "T" => 1000_f64.powi(4), + "P" => 1000_f64.powi(5), + "E" => 1000_f64.powi(6), + _ => { + return Err(Status::failed_precondition(format!( + "invalid docker memory_limit suffix '{suffix}'", + ))); + } + }; + + Ok(Some((amount * multiplier).round() as i64)) +} + +fn sandbox_from_container_summary( + summary: &ContainerSummary, + readiness: &dyn SupervisorReadiness, +) -> Option { + let labels = summary.labels.as_ref()?; + let id = labels.get(LABEL_SANDBOX_ID)?.clone(); + let name = labels.get(LABEL_SANDBOX_NAME)?.clone(); + let namespace = labels + .get(LABEL_SANDBOX_NAMESPACE) + .cloned() + .unwrap_or_default(); + let workspace = labels + .get(LABEL_SANDBOX_WORKSPACE) + .cloned() + .unwrap_or_default(); + + let supervisor_connected = readiness.is_supervisor_connected(&id); + Some(DriverSandbox { + id, + name: name.clone(), + namespace, + spec: None, + status: Some(driver_status_from_summary( + summary, + &name, + supervisor_connected, + )), + workspace, + }) +} + +fn driver_status_from_summary( + summary: &ContainerSummary, + sandbox_name: &str, + supervisor_connected: bool, +) -> DriverSandboxStatus { + let state = summary.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); + let (ready, reason, message, deleting) = container_ready_condition(state, supervisor_connected); + + DriverSandboxStatus { + sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), + instance_id: summary.id.clone().unwrap_or_default(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![DriverCondition { + r#type: "Ready".to_string(), + status: ready.to_string(), + reason: reason.to_string(), + message: message.to_string(), + last_transition_time: String::new(), + }], + deleting, + } +} + +fn container_ready_condition( + state: ContainerSummaryStateEnum, + supervisor_connected: bool, +) -> (&'static str, &'static str, &'static str, bool) { + match state { + ContainerSummaryStateEnum::RUNNING => { + if supervisor_connected { + ( + "True", + "SupervisorConnected", + "Supervisor relay is live", + false, + ) + } else { + ( + "False", + "DependenciesNotReady", + "Container is running; waiting for supervisor relay", + false, + ) + } + } + ContainerSummaryStateEnum::CREATED => ("False", "Starting", "Container created", false), + ContainerSummaryStateEnum::RESTARTING => ( + "False", + "ContainerRestarting", + "Container is restarting after a failure", + false, + ), + ContainerSummaryStateEnum::EMPTY => { + ("False", "Starting", "Container state is unknown", false) + } + ContainerSummaryStateEnum::REMOVING => { + ("False", "Deleting", "Container is being removed", true) + } + ContainerSummaryStateEnum::PAUSED => { + ("False", "ContainerPaused", "Container is paused", false) + } + ContainerSummaryStateEnum::EXITED => { + ("False", "ContainerExited", "Container exited", false) + } + ContainerSummaryStateEnum::DEAD => ("False", "ContainerDead", "Container is dead", false), + } +} + +fn summary_container_name(summary: &ContainerSummary) -> Option { + summary + .names + .as_ref() + .and_then(|names| names.first()) + .map(|name| name.trim_start_matches('/').to_string()) + .filter(|name| !name.is_empty()) +} + +fn summary_container_target(summary: &ContainerSummary) -> Option { + // Prefer the container ID: it's stable while the container exists and is + // accepted by Docker APIs just like a name. Fall back to the parsed name + // for transient summaries that do not include an ID. + summary + .id + .as_deref() + .filter(|id| !id.is_empty()) + .map(str::to_string) + .or_else(|| summary_container_name(summary)) +} + +fn container_state_needs_shutdown_stop(state: ContainerSummaryStateEnum) -> bool { + matches!( + state, + ContainerSummaryStateEnum::RUNNING + | ContainerSummaryStateEnum::RESTARTING + | ContainerSummaryStateEnum::PAUSED + ) +} + +/// States from which a managed container can be brought back to running by +/// `start_container`. Skip `Restarting` (already coming up), `Removing`, +/// `Dead` (terminal), `Paused` (needs `unpause`, not `start`), and +/// `Running` (nothing to do). +fn container_state_needs_resume(state: ContainerSummaryStateEnum) -> bool { + matches!( + state, + ContainerSummaryStateEnum::EXITED | ContainerSummaryStateEnum::CREATED + ) +} + +fn docker_stop_timeout_secs(timeout_secs: u32) -> i32 { + i32::try_from(timeout_secs).unwrap_or(i32::MAX) +} + +fn emit_snapshot_diff( + events: &broadcast::Sender, + previous: &HashMap, + current: &HashMap, +) { + for (sandbox_id, sandbox) in current { + if previous.get(sandbox_id) == Some(sandbox) { + continue; + } + let _ = events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox.clone()), + }, + )), + }); + } + + for sandbox_id in previous.keys() { + if current.contains_key(sandbox_id) { + continue; + } + let _ = events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { + sandbox_id: sandbox_id.clone(), + }, + )), + }); + } +} + +fn label_filters(values: impl IntoIterator) -> HashMap> { + HashMap::from([("label".to_string(), values.into_iter().collect())]) +} + +fn managed_container_label_filters( + sandbox_namespace: &str, + extra_values: impl IntoIterator, +) -> HashMap> { + let mut values = vec![ + format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), + ]; + values.extend(extra_values); + label_filters(values) +} + +/// Maximum Docker container name length. Docker's own limit is 253 bytes, but +/// we cap at a conservative 200 to leave headroom for tooling that truncates +/// names further. +const MAX_CONTAINER_NAME_LEN: usize = 200; +const CONTAINER_NAME_PREFIX: &str = "openshell-"; + +fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String { + let id_suffix = sanitize_docker_name(&sandbox.id); + let workspace = sanitize_docker_name(&sandbox.workspace); + let name = sanitize_docker_name(&sandbox.name); + + // Format: openshell-{workspace}--{name}-{id} + // The workspace and id are never truncated — they ensure uniqueness. + // Only the sandbox name portion is truncated when the total exceeds + // MAX_CONTAINER_NAME_LEN. + + if name.is_empty() { + let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); + if base.len() > MAX_CONTAINER_NAME_LEN { + base.truncate(MAX_CONTAINER_NAME_LEN); + } + return trim_container_name_tail(base); + } + + // Reserve space for fixed parts: prefix + workspace + "--" + "-" + id + let reserved = CONTAINER_NAME_PREFIX.len() + workspace.len() + 2 + 1 + id_suffix.len(); + if reserved >= MAX_CONTAINER_NAME_LEN { + let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); + base.truncate(MAX_CONTAINER_NAME_LEN); + return trim_container_name_tail(base); + } + + let name_budget = MAX_CONTAINER_NAME_LEN - reserved; + let truncated_name = if name.len() > name_budget { + trim_container_name_tail(name[..name_budget].to_string()) + } else { + name + }; + format!("{CONTAINER_NAME_PREFIX}{workspace}--{truncated_name}-{id_suffix}") +} + +/// Docker container names may not end with `-`, `.`, or `_`. Truncation can +/// leave one of those trailing, so strip them before returning. +fn trim_container_name_tail(mut value: String) -> String { + while value + .chars() + .last() + .is_some_and(|ch| matches!(ch, '-' | '.' | '_')) + { + value.pop(); + } + value +} + +fn sanitize_docker_name(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '-') { + ch.to_ascii_lowercase() + } else { + '-' + } + }) + .collect::() + .trim_matches('-') + .to_string() +} + +fn normalize_docker_arch(arch: &str) -> String { + match arch { + "x86_64" => "amd64".to_string(), + "aarch64" => "arm64".to_string(), + other => other.to_ascii_lowercase(), + } +} + +#[derive(Debug, Eq, PartialEq)] +enum SupervisorBinSource { + Binary(PathBuf), + Image(String), +} + +fn resolve_supervisor_bin_source( + docker_config: &DockerComputeConfig, + current_exe: Option<&Path>, + target_candidates: &[PathBuf], +) -> CoreResult { + // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. + if let Some(path) = docker_config.supervisor_bin.clone() { + let path = canonicalize_existing_file(&path, "docker supervisor binary")?; + validate_linux_elf_binary(&path)?; + return Ok(SupervisorBinSource::Binary(path)); + } + + // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. + // A configured image should be the source of truth even when a local + // developer build is present under target/. + if let Some(image) = docker_config.supervisor_image.clone() { + return Ok(SupervisorBinSource::Image(image)); + } + + // Tier 3: sibling `openshell-sandbox` next to the running gateway + // (release artifact layout). Linux-only because the sibling must be a + // Linux ELF to bind-mount into a Linux container. + if cfg!(target_os = "linux") + && let Some(current_exe) = current_exe + && let Some(parent) = current_exe.parent() + { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; + if validate_linux_elf_binary(&path).is_ok() { + return Ok(SupervisorBinSource::Binary(path)); + } + } + } + + // Tier 4: local cargo target build (developer workflow). Preferred + // over the default registry image when available because it matches + // whatever the developer just built. + for candidate in target_candidates { + if candidate.is_file() { + let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; + if validate_linux_elf_binary(&path).is_ok() { + return Ok(SupervisorBinSource::Binary(path)); + } + } + } + + // Tier 5: pull the release-matched default supervisor image and extract + // the binary to a host-side cache keyed by image content digest. + Ok(SupervisorBinSource::Image( + openshell_core::config::default_supervisor_image(), + )) +} + +pub(crate) async fn resolve_supervisor_bin( + docker: &Docker, + docker_config: &DockerComputeConfig, + daemon_arch: &str, +) -> CoreResult { + let current_exe = + if cfg!(target_os = "linux") + && docker_config.supervisor_bin.is_none() + && docker_config.supervisor_image.is_none() + { + Some(std::env::current_exe().map_err(|err| { + Error::config(format!("failed to resolve current executable: {err}")) + })?) + } else { + None + }; + let target_candidates = linux_supervisor_candidates(daemon_arch); + + match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? + { + SupervisorBinSource::Binary(path) => Ok(path), + SupervisorBinSource::Image(image) => { + extract_supervisor_bin_from_image(docker, &image).await + } + } +} + +fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { + match daemon_arch { + "arm64" => vec![PathBuf::from( + "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", + )], + "amd64" => vec![PathBuf::from( + "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", + )], + _ => Vec::new(), + } +} + +/// Pull the supervisor image (if not already local), extract +/// `/openshell-sandbox` to a host cache keyed by the image's content +/// digest, and return the cache path. +/// +/// The extraction is atomic: the binary is written to a sibling temp file +/// inside the digest-keyed directory and renamed into place, so concurrent +/// gateway starts don't observe a partial file. +async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { + let refresh_attempted = if supervisor_image_should_refresh(image) { + info!(image = image, "Refreshing mutable docker supervisor image"); + match pull_supervisor_image(docker, image).await { + Ok(()) => true, + Err(err) => { + warn!( + image = image, + error = %err, + "failed to refresh mutable docker supervisor image; falling back to local image if present", + ); + true + } + } + } else { + false + }; + + // Inspect first to see if the image is already present; only pull on miss. + let inspect = match docker.inspect_image(image).await { + Ok(inspect) => inspect, + Err(err) if is_not_found_error(&err) && !refresh_attempted => { + info!(image = image, "Pulling docker supervisor image"); + pull_supervisor_image(docker, image).await?; + docker.inspect_image(image).await.map_err(|err| { + Error::config(format!( + "failed to inspect docker supervisor image '{image}' after pull: {err}", + )) + })? + } + Err(err) if is_not_found_error(&err) => { + return Err(Error::config(format!( + "docker supervisor image '{image}' is not present locally after refresh attempt", + ))); + } + Err(err) => { + return Err(Error::config(format!( + "failed to inspect docker supervisor image '{image}': {err}", + ))); + } + }; + + let digest = inspect.id.clone().ok_or_else(|| { + Error::config(format!( + "docker supervisor image '{image}' inspect response has no Id", + )) + })?; + + let cache_path = supervisor_cache_path(&digest)?; + if cache_path.is_file() { + validate_linux_elf_binary(&cache_path)?; + return Ok(cache_path); + } + + let cache_dir = cache_path.parent().ok_or_else(|| { + Error::config(format!( + "docker supervisor cache path '{}' has no parent directory", + cache_path.display(), + )) + })?; + std::fs::create_dir_all(cache_dir).map_err(|err| { + Error::config(format!( + "failed to create docker supervisor cache dir '{}': {err}", + cache_dir.display(), + )) + })?; + + info!( + image = image, + digest = digest, + cache_path = %cache_path.display(), + "Extracting supervisor binary from image to host cache", + ); + + let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; + write_cache_binary_atomic(&cache_path, &binary_bytes)?; + validate_linux_elf_binary(&cache_path)?; + Ok(cache_path) +} + +async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { + let mut stream = docker.create_image( + Some(CreateImageOptions { + from_image: Some(image.to_string()), + ..Default::default() + }), + None, + None, + ); + while let Some(result) = stream.next().await { + result.map_err(|err| { + Error::config(format!( + "failed to pull docker supervisor image '{image}': {err}", + )) + })?; + } + Ok(()) +} + +/// Create a short-lived container from `image`, stream out the supervisor +/// binary as a tar archive, and return the untarred file bytes. The +/// container is always removed, even on error paths. +async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + let container_name = temp_extract_container_name(); + docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + ContainerCreateBody { + image: Some(image.to_string()), + entrypoint: Some(vec![SUPERVISOR_IMAGE_BINARY_PATH.to_string()]), + cmd: Some(Vec::new()), + ..Default::default() + }, + ) + .await + .map_err(|err| { + Error::config(format!( + "failed to create extractor container from '{image}': {err}", + )) + })?; + + // Always tear down the extractor container, even if extraction fails. + let result = download_binary_from_container(docker, &container_name).await; + if let Err(remove_err) = docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + warn!( + container = container_name, + error = %remove_err, + "Failed to remove supervisor extractor container", + ); + } + result +} + +async fn download_binary_from_container( + docker: &Docker, + container_name: &str, +) -> CoreResult> { + let options = DownloadFromContainerOptionsBuilder::default() + .path(SUPERVISOR_IMAGE_BINARY_PATH) + .build(); + let mut stream = docker.download_from_container(container_name, Some(options)); + + let mut tar_bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk: Bytes = chunk.map_err(|err| { + Error::config(format!( + "failed to read supervisor binary stream from '{container_name}': {err}", + )) + })?; + tar_bytes.extend_from_slice(&chunk); + } + + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) +} + +/// Extract the payload of the first regular-file entry in a tar archive. +/// Docker's `/containers//archive` endpoint returns a single-file tar +/// when `path` points to a file, so we only need the first entry. +fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { + let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); + let mut entries = archive + .entries() + .map_err(|err| format!("open tar archive: {err}"))?; + let mut entry = entries + .next() + .ok_or_else(|| "tar archive was empty".to_string())? + .map_err(|err| format!("read tar entry: {err}"))?; + let mut bytes = Vec::new(); + entry + .read_to_end(&mut bytes) + .map_err(|err| format!("read tar entry payload: {err}"))?; + Ok(bytes) +} + +fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { + let dir = final_path.parent().ok_or_else(|| { + Error::config(format!( + "docker supervisor cache path '{}' has no parent directory", + final_path.display(), + )) + })?; + let mut temp = tempfile::Builder::new() + .prefix(".openshell-sandbox-") + .tempfile_in(dir) + .map_err(|err| { + Error::config(format!( + "failed to create temp file for supervisor binary in '{}': {err}", + dir.display(), + )) + })?; + std::io::Write::write_all(&mut temp, bytes).map_err(|err| { + Error::config(format!( + "failed to write supervisor binary to temp file: {err}", + )) + })?; + temp.as_file().sync_all().map_err(|err| { + Error::config(format!("failed to sync supervisor binary temp file: {err}")) + })?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( + |err| { + Error::config(format!( + "failed to chmod supervisor binary temp file: {err}", + )) + }, + )?; + } + + temp.persist(final_path).map_err(|err| { + Error::config(format!( + "failed to rename supervisor binary into '{}': {}", + final_path.display(), + err.error, + )) + })?; + Ok(()) +} + +/// Cache path for an extracted supervisor binary, keyed by the image's +/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed +/// directory keeps stale extractions from earlier releases isolated so they +/// can be GC'd without affecting the active binary. +fn supervisor_cache_path(digest: &str) -> CoreResult { + let base = openshell_core::paths::xdg_data_dir() + .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; + Ok(supervisor_cache_path_with_base(&base, digest)) +} + +fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { + let sanitized = digest.replace(':', "-"); + base.join("openshell") + .join("docker-supervisor") + .join(sanitized) + .join("openshell-sandbox") +} + +fn temp_extract_container_name() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let pid = std::process::id(); + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + format!("openshell-supervisor-extract-{pid}-{seq}") +} + +fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { + if !path.is_file() { + return Err(Error::config(format!( + "{description} '{}' does not exist or is not a file", + path.display() + ))); + } + std::fs::canonicalize(path).map_err(|err| { + Error::config(format!( + "failed to resolve {description} '{}': {err}", + path.display() + )) + }) +} + +pub(crate) fn validate_linux_elf_binary(path: &Path) -> CoreResult<()> { + let mut file = std::fs::File::open(path).map_err(|err| { + Error::config(format!( + "failed to open docker supervisor binary '{}': {err}", + path.display() + )) + })?; + let mut magic = [0_u8; 4]; + file.read_exact(&mut magic).map_err(|err| { + Error::config(format!( + "failed to read docker supervisor binary '{}': {err}", + path.display() + )) + })?; + if magic != [0x7f, b'E', b'L', b'F'] { + return Err(Error::config(format!( + "docker supervisor binary '{}' must be a Linux ELF executable", + path.display() + ))); + } + Ok(()) +} + +fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { + docker_config.guest_tls_ca.is_some() + && docker_config.guest_tls_cert.is_some() + && docker_config.guest_tls_key.is_some() +} + +pub(crate) fn docker_guest_tls_paths( + docker_config: &DockerComputeConfig, +) -> CoreResult> { + let tls_flags_provided = docker_config.guest_tls_ca.is_some() + || docker_config.guest_tls_cert.is_some() + || docker_config.guest_tls_key.is_some(); + + if !docker_config.grpc_endpoint.starts_with("https://") { + if tls_flags_provided { + return Err(Error::config(format!( + "guest_tls_ca/guest_tls_cert/guest_tls_key were provided but grpc_endpoint is '{}'; TLS materials require an https:// endpoint", + docker_config.grpc_endpoint, + ))); + } + return Ok(None); + } + + let provided = [ + docker_config.guest_tls_ca.as_ref(), + docker_config.guest_tls_cert.as_ref(), + docker_config.guest_tls_key.as_ref(), + ]; + if provided.iter().all(Option::is_none) { + return Err(Error::config( + "docker compute driver requires guest_tls_ca, guest_tls_cert, and guest_tls_key when grpc_endpoint uses https://", + )); + } + + let Some(ca) = docker_config.guest_tls_ca.clone() else { + return Err(Error::config( + "guest_tls_ca is required when Docker sandbox TLS materials are configured", + )); + }; + let Some(cert) = docker_config.guest_tls_cert.clone() else { + return Err(Error::config( + "guest_tls_cert is required when Docker sandbox TLS materials are configured", + )); + }; + let Some(key) = docker_config.guest_tls_key.clone() else { + return Err(Error::config( + "guest_tls_key is required when Docker sandbox TLS materials are configured", + )); + }; + + Ok(Some(DockerGuestTlsPaths { + ca: canonicalize_existing_file(&ca, "docker TLS CA certificate")?, + cert: canonicalize_existing_file(&cert, "docker TLS client certificate")?, + key: canonicalize_existing_file(&key, "docker TLS client private key")?, + })) +} + +fn is_not_found_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 404, + .. + } + ) +} + +fn is_conflict_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 409, + .. + } + ) +} + +fn is_not_modified_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 304, + .. + } + ) +} + +fn create_status_from_docker_error(operation: &str, err: BollardError) -> Status { + if matches!( + err, + BollardError::DockerResponseServerError { + status_code: 409, + .. + } + ) { + Status::already_exists("sandbox already exists") + } else { + internal_status(operation, err) + } +} + +fn internal_status(operation: &str, err: BollardError) -> Status { + Status::internal(format!("{operation} failed: {err}")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-driver-docker/src/windows.rs b/crates/openshell-driver-docker/src/windows.rs new file mode 100644 index 0000000000..87313dc9c2 --- /dev/null +++ b/crates/openshell-driver-docker/src/windows.rs @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_core::config::DEFAULT_DOCKER_NETWORK_NAME; +use std::path::PathBuf; + +const DEFAULT_DOCKER_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/supervisor"; +const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; + +#[must_use] +pub fn default_docker_supervisor_image() -> String { + format!( + "{DEFAULT_DOCKER_SUPERVISOR_IMAGE_REPO}:{}", + default_docker_supervisor_image_tag() + ) +} + +fn default_docker_supervisor_image_tag() -> String { + let tag = option_env!("OPENSHELL_IMAGE_TAG") + .filter(|tag| !tag.is_empty()) + .or_else(|| option_env!("IMAGE_TAG").filter(|tag| !tag.is_empty())) + .unwrap_or_else(|| { + if env!("CARGO_PKG_VERSION").is_empty() || env!("CARGO_PKG_VERSION") == "0.0.0" { + "dev" + } else { + env!("CARGO_PKG_VERSION") + } + }); + + tag.replace('+', "-") +} + +/// Gateway-local configuration for the Docker compute driver. +/// +/// Windows builds keep this type so existing config files continue to parse, +/// but Docker runtime support is intentionally unavailable on Windows. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DockerComputeConfig { + pub socket_path: Option, + pub default_image: String, + pub image_pull_policy: String, + pub sandbox_namespace: String, + pub grpc_endpoint: String, + pub supervisor_bin: Option, + pub supervisor_image: Option, + pub guest_tls_ca: Option, + pub guest_tls_cert: Option, + pub guest_tls_key: Option, + pub network_name: String, + pub host_gateway_ip: String, + pub ssh_socket_path: String, + pub sandbox_pids_limit: i64, + pub enable_bind_mounts: bool, +} + +impl Default for DockerComputeConfig { + fn default() -> Self { + Self { + socket_path: None, + default_image: openshell_core::image::default_sandbox_image(), + image_pull_policy: String::new(), + sandbox_namespace: "default".to_string(), + grpc_endpoint: String::new(), + supervisor_bin: None, + supervisor_image: None, + guest_tls_ca: None, + guest_tls_cert: None, + guest_tls_key: None, + network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), + host_gateway_ip: String::new(), + ssh_socket_path: "/run/openshell/ssh.sock".to_string(), + sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + enable_bind_mounts: false, + } + } +} diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 2c02f864ab..e72ac4f974 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -17,7 +17,9 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-policy = { path = "../openshell-policy" } +serde = { workspace = true } +[target.'cfg(not(target_os = "windows"))'.dependencies] tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } prost = { workspace = true } @@ -27,7 +29,6 @@ tokio-stream = { workspace = true } kube = { workspace = true } kube-runtime = { workspace = true } k8s-openapi = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } clap = { workspace = true } tracing = { workspace = true } @@ -37,6 +38,7 @@ miette = { workspace = true } [dev-dependencies] temp-env = "0.3" +serde_json = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 7c56c8de5b..0bb796a37b 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 pub mod config; + +#[cfg(not(target_os = "windows"))] pub mod driver; +#[cfg(not(target_os = "windows"))] pub mod grpc; pub use config::{ @@ -10,5 +13,8 @@ pub use config::{ DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, SupervisorSideloadMethod, SupervisorTopology, }; + +#[cfg(not(target_os = "windows"))] pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; +#[cfg(not(target_os = "windows"))] pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index c733b8a45b..c151df22c3 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -1,181 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::{ArgAction, Parser}; -use miette::{IntoDiagnostic, Result}; -use std::net::SocketAddr; -use tracing::info; -use tracing_subscriber::EnvFilter; +#[cfg(not(target_os = "windows"))] +include!("main_unix.rs"); -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, -}; - -#[derive(Parser, Debug)] -#[command(name = "openshell-driver-kubernetes")] -#[command(version = VERSION)] -struct Args { - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_BIND", - default_value = "127.0.0.1:50061" - )] - bind_address: SocketAddr, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] - sandbox_namespace: String, - - #[arg( - long, - env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", - default_value = DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME - )] - sandbox_service_account: String, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_IMAGE_PULL_SECRETS", - value_delimiter = ',' - )] - sandbox_image_pull_secrets: Vec, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", - default_value = "/run/openshell/ssh.sock" - )] - sandbox_ssh_socket_path: String, - - #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] - client_tls_secret_name: Option, - - #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: Option, - - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, - - #[arg( - long, - env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", - default_value = "image-volume" - )] - supervisor_sideload_method: SupervisorSideloadMethod, - - #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] - topology: SupervisorTopology, - - #[arg( - long = "sidecar-proxy-uid", - alias = "proxy-uid", - env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", - default_value_t = DEFAULT_PROXY_UID - )] - sidecar_proxy_uid: u32, - - #[arg( - long = "sidecar-process-binary-aware-network-policy", - env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", - default_value_t = true, - action = ArgAction::Set - )] - sidecar_process_binary_aware_network_policy: bool, - - #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] - enable_user_namespaces: bool, - - #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] - app_armor_profile: Option, - - /// Lifetime (seconds) of the projected `ServiceAccount` token - /// kubelet writes into each sandbox pod for the `IssueSandboxToken` - /// bootstrap exchange. Kubelet enforces a minimum of 600s; the - /// gateway clamps values outside `[600, 86400]`. Default 3600. - #[arg(long, env = "OPENSHELL_K8S_SA_TOKEN_TTL_SECS", default_value_t = 3600)] - sa_token_ttl_secs: i64, - - #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] - provider_spiffe_workload_api_socket_path: Option, - - #[arg(long, env = "OPENSHELL_K8S_SANDBOX_UID")] - sandbox_uid: Option, - - #[arg(long, env = "OPENSHELL_K8S_SANDBOX_GID")] - sandbox_gid: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { - namespace: args.sandbox_namespace, - service_account_name: args.sandbox_service_account, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), - image_pull_secrets: args.sandbox_image_pull_secrets, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, - }, - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - ssh_socket_path: args.sandbox_ssh_socket_path, - client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), - host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), - enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, - workspace_default_storage_size: std::env::var( - "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", - ) - .unwrap_or_else(|_| { - openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() - }), - default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") - .unwrap_or_default(), - sa_token_ttl_secs: args.sa_token_ttl_secs, - provider_spiffe_workload_api_socket_path: args - .provider_spiffe_workload_api_socket_path - .unwrap_or_default(), - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) - .await - .into_diagnostic()?; - - info!(address = %args.bind_address, "Starting Kubernetes compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve(args.bind_address) - .await - .into_diagnostic() +#[cfg(target_os = "windows")] +fn main() { + eprintln!("openshell-driver-kubernetes is unsupported on Windows"); + std::process::exit(1); } diff --git a/crates/openshell-driver-kubernetes/src/main_unix.rs b/crates/openshell-driver-kubernetes/src/main_unix.rs new file mode 100644 index 0000000000..c733b8a45b --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/main_unix.rs @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use clap::{ArgAction, Parser}; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_kubernetes::{ + AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, + SupervisorSideloadMethod, SupervisorTopology, +}; + +#[derive(Parser, Debug)] +#[command(name = "openshell-driver-kubernetes")] +#[command(version = VERSION)] +struct Args { + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] + sandbox_namespace: String, + + #[arg( + long, + env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", + default_value = DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME + )] + sandbox_service_account: String, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] + sandbox_image_pull_policy: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_IMAGE_PULL_SECRETS", + value_delimiter = ',' + )] + sandbox_image_pull_secrets: Vec, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = "/run/openshell/ssh.sock" + )] + sandbox_ssh_socket_path: String, + + #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] + client_tls_secret_name: Option, + + #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] + host_gateway_ip: Option, + + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + supervisor_image: Option, + + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] + supervisor_image_pull_policy: Option, + + #[arg( + long, + env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", + default_value = "image-volume" + )] + supervisor_sideload_method: SupervisorSideloadMethod, + + #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] + topology: SupervisorTopology, + + #[arg( + long = "sidecar-proxy-uid", + alias = "proxy-uid", + env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", + default_value_t = DEFAULT_PROXY_UID + )] + sidecar_proxy_uid: u32, + + #[arg( + long = "sidecar-process-binary-aware-network-policy", + env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", + default_value_t = true, + action = ArgAction::Set + )] + sidecar_process_binary_aware_network_policy: bool, + + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] + enable_user_namespaces: bool, + + #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] + app_armor_profile: Option, + + /// Lifetime (seconds) of the projected `ServiceAccount` token + /// kubelet writes into each sandbox pod for the `IssueSandboxToken` + /// bootstrap exchange. Kubelet enforces a minimum of 600s; the + /// gateway clamps values outside `[600, 86400]`. Default 3600. + #[arg(long, env = "OPENSHELL_K8S_SA_TOKEN_TTL_SECS", default_value_t = 3600)] + sa_token_ttl_secs: i64, + + #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] + provider_spiffe_workload_api_socket_path: Option, + + #[arg(long, env = "OPENSHELL_K8S_SANDBOX_UID")] + sandbox_uid: Option, + + #[arg(long, env = "OPENSHELL_K8S_SANDBOX_GID")] + sandbox_gid: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { + namespace: args.sandbox_namespace, + service_account_name: args.sandbox_service_account, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_secrets: args.sandbox_image_pull_secrets, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_sideload_method: args.supervisor_sideload_method, + topology: args.topology, + sidecar: KubernetesSidecarConfig { + proxy_uid: args.sidecar_proxy_uid, + process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, + }, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + ssh_socket_path: args.sandbox_ssh_socket_path, + client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), + host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), + enable_user_namespaces: args.enable_user_namespaces, + app_armor_profile: args.app_armor_profile, + workspace_default_storage_size: std::env::var( + "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", + ) + .unwrap_or_else(|_| { + openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() + }), + default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") + .unwrap_or_default(), + sa_token_ttl_secs: args.sa_token_ttl_secs, + provider_spiffe_workload_api_socket_path: args + .provider_spiffe_workload_api_socket_path + .unwrap_or_default(), + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, + }) + .await + .into_diagnostic()?; + + info!(address = %args.bind_address, "Starting Kubernetes compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) + .serve(args.bind_address) + .await + .into_diagnostic() +} diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index ed798c0ab2..90a5e6d761 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -14,7 +14,10 @@ repository.workspace = true name = "openshell-driver-podman" path = "src/main.rs" -[dependencies] +[target.'cfg(target_os = "windows")'.dependencies] +serde = { workspace = true } + +[target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } tokio = { workspace = true } diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index 5847a10ea6..b76c9e0fb3 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -1,15 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub(crate) mod client; -pub mod config; -pub(crate) mod container; -pub mod driver; -pub mod grpc; -#[cfg(test)] -pub(crate) mod test_utils; -pub(crate) mod watcher; +#[cfg(target_os = "windows")] +mod windows; -pub use config::PodmanComputeConfig; -pub use driver::PodmanComputeDriver; -pub use grpc::ComputeDriverService; +#[cfg(target_os = "windows")] +pub use windows::PodmanComputeConfig; + +#[cfg(not(target_os = "windows"))] +include!("lib_unix.rs"); diff --git a/crates/openshell-driver-podman/src/lib_unix.rs b/crates/openshell-driver-podman/src/lib_unix.rs new file mode 100644 index 0000000000..5847a10ea6 --- /dev/null +++ b/crates/openshell-driver-podman/src/lib_unix.rs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) mod client; +pub mod config; +pub(crate) mod container; +pub mod driver; +pub mod grpc; +#[cfg(test)] +pub(crate) mod test_utils; +pub(crate) mod watcher; + +pub use config::PodmanComputeConfig; +pub use driver::PodmanComputeDriver; +pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 4a38643f38..42060fd75c 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -1,185 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use std::net::SocketAddr; -use std::path::PathBuf; -use tracing::info; -use tracing_subscriber::EnvFilter; - -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; -use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; - -#[derive(Parser)] -#[command(name = "openshell-driver-podman")] -#[command(version = VERSION)] -struct Args { - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_BIND", - default_value = "127.0.0.1:50061" - )] - bind_address: SocketAddr, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - /// Path to the Podman API Unix socket. - #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] - podman_socket: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing - )] - sandbox_image_pull_policy: ImagePullPolicy, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - /// Port the gateway server is listening on. - /// - /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint - /// that sandbox containers dial back to. - #[arg( - long, - env = "OPENSHELL_GATEWAY_PORT", - default_value_t = openshell_core::config::DEFAULT_SERVER_PORT - )] - gateway_port: u16, - - /// Host gateway IP used for sandbox host aliases. - /// - /// Empty uses Podman's `host-gateway` resolver. - #[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", - default_value = "/run/openshell/ssh.sock" - )] - sandbox_ssh_socket_path: String, - - /// Podman bridge network name. - #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] - network_name: String, - - /// Container stop timeout in seconds (SIGTERM → SIGKILL). - #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] - stop_timeout: u32, - - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit - /// Podman's runtime/default PID limit. - #[arg( - long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT - )] - sandbox_pids_limit: i64, - - /// OCI image containing the openshell-sandbox supervisor binary. - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: Option, - - /// Host path to the CA certificate for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] - podman_tls_ca: Option, - - /// Host path to the client certificate for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")] - podman_tls_cert: Option, - - /// Host path to the client private key for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] - podman_tls_key: Option, - - /// Corporate forward proxy URL for the supervisor's upstream TLS dials, - /// in explicit `http://host:port` form (scheme and port required). - /// Credentials must not be embedded in the URL; use - /// `--sandbox-proxy-auth-file` instead. - #[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")] - sandbox_https_proxy: Option, - - /// Comma-separated `NO_PROXY` list injected alongside the proxy URL. - #[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")] - sandbox_no_proxy: Option, - - /// Path to a file containing the corporate proxy credentials as - /// `user:pass`. Delivered to the supervisor through a root-only secret - /// mount so the credentials never appear in config or container metadata. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")] - sandbox_proxy_auth_file: Option, - - /// Explicit acknowledgement (`true`) that the proxy credential is sent - /// as cleartext Basic auth over the plain-TCP connection to the http:// - /// proxy. Required when `--sandbox-proxy-auth-file` is set. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")] - sandbox_proxy_auth_allow_insecure: Option, - - /// Send the destination hostname in CONNECT requests to the corporate - /// proxy instead of a validated IP. Only for proxies whose ACLs filter - /// on hostnames: the proxy then resolves the name itself, so sandbox - /// SSRF/`allowed_ips` validation no longer binds the connection. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] - sandbox_proxy_connect_by_hostname: Option, +#[cfg(target_os = "windows")] +fn main() { + eprintln!("openshell-driver-podman is unsupported on Windows"); + std::process::exit(1); } -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let driver = PodmanComputeDriver::new(PodmanComputeConfig { - socket_path: args.podman_socket, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy, - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - gateway_port: args.gateway_port, - host_gateway_ip: args - .host_gateway_ip - .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, - network_name: args.network_name, - stop_timeout_secs: args.stop_timeout, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - guest_tls_ca: args.podman_tls_ca, - guest_tls_cert: args.podman_tls_cert, - guest_tls_key: args.podman_tls_key, - sandbox_pids_limit: args.sandbox_pids_limit, - https_proxy: args.sandbox_https_proxy, - no_proxy: args.sandbox_no_proxy, - proxy_auth_file: args.sandbox_proxy_auth_file, - proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, - ..PodmanComputeConfig::default() - }) - .await - .into_diagnostic()?; - - info!(address = %args.bind_address, "Starting Podman compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async { - tokio::signal::ctrl_c().await.ok(); - info!("Received shutdown signal, draining in-flight requests"); - }) - .await - .into_diagnostic() -} +#[cfg(not(target_os = "windows"))] +include!("main_unix.rs"); diff --git a/crates/openshell-driver-podman/src/main_unix.rs b/crates/openshell-driver-podman/src/main_unix.rs new file mode 100644 index 0000000000..4a38643f38 --- /dev/null +++ b/crates/openshell-driver-podman/src/main_unix.rs @@ -0,0 +1,185 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use std::path::PathBuf; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_podman::config::{ + DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, + ImagePullPolicy, +}; +use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; + +#[derive(Parser)] +#[command(name = "openshell-driver-podman")] +#[command(version = VERSION)] +struct Args { + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + /// Path to the Podman API Unix socket. + #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] + podman_socket: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", + default_value_t = ImagePullPolicy::Missing + )] + sandbox_image_pull_policy: ImagePullPolicy, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + /// Port the gateway server is listening on. + /// + /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint + /// that sandbox containers dial back to. + #[arg( + long, + env = "OPENSHELL_GATEWAY_PORT", + default_value_t = openshell_core::config::DEFAULT_SERVER_PORT + )] + gateway_port: u16, + + /// Host gateway IP used for sandbox host aliases. + /// + /// Empty uses Podman's `host-gateway` resolver. + #[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")] + host_gateway_ip: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = "/run/openshell/ssh.sock" + )] + sandbox_ssh_socket_path: String, + + /// Podman bridge network name. + #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] + network_name: String, + + /// Container stop timeout in seconds (SIGTERM → SIGKILL). + #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] + stop_timeout: u32, + + /// Container cgroup PID limit for sandbox containers. Set 0 to inherit + /// Podman's runtime/default PID limit. + #[arg( + long, + env = "OPENSHELL_SANDBOX_PIDS_LIMIT", + default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + )] + sandbox_pids_limit: i64, + + /// OCI image containing the openshell-sandbox supervisor binary. + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + supervisor_image: Option, + + /// Host path to the CA certificate for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] + podman_tls_ca: Option, + + /// Host path to the client certificate for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")] + podman_tls_cert: Option, + + /// Host path to the client private key for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] + podman_tls_key: Option, + + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, + /// in explicit `http://host:port` form (scheme and port required). + /// Credentials must not be embedded in the URL; use + /// `--sandbox-proxy-auth-file` instead. + #[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")] + sandbox_https_proxy: Option, + + /// Comma-separated `NO_PROXY` list injected alongside the proxy URL. + #[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")] + sandbox_no_proxy: Option, + + /// Path to a file containing the corporate proxy credentials as + /// `user:pass`. Delivered to the supervisor through a root-only secret + /// mount so the credentials never appear in config or container metadata. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")] + sandbox_proxy_auth_file: Option, + + /// Explicit acknowledgement (`true`) that the proxy credential is sent + /// as cleartext Basic auth over the plain-TCP connection to the http:// + /// proxy. Required when `--sandbox-proxy-auth-file` is set. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")] + sandbox_proxy_auth_allow_insecure: Option, + + /// Send the destination hostname in CONNECT requests to the corporate + /// proxy instead of a validated IP. Only for proxies whose ACLs filter + /// on hostnames: the proxy then resolves the name itself, so sandbox + /// SSRF/`allowed_ips` validation no longer binds the connection. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] + sandbox_proxy_connect_by_hostname: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let driver = PodmanComputeDriver::new(PodmanComputeConfig { + socket_path: args.podman_socket, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + gateway_port: args.gateway_port, + host_gateway_ip: args + .host_gateway_ip + .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), + sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + network_name: args.network_name, + stop_timeout_secs: args.stop_timeout, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + guest_tls_ca: args.podman_tls_ca, + guest_tls_cert: args.podman_tls_cert, + guest_tls_key: args.podman_tls_key, + sandbox_pids_limit: args.sandbox_pids_limit, + https_proxy: args.sandbox_https_proxy, + no_proxy: args.sandbox_no_proxy, + proxy_auth_file: args.sandbox_proxy_auth_file, + proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, + ..PodmanComputeConfig::default() + }) + .await + .into_diagnostic()?; + + info!(address = %args.bind_address, "Starting Podman compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) + .serve_with_shutdown(args.bind_address, async { + tokio::signal::ctrl_c().await.ok(); + info!("Received shutdown signal, draining in-flight requests"); + }) + .await + .into_diagnostic() +} diff --git a/crates/openshell-driver-podman/src/windows.rs b/crates/openshell-driver-podman/src/windows.rs new file mode 100644 index 0000000000..8bfe03136a --- /dev/null +++ b/crates/openshell-driver-podman/src/windows.rs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +const DEFAULT_SANDBOX_IMAGE: &str = "ghcr.io/nvidia/openshell/sandbox:latest"; +const DEFAULT_SUPERVISOR_IMAGE: &str = "ghcr.io/nvidia/openshell/supervisor:latest"; +const DEFAULT_SERVER_PORT: u16 = 8080; +const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; +const DEFAULT_NETWORK_NAME: &str = "openshell"; +const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; +const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ImagePullPolicy { + Always, + #[default] + Missing, + Never, + Newer, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct PodmanComputeConfig { + pub socket_path: Option, + pub default_image: String, + pub image_pull_policy: ImagePullPolicy, + pub grpc_endpoint: String, + pub gateway_port: u16, + pub host_gateway_ip: String, + pub sandbox_ssh_socket_path: String, + pub network_name: String, + pub stop_timeout_secs: u32, + pub supervisor_image: String, + pub guest_tls_ca: Option, + pub guest_tls_cert: Option, + pub guest_tls_key: Option, + pub sandbox_pids_limit: i64, + pub enable_bind_mounts: bool, + pub health_check_interval_secs: u64, +} + +impl Default for PodmanComputeConfig { + fn default() -> Self { + Self { + socket_path: None, + default_image: DEFAULT_SANDBOX_IMAGE.to_string(), + image_pull_policy: ImagePullPolicy::default(), + grpc_endpoint: String::new(), + gateway_port: DEFAULT_SERVER_PORT, + host_gateway_ip: String::new(), + sandbox_ssh_socket_path: String::new(), + network_name: DEFAULT_NETWORK_NAME.to_string(), + stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, + supervisor_image: DEFAULT_SUPERVISOR_IMAGE.to_string(), + guest_tls_ca: None, + guest_tls_cert: None, + guest_tls_key: None, + sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + enable_bind_mounts: false, + health_check_interval_secs: DEFAULT_HEALTH_CHECK_INTERVAL_SECS, + } + } +} diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index cef3e67f88..0adf7196d7 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" name = "openshell-driver-vm" path = "src/main.rs" -[dependencies] +[target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-policy = { path = "../openshell-policy" } openshell-vfio = { path = "../openshell-vfio" } diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index 88e2c3b201..e0d663bc44 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -1,23 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub mod driver; -mod embedded_runtime; -mod ffi; -pub mod gpu; -pub mod lifecycle; -mod nft_ruleset; -pub mod procguard; -mod rootfs; -mod runtime; +#[cfg(target_os = "windows")] +pub fn windows_unsupported() -> &'static str { + "openshell-driver-vm is unsupported on Windows" +} -pub use driver::{VmDriver, VmDriverConfig}; -pub use lifecycle::{ - BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, - LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, - RestoreContext, -}; -pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, - configured_runtime_dir, run_vm, -}; +#[cfg(not(target_os = "windows"))] +include!("lib_unix.rs"); diff --git a/crates/openshell-driver-vm/src/lib_unix.rs b/crates/openshell-driver-vm/src/lib_unix.rs new file mode 100644 index 0000000000..88e2c3b201 --- /dev/null +++ b/crates/openshell-driver-vm/src/lib_unix.rs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub mod driver; +mod embedded_runtime; +mod ffi; +pub mod gpu; +pub mod lifecycle; +mod nft_ruleset; +pub mod procguard; +mod rootfs; +mod runtime; + +pub use driver::{VmDriver, VmDriverConfig}; +pub use lifecycle::{ + BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, + LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, + RestoreContext, +}; +pub use runtime::{ + VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, + configured_runtime_dir, run_vm, +}; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 0ae694effa..c41b94f247 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -1,728 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; -use futures::Stream; -use miette::{IntoDiagnostic, Result}; -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -#[cfg(target_os = "macos")] -use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; -use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; -use std::io; -use std::net::SocketAddr; -use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::net::{UnixListener, UnixStream}; -use tracing::info; -use tracing_subscriber::EnvFilter; - -#[derive(Parser, Debug)] -#[command(name = "openshell-driver-vm")] -#[command(version = VERSION)] -#[allow(clippy::struct_excessive_bools)] -struct Args { - #[arg(long, hide = true, default_value_t = false)] - internal_run_vm: bool, - - #[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")] - vm_root_disk: Option, - - #[arg(long = "vm-overlay-disk", hide = true)] - vm_overlay_disk: Option, - - #[arg(long = "vm-image-disk", hide = true)] - vm_image_disk: Option, - - #[arg(long = "vm-kernel-image", hide = true)] - vm_kernel_image: Option, - - #[arg(long, hide = true)] - vm_exec: Option, - - #[arg(long, hide = true, default_value = "/")] - vm_workdir: String, - - #[arg(long, hide = true)] - vm_env: Vec, - - #[arg(long, hide = true)] - vm_console_output: Option, - - #[arg(long, hide = true, default_value_t = 2)] - vm_vcpus: u8, - - #[arg(long, hide = true, default_value_t = 2048)] - vm_mem_mib: u32, - - #[arg(long, hide = true, default_value_t = 1)] - vm_krun_log_level: u32, - - #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")] - bind_address: Option, - - #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] - bind_socket: Option, - - #[arg(long, hide = true)] - expected_peer_pid: Option, - - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP", - default_value_t = false - )] - allow_unauthenticated_tcp: bool, - - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER", - default_value_t = false - )] - allow_same_uid_peer: bool, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] - default_image: String, - - #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] - bootstrap_image: String, - - #[arg( - long, - env = "OPENSHELL_VM_DRIVER_STATE_DIR", - default_value = "target/openshell-vm-driver" - )] - state_dir: PathBuf, - - #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] - guest_tls_ca: Option, - - #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] - guest_tls_cert: Option, - - #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] - guest_tls_key: Option, - - #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] - krun_log_level: u32, - - #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] - vcpus: u8, - - #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] - mem_mib: u32, - - #[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)] - overlay_disk_mib: u64, - - #[arg(long, env = "OPENSHELL_VM_GPU")] - gpu: bool, - - #[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)] - gpu_mem_mib: u32, - - #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] - gpu_vcpus: u8, - - #[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")] - sandbox_uid: Option, - - #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] - sandbox_gid: Option, - - #[arg(long, hide = true)] - vm_backend: Option, - - #[arg(long, hide = true)] - vm_gpu_bdf: Option, - - #[arg(long, hide = true)] - vm_tap_device: Option, - - #[arg(long, hide = true)] - vm_guest_ip: Option, - - #[arg(long, hide = true)] - vm_host_ip: Option, - - #[arg(long, hide = true)] - vm_vsock_cid: Option, - - #[arg(long, hide = true)] - vm_guest_mac: Option, - - #[arg(long, hide = true)] - vm_gateway_port: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - if args.internal_run_vm { - // We intentionally defer procguard arming until `run_vm()` so - // that the only arm is the one that knows how to clean up - // gvproxy. Racing two watchers against the same parent-death - // event causes the bare arm's `exit(1)` to win, skipping the - // gvproxy cleanup and leaking the helper. The risk window - // before `run_vm` arms procguard is ~a few syscalls long - // (`build_vm_launch_config`, `configured_runtime_dir`), which - // is negligible next to the parent gRPC server's uptime. - maybe_reexec_internal_vm_with_runtime_env()?; - let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; - run_vm(&config).map_err(|err| miette::miette!("{err}"))?; - return Ok(()); - } - - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; - - // Arm procguard so that if the gateway is killed (SIGKILL or crash) - // we also die. Without this the driver is reparented to init and - // keeps its per-sandbox VM launchers alive forever. Launchers have - // their own procguards (armed in `run_vm`) which cascade cleanup of - // gvproxy and the libkrun worker the moment this driver exits. - if let Err(err) = procguard::die_with_parent() { - tracing::warn!( - error = %err, - "procguard arm failed; gateway crashes may orphan this driver" - ); - } - - let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint - .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, - state_dir: args.state_dir.clone(), - launcher_bin: None, - default_image: args.default_image.clone(), - bootstrap_image: args.bootstrap_image.clone(), - log_level: args.log_level.clone(), - krun_log_level: args.krun_log_level, - vcpus: args.vcpus, - mem_mib: args.mem_mib, - overlay_disk_mib: args.overlay_disk_mib, - guest_tls_ca: args.guest_tls_ca.clone(), - guest_tls_cert: args.guest_tls_cert.clone(), - guest_tls_key: args.guest_tls_key.clone(), - gpu_enabled: args.gpu, - gpu_mem_mib: args.gpu_mem_mib, - gpu_vcpus: args.gpu_vcpus, - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) - .await - .map_err(|err| miette::miette!("{err}"))?; - - match listen_mode { - ComputeDriverListenMode::Unix { - socket_path, - expected_peer_pid, - } => { - prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?; - - info!(socket = %socket_path.display(), "Starting vm compute driver"); - let listener = UnixListener::bind(&socket_path).into_diagnostic()?; - restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?; - let result = tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(driver)) - .serve_with_incoming(AuthenticatedUnixIncoming::new(listener, expected_peer_pid)) - .await - .into_diagnostic(); - let _ = std::fs::remove_file(&socket_path); - result - } - ComputeDriverListenMode::Tcp(bind_address) => { - info!(address = %bind_address, "Starting unauthenticated dev vm compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(driver)) - .serve(bind_address) - .await - .into_diagnostic() - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum ComputeDriverListenMode { - Unix { - socket_path: PathBuf, - expected_peer_pid: Option, - }, - Tcp(SocketAddr), -} - -fn compute_driver_listen_mode(args: &Args) -> std::result::Result { - if let Some(socket_path) = args.bind_socket.clone() { - if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer { - return Err( - "--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development" - .to_string(), - ); - } - return Ok(ComputeDriverListenMode::Unix { - socket_path, - expected_peer_pid: args.expected_peer_pid, - }); - } - - if !args.allow_unauthenticated_tcp { - return Err( - "--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development" - .to_string(), - ); - } - - let Some(bind_address) = args.bind_address else { - return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string()); - }; - - Ok(ComputeDriverListenMode::Tcp(bind_address)) -} - -fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> { - let Some(parent) = socket_path.parent() else { - return Err(format!( - "vm compute driver socket path '{}' has no parent directory", - socket_path.display() - )); - }; - let expected_uid = current_euid(); - prepare_private_socket_dir(parent, expected_uid)?; - remove_stale_socket(socket_path, expected_uid) -} - -fn current_euid() -> u32 { - rustix::process::geteuid().as_raw() -} - -fn prepare_private_socket_dir( - socket_dir: &Path, - expected_uid: u32, -) -> std::result::Result<(), String> { - std::fs::create_dir_all(socket_dir) - .map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?; - let metadata = std::fs::symlink_metadata(socket_dir) - .map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?; - let file_type = metadata.file_type(); - if file_type.is_symlink() { - return Err(format!( - "socket dir {} is a symlink; refusing to use it", - socket_dir.display() - )); - } - if !file_type.is_dir() { - return Err(format!( - "socket dir {} is not a directory", - socket_dir.display() - )); - } - if metadata.uid() != expected_uid { - return Err(format!( - "socket dir {} is owned by uid {} but current euid is {}", - socket_dir.display(), - metadata.uid(), - expected_uid - )); - } - std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700)) - .map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display())) -} - -fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> { - let metadata = match std::fs::symlink_metadata(socket_path) { - Ok(metadata) => metadata, - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())), - }; - let file_type = metadata.file_type(); - if file_type.is_symlink() { - return Err(format!( - "socket {} is a symlink; refusing to remove it", - socket_path.display() - )); - } - if metadata.uid() != expected_uid { - return Err(format!( - "socket {} is owned by uid {} but current euid is {}", - socket_path.display(), - metadata.uid(), - expected_uid - )); - } - if !file_type.is_socket() { - return Err(format!( - "socket path {} exists but is not a Unix socket", - socket_path.display() - )); - } - std::fs::remove_file(socket_path) - .map_err(|err| format!("remove stale socket {}: {err}", socket_path.display())) -} - -fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> { - std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) - .map_err(|err| format!("chmod socket {}: {err}", socket_path.display())) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct PeerCredentials { - uid: u32, - pid: Option, -} - -fn peer_credentials(stream: &UnixStream) -> std::result::Result { - let credentials = stream - .peer_cred() - .map_err(|err| format!("read peer credentials: {err}"))?; - Ok(PeerCredentials { - uid: credentials.uid(), - pid: credentials.pid(), - }) -} - -fn authorize_peer_credentials( - peer: PeerCredentials, - driver_uid: u32, - gateway_pid: Option, -) -> std::result::Result<(), String> { - if peer.uid != driver_uid { - return Err(format!( - "peer uid {} does not match current euid {}", - peer.uid, driver_uid - )); - } - let Some(gateway_pid) = gateway_pid else { - return Ok(()); - }; - let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else { - return Err(format!( - "peer pid is unavailable; expected gateway pid {gateway_pid}" - )); - }; - if peer_process_id != gateway_pid { - return Err(format!( - "peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}" - )); - } - Ok(()) -} - -struct AuthenticatedUnixIncoming { - listener: UnixListener, - expected_uid: u32, - expected_peer_pid: Option, -} - -impl AuthenticatedUnixIncoming { - fn new(listener: UnixListener, expected_peer_pid: Option) -> Self { - Self { - listener, - expected_uid: current_euid(), - expected_peer_pid, - } - } -} - -impl Stream for AuthenticatedUnixIncoming { - type Item = io::Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - loop { - match this.listener.poll_accept(cx) { - Poll::Ready(Ok((stream, _addr))) => { - let authorized = peer_credentials(&stream).and_then(|peer| { - authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid) - }); - match authorized { - Ok(()) => return Poll::Ready(Some(Ok(stream))), - Err(err) => { - tracing::warn!( - error = %err, - "rejected vm compute driver UDS client" - ); - } - } - } - Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), - Poll::Pending => return Poll::Pending, - } - } - } +#[cfg(target_os = "windows")] +fn main() { + eprintln!("openshell-driver-vm is unsupported on Windows"); + std::process::exit(1); } -fn build_vm_launch_config(args: &Args) -> std::result::Result { - let root_disk = args - .vm_root_disk - .clone() - .ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?; - let overlay_disk = args - .vm_overlay_disk - .clone() - .ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?; - let image_disk = args.vm_image_disk.clone(); - let exec_path = args - .vm_exec - .clone() - .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; - let console_output = args - .vm_console_output - .clone() - .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; - - let backend = match args.vm_backend.as_deref() { - Some("qemu") => VmBackend::Qemu, - Some("libkrun") | None => VmBackend::Libkrun, - Some(other) => return Err(format!("unknown VM backend: {other}")), - }; - - Ok(VmLaunchConfig { - root_disk, - overlay_disk, - image_disk, - kernel_image: args.vm_kernel_image.clone(), - vcpus: args.vm_vcpus, - mem_mib: args.vm_mem_mib, - exec_path, - args: Vec::new(), - env: args.vm_env.clone(), - workdir: args.vm_workdir.clone(), - log_level: args.vm_krun_log_level, - console_output, - backend, - gpu_bdf: args.vm_gpu_bdf.clone(), - tap_device: args.vm_tap_device.clone(), - guest_ip: args.vm_guest_ip.clone(), - host_ip: args.vm_host_ip.clone(), - vsock_cid: args.vm_vsock_cid, - guest_mac: args.vm_guest_mac.clone(), - gateway_port: args.vm_gateway_port, - }) -} - -#[cfg(target_os = "macos")] -fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { - use std::os::unix::process::CommandExt as _; - - const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; - - if std::env::var_os(REEXEC_ENV).is_some() { - return Ok(()); - } - - let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; - let runtime_str = runtime_dir.to_string_lossy(); - let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") - .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); - if !needs_reexec { - return Ok(()); - } - - let mut dyld_paths = vec![runtime_dir.clone()]; - if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { - dyld_paths.extend(std::env::split_paths(&existing)); - } - let joined = std::env::join_paths(&dyld_paths) - .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; - let exe = std::env::current_exe().into_diagnostic()?; - let args: Vec = std::env::args().skip(1).collect(); - - // Use execvp() so the current process is *replaced* by the re-exec'd - // binary — no wrapper process sits between the compute driver and - // the actually-running VM launcher. That avoids two problems: - // 1. An extra process level that survives SIGKILL of the driver - // (the wrapper was reparenting the re-exec'd child to init). - // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper - // doesn't reach the child unless we hand-roll forwarding. - // After exec, the child inherits our PID and our procguard arming. - let err = std::process::Command::new(exe) - .args(&args) - .env("DYLD_LIBRARY_PATH", &joined) - .env(VM_RUNTIME_DIR_ENV, runtime_dir) - .env(REEXEC_ENV, "1") - .exec(); - // `exec()` only returns on failure. - Err(miette::miette!("failed to re-exec with runtime env: {err}")) -} - -#[cfg(not(target_os = "macos"))] -// Signature must match the macOS variant which can fail. -#[allow(clippy::unnecessary_wraps)] -fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::{ - Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, - compute_driver_listen_mode, - }; - use clap::Parser; - use std::path::PathBuf; - - #[test] - fn peer_authorization_accepts_matching_uid_and_pid() { - authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: Some(42), - }, - 1000, - Some(42), - ) - .unwrap(); - } - - #[test] - fn peer_authorization_rejects_wrong_pid() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: Some(7), - }, - 1000, - Some(42), - ) - .expect_err("wrong pid should be rejected"); - assert!(err.contains("does not match expected gateway pid")); - } - - #[test] - fn peer_authorization_rejects_wrong_uid() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1001, - pid: Some(42), - }, - 1000, - Some(42), - ) - .expect_err("wrong uid should be rejected"); - assert!(err.contains("does not match current euid")); - } - - #[test] - fn peer_authorization_rejects_missing_pid_when_expected() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: None, - }, - 1000, - Some(42), - ) - .expect_err("missing pid should be rejected"); - assert!(err.contains("peer pid is unavailable")); - } - - #[test] - fn peer_authorization_accepts_matching_uid_without_expected_pid() { - authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: None, - }, - 1000, - None, - ) - .unwrap(); - } - - #[test] - fn listen_mode_rejects_default_tcp() { - let args = Args::parse_from(["openshell-driver-vm"]); - let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled"); - assert!(err.contains("--bind-socket is required")); - } - - #[test] - fn listen_mode_rejects_bind_address_without_tcp_opt_in() { - let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); - let err = - compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in"); - assert!(err.contains("--allow-unauthenticated-tcp")); - } - - #[test] - fn listen_mode_requires_bind_address_with_tcp_opt_in() { - let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]); - let err = - compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address"); - assert!(err.contains("--bind-address is required")); - } - - #[test] - fn listen_mode_accepts_explicit_unauthenticated_tcp() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--allow-unauthenticated-tcp", - "--bind-address", - "127.0.0.1:50061", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap()) - ); - } - - #[test] - fn listen_mode_requires_expected_peer_pid_for_uds() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - ]); - let err = compute_driver_listen_mode(&args) - .expect_err("UDS should require gateway peer pid by default"); - assert!(err.contains("--expected-peer-pid is required")); - } - - #[test] - fn listen_mode_accepts_uds_with_expected_peer_pid() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - "--expected-peer-pid", - "42", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Unix { - socket_path: PathBuf::from("/tmp/compute-driver.sock"), - expected_peer_pid: Some(42), - } - ); - } - - #[test] - fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - "--allow-same-uid-peer", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Unix { - socket_path: PathBuf::from("/tmp/compute-driver.sock"), - expected_peer_pid: None, - } - ); - } -} +#[cfg(not(target_os = "windows"))] +include!("main_unix.rs"); diff --git a/crates/openshell-driver-vm/src/main_unix.rs b/crates/openshell-driver-vm/src/main_unix.rs new file mode 100644 index 0000000000..0ae694effa --- /dev/null +++ b/crates/openshell-driver-vm/src/main_unix.rs @@ -0,0 +1,728 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use clap::Parser; +use futures::Stream; +use miette::{IntoDiagnostic, Result}; +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +#[cfg(target_os = "macos")] +use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; +use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; +use std::io; +use std::net::SocketAddr; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::net::{UnixListener, UnixStream}; +use tracing::info; +use tracing_subscriber::EnvFilter; + +#[derive(Parser, Debug)] +#[command(name = "openshell-driver-vm")] +#[command(version = VERSION)] +#[allow(clippy::struct_excessive_bools)] +struct Args { + #[arg(long, hide = true, default_value_t = false)] + internal_run_vm: bool, + + #[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")] + vm_root_disk: Option, + + #[arg(long = "vm-overlay-disk", hide = true)] + vm_overlay_disk: Option, + + #[arg(long = "vm-image-disk", hide = true)] + vm_image_disk: Option, + + #[arg(long = "vm-kernel-image", hide = true)] + vm_kernel_image: Option, + + #[arg(long, hide = true)] + vm_exec: Option, + + #[arg(long, hide = true, default_value = "/")] + vm_workdir: String, + + #[arg(long, hide = true)] + vm_env: Vec, + + #[arg(long, hide = true)] + vm_console_output: Option, + + #[arg(long, hide = true, default_value_t = 2)] + vm_vcpus: u8, + + #[arg(long, hide = true, default_value_t = 2048)] + vm_mem_mib: u32, + + #[arg(long, hide = true, default_value_t = 1)] + vm_krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")] + bind_address: Option, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + + #[arg(long, hide = true)] + expected_peer_pid: Option, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP", + default_value_t = false + )] + allow_unauthenticated_tcp: bool, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER", + default_value_t = false + )] + allow_same_uid_peer: bool, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + openshell_endpoint: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] + default_image: String, + + #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] + bootstrap_image: String, + + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_STATE_DIR", + default_value = "target/openshell-vm-driver" + )] + state_dir: PathBuf, + + #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] + guest_tls_ca: Option, + + #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] + guest_tls_cert: Option, + + #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] + guest_tls_key: Option, + + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] + krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] + vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] + mem_mib: u32, + + #[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)] + overlay_disk_mib: u64, + + #[arg(long, env = "OPENSHELL_VM_GPU")] + gpu: bool, + + #[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)] + gpu_mem_mib: u32, + + #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] + gpu_vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")] + sandbox_uid: Option, + + #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] + sandbox_gid: Option, + + #[arg(long, hide = true)] + vm_backend: Option, + + #[arg(long, hide = true)] + vm_gpu_bdf: Option, + + #[arg(long, hide = true)] + vm_tap_device: Option, + + #[arg(long, hide = true)] + vm_guest_ip: Option, + + #[arg(long, hide = true)] + vm_host_ip: Option, + + #[arg(long, hide = true)] + vm_vsock_cid: Option, + + #[arg(long, hide = true)] + vm_guest_mac: Option, + + #[arg(long, hide = true)] + vm_gateway_port: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + if args.internal_run_vm { + // We intentionally defer procguard arming until `run_vm()` so + // that the only arm is the one that knows how to clean up + // gvproxy. Racing two watchers against the same parent-death + // event causes the bare arm's `exit(1)` to win, skipping the + // gvproxy cleanup and leaking the helper. The risk window + // before `run_vm` arms procguard is ~a few syscalls long + // (`build_vm_launch_config`, `configured_runtime_dir`), which + // is negligible next to the parent gRPC server's uptime. + maybe_reexec_internal_vm_with_runtime_env()?; + let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; + run_vm(&config).map_err(|err| miette::miette!("{err}"))?; + return Ok(()); + } + + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; + + // Arm procguard so that if the gateway is killed (SIGKILL or crash) + // we also die. Without this the driver is reparented to init and + // keeps its per-sandbox VM launchers alive forever. Launchers have + // their own procguards (armed in `run_vm`) which cascade cleanup of + // gvproxy and the libkrun worker the moment this driver exits. + if let Err(err) = procguard::die_with_parent() { + tracing::warn!( + error = %err, + "procguard arm failed; gateway crashes may orphan this driver" + ); + } + + let driver = VmDriver::new(VmDriverConfig { + openshell_endpoint: args + .openshell_endpoint + .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, + state_dir: args.state_dir.clone(), + launcher_bin: None, + default_image: args.default_image.clone(), + bootstrap_image: args.bootstrap_image.clone(), + log_level: args.log_level.clone(), + krun_log_level: args.krun_log_level, + vcpus: args.vcpus, + mem_mib: args.mem_mib, + overlay_disk_mib: args.overlay_disk_mib, + guest_tls_ca: args.guest_tls_ca.clone(), + guest_tls_cert: args.guest_tls_cert.clone(), + guest_tls_key: args.guest_tls_key.clone(), + gpu_enabled: args.gpu, + gpu_mem_mib: args.gpu_mem_mib, + gpu_vcpus: args.gpu_vcpus, + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, + }) + .await + .map_err(|err| miette::miette!("{err}"))?; + + match listen_mode { + ComputeDriverListenMode::Unix { + socket_path, + expected_peer_pid, + } => { + prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?; + + info!(socket = %socket_path.display(), "Starting vm compute driver"); + let listener = UnixListener::bind(&socket_path).into_diagnostic()?; + restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?; + let result = tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming(AuthenticatedUnixIncoming::new(listener, expected_peer_pid)) + .await + .into_diagnostic(); + let _ = std::fs::remove_file(&socket_path); + result + } + ComputeDriverListenMode::Tcp(bind_address) => { + info!(address = %bind_address, "Starting unauthenticated dev vm compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve(bind_address) + .await + .into_diagnostic() + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum ComputeDriverListenMode { + Unix { + socket_path: PathBuf, + expected_peer_pid: Option, + }, + Tcp(SocketAddr), +} + +fn compute_driver_listen_mode(args: &Args) -> std::result::Result { + if let Some(socket_path) = args.bind_socket.clone() { + if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer { + return Err( + "--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development" + .to_string(), + ); + } + return Ok(ComputeDriverListenMode::Unix { + socket_path, + expected_peer_pid: args.expected_peer_pid, + }); + } + + if !args.allow_unauthenticated_tcp { + return Err( + "--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development" + .to_string(), + ); + } + + let Some(bind_address) = args.bind_address else { + return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string()); + }; + + Ok(ComputeDriverListenMode::Tcp(bind_address)) +} + +fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> { + let Some(parent) = socket_path.parent() else { + return Err(format!( + "vm compute driver socket path '{}' has no parent directory", + socket_path.display() + )); + }; + let expected_uid = current_euid(); + prepare_private_socket_dir(parent, expected_uid)?; + remove_stale_socket(socket_path, expected_uid) +} + +fn current_euid() -> u32 { + rustix::process::geteuid().as_raw() +} + +fn prepare_private_socket_dir( + socket_dir: &Path, + expected_uid: u32, +) -> std::result::Result<(), String> { + std::fs::create_dir_all(socket_dir) + .map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?; + let metadata = std::fs::symlink_metadata(socket_dir) + .map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(format!( + "socket dir {} is a symlink; refusing to use it", + socket_dir.display() + )); + } + if !file_type.is_dir() { + return Err(format!( + "socket dir {} is not a directory", + socket_dir.display() + )); + } + if metadata.uid() != expected_uid { + return Err(format!( + "socket dir {} is owned by uid {} but current euid is {}", + socket_dir.display(), + metadata.uid(), + expected_uid + )); + } + std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display())) +} + +fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> { + let metadata = match std::fs::symlink_metadata(socket_path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())), + }; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(format!( + "socket {} is a symlink; refusing to remove it", + socket_path.display() + )); + } + if metadata.uid() != expected_uid { + return Err(format!( + "socket {} is owned by uid {} but current euid is {}", + socket_path.display(), + metadata.uid(), + expected_uid + )); + } + if !file_type.is_socket() { + return Err(format!( + "socket path {} exists but is not a Unix socket", + socket_path.display() + )); + } + std::fs::remove_file(socket_path) + .map_err(|err| format!("remove stale socket {}: {err}", socket_path.display())) +} + +fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> { + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) + .map_err(|err| format!("chmod socket {}: {err}", socket_path.display())) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PeerCredentials { + uid: u32, + pid: Option, +} + +fn peer_credentials(stream: &UnixStream) -> std::result::Result { + let credentials = stream + .peer_cred() + .map_err(|err| format!("read peer credentials: {err}"))?; + Ok(PeerCredentials { + uid: credentials.uid(), + pid: credentials.pid(), + }) +} + +fn authorize_peer_credentials( + peer: PeerCredentials, + driver_uid: u32, + gateway_pid: Option, +) -> std::result::Result<(), String> { + if peer.uid != driver_uid { + return Err(format!( + "peer uid {} does not match current euid {}", + peer.uid, driver_uid + )); + } + let Some(gateway_pid) = gateway_pid else { + return Ok(()); + }; + let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else { + return Err(format!( + "peer pid is unavailable; expected gateway pid {gateway_pid}" + )); + }; + if peer_process_id != gateway_pid { + return Err(format!( + "peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}" + )); + } + Ok(()) +} + +struct AuthenticatedUnixIncoming { + listener: UnixListener, + expected_uid: u32, + expected_peer_pid: Option, +} + +impl AuthenticatedUnixIncoming { + fn new(listener: UnixListener, expected_peer_pid: Option) -> Self { + Self { + listener, + expected_uid: current_euid(), + expected_peer_pid, + } + } +} + +impl Stream for AuthenticatedUnixIncoming { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.listener.poll_accept(cx) { + Poll::Ready(Ok((stream, _addr))) => { + let authorized = peer_credentials(&stream).and_then(|peer| { + authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid) + }); + match authorized { + Ok(()) => return Poll::Ready(Some(Ok(stream))), + Err(err) => { + tracing::warn!( + error = %err, + "rejected vm compute driver UDS client" + ); + } + } + } + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + Poll::Pending => return Poll::Pending, + } + } + } +} + +fn build_vm_launch_config(args: &Args) -> std::result::Result { + let root_disk = args + .vm_root_disk + .clone() + .ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?; + let overlay_disk = args + .vm_overlay_disk + .clone() + .ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?; + let image_disk = args.vm_image_disk.clone(); + let exec_path = args + .vm_exec + .clone() + .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; + let console_output = args + .vm_console_output + .clone() + .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; + + let backend = match args.vm_backend.as_deref() { + Some("qemu") => VmBackend::Qemu, + Some("libkrun") | None => VmBackend::Libkrun, + Some(other) => return Err(format!("unknown VM backend: {other}")), + }; + + Ok(VmLaunchConfig { + root_disk, + overlay_disk, + image_disk, + kernel_image: args.vm_kernel_image.clone(), + vcpus: args.vm_vcpus, + mem_mib: args.vm_mem_mib, + exec_path, + args: Vec::new(), + env: args.vm_env.clone(), + workdir: args.vm_workdir.clone(), + log_level: args.vm_krun_log_level, + console_output, + backend, + gpu_bdf: args.vm_gpu_bdf.clone(), + tap_device: args.vm_tap_device.clone(), + guest_ip: args.vm_guest_ip.clone(), + host_ip: args.vm_host_ip.clone(), + vsock_cid: args.vm_vsock_cid, + guest_mac: args.vm_guest_mac.clone(), + gateway_port: args.vm_gateway_port, + }) +} + +#[cfg(target_os = "macos")] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + use std::os::unix::process::CommandExt as _; + + const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; + + if std::env::var_os(REEXEC_ENV).is_some() { + return Ok(()); + } + + let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; + let runtime_str = runtime_dir.to_string_lossy(); + let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") + .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); + if !needs_reexec { + return Ok(()); + } + + let mut dyld_paths = vec![runtime_dir.clone()]; + if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { + dyld_paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(&dyld_paths) + .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; + let exe = std::env::current_exe().into_diagnostic()?; + let args: Vec = std::env::args().skip(1).collect(); + + // Use execvp() so the current process is *replaced* by the re-exec'd + // binary — no wrapper process sits between the compute driver and + // the actually-running VM launcher. That avoids two problems: + // 1. An extra process level that survives SIGKILL of the driver + // (the wrapper was reparenting the re-exec'd child to init). + // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper + // doesn't reach the child unless we hand-roll forwarding. + // After exec, the child inherits our PID and our procguard arming. + let err = std::process::Command::new(exe) + .args(&args) + .env("DYLD_LIBRARY_PATH", &joined) + .env(VM_RUNTIME_DIR_ENV, runtime_dir) + .env(REEXEC_ENV, "1") + .exec(); + // `exec()` only returns on failure. + Err(miette::miette!("failed to re-exec with runtime env: {err}")) +} + +#[cfg(not(target_os = "macos"))] +// Signature must match the macOS variant which can fail. +#[allow(clippy::unnecessary_wraps)] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, + compute_driver_listen_mode, + }; + use clap::Parser; + use std::path::PathBuf; + + #[test] + fn peer_authorization_accepts_matching_uid_and_pid() { + authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: Some(42), + }, + 1000, + Some(42), + ) + .unwrap(); + } + + #[test] + fn peer_authorization_rejects_wrong_pid() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: Some(7), + }, + 1000, + Some(42), + ) + .expect_err("wrong pid should be rejected"); + assert!(err.contains("does not match expected gateway pid")); + } + + #[test] + fn peer_authorization_rejects_wrong_uid() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1001, + pid: Some(42), + }, + 1000, + Some(42), + ) + .expect_err("wrong uid should be rejected"); + assert!(err.contains("does not match current euid")); + } + + #[test] + fn peer_authorization_rejects_missing_pid_when_expected() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: None, + }, + 1000, + Some(42), + ) + .expect_err("missing pid should be rejected"); + assert!(err.contains("peer pid is unavailable")); + } + + #[test] + fn peer_authorization_accepts_matching_uid_without_expected_pid() { + authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: None, + }, + 1000, + None, + ) + .unwrap(); + } + + #[test] + fn listen_mode_rejects_default_tcp() { + let args = Args::parse_from(["openshell-driver-vm"]); + let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled"); + assert!(err.contains("--bind-socket is required")); + } + + #[test] + fn listen_mode_rejects_bind_address_without_tcp_opt_in() { + let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); + let err = + compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in"); + assert!(err.contains("--allow-unauthenticated-tcp")); + } + + #[test] + fn listen_mode_requires_bind_address_with_tcp_opt_in() { + let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]); + let err = + compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address"); + assert!(err.contains("--bind-address is required")); + } + + #[test] + fn listen_mode_accepts_explicit_unauthenticated_tcp() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--allow-unauthenticated-tcp", + "--bind-address", + "127.0.0.1:50061", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap()) + ); + } + + #[test] + fn listen_mode_requires_expected_peer_pid_for_uds() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + ]); + let err = compute_driver_listen_mode(&args) + .expect_err("UDS should require gateway peer pid by default"); + assert!(err.contains("--expected-peer-pid is required")); + } + + #[test] + fn listen_mode_accepts_uds_with_expected_peer_pid() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + "--expected-peer-pid", + "42", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Unix { + socket_path: PathBuf::from("/tmp/compute-driver.sock"), + expected_peer_pid: Some(42), + } + ); + } + + #[test] + fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + "--allow-same-uid-peer", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Unix { + socket_path: PathBuf::from("/tmp/compute-driver.sock"), + expected_peer_pid: None, + } + ); + } +} diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 4c4f289ed6..7f660a06df 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -93,7 +93,7 @@ async-trait = "0.1" url = { workspace = true } glob = { workspace = true } hex = "0.4" -russh = "0.57" +russh = { version = "0.57", default-features = false, features = ["flate2", "ring", "rsa"] } rand = { workspace = true } petname = "2" ipnet = "2" diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 269b9f40e9..e9caac5de4 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -1171,11 +1171,15 @@ mod tests { let expected = format!( "sqlite:{}", - tmp.path().join("openshell/gateway/openshell.db").display() + tmp.path() + .join("openshell") + .join("gateway") + .join("openshell.db") + .display() ); assert!(local_tls.is_none()); assert_eq!(args.db_url.as_deref(), Some(expected.as_str())); - assert!(tmp.path().join("openshell/gateway").is_dir()); + assert!(tmp.path().join("openshell").join("gateway").is_dir()); } #[test] diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index cf6b17c7e9..e0b0121633 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -38,11 +38,21 @@ use openshell_core::proto::{ SandboxTemplate, ServiceEndpoint, SshSession, }; use openshell_core::{ObjectLabels, ObjectWorkspace}; +#[cfg(not(target_os = "windows"))] use openshell_driver_docker::DockerComputeDriver; +#[cfg(target_os = "windows")] +use openshell_driver_kubernetes::KubernetesComputeConfig; +#[cfg(not(target_os = "windows"))] use openshell_driver_kubernetes::{ - ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, + ComputeDriverService as KubernetesDriverService, KubernetesComputeConfig, + KubernetesComputeDriver, +}; +#[cfg(target_os = "windows")] +use openshell_driver_podman::PodmanComputeConfig; +#[cfg(not(target_os = "windows"))] +use openshell_driver_podman::{ + ComputeDriverService as PodmanDriverService, PodmanComputeConfig, PodmanComputeDriver, }; -use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; use std::collections::HashMap; use std::fmt; @@ -165,6 +175,7 @@ trait ShutdownCleanup: Send + Sync { } #[tonic::async_trait] +#[cfg(not(target_os = "windows"))] impl ShutdownCleanup for DockerComputeDriver { async fn cleanup_on_shutdown(&self) -> Result<(), String> { let stopped = self @@ -190,6 +201,7 @@ trait StartupResume: Send + Sync { } #[tonic::async_trait] +#[cfg(not(target_os = "windows"))] impl StartupResume for DockerComputeDriver { async fn resume_sandbox(&self, sandbox_id: &str, sandbox_name: &str) -> Result { Self::resume_sandbox(self, sandbox_id, sandbox_name) @@ -207,6 +219,11 @@ const ORPHAN_GRACE_PERIOD: Duration = Duration::from_secs(300); // Re-export the shared error type under the name used by this module. pub use openshell_core::ComputeDriverError as ComputeError; +#[cfg(target_os = "windows")] +fn unsupported_compute_driver_error(driver: &str) -> ComputeError { + ComputeError::Message(format!("{driver} compute driver is unsupported on Windows")) +} + #[derive(Debug)] pub struct ManagedDriverProcess { child: std::sync::Mutex>, @@ -214,6 +231,7 @@ pub struct ManagedDriverProcess { } impl ManagedDriverProcess { + #[cfg(unix)] pub(crate) fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { Self { child: std::sync::Mutex::new(Some(child)), @@ -457,6 +475,7 @@ impl ComputeRuntime { self.delete_gates.entry_count() } + #[cfg(not(target_os = "windows"))] pub async fn new_docker( config: openshell_core::Config, docker_config: DockerComputeConfig, @@ -491,6 +510,20 @@ impl ComputeRuntime { .await } + #[cfg(target_os = "windows")] + pub async fn new_docker( + _config: openshell_core::Config, + _docker_config: DockerComputeConfig, + _store: Arc, + _sandbox_index: SandboxIndex, + _sandbox_watch_bus: SandboxWatchBus, + _tracing_log_bus: TracingLogBus, + _supervisor_sessions: Arc, + ) -> Result { + Err(unsupported_compute_driver_error("Docker")) + } + + #[cfg(not(target_os = "windows"))] pub async fn new_kubernetes( config: KubernetesComputeConfig, store: Arc, @@ -519,6 +552,18 @@ impl ComputeRuntime { .await } + #[cfg(target_os = "windows")] + pub async fn new_kubernetes( + _config: KubernetesComputeConfig, + _store: Arc, + _sandbox_index: SandboxIndex, + _sandbox_watch_bus: SandboxWatchBus, + _tracing_log_bus: TracingLogBus, + _supervisor_sessions: Arc, + ) -> Result { + Err(unsupported_compute_driver_error("Kubernetes")) + } + pub(crate) async fn new_remote_driver( endpoint: AcquiredRemoteDriverEndpoint, store: Arc, @@ -544,6 +589,7 @@ impl ComputeRuntime { .await } + #[cfg(not(target_os = "windows"))] pub async fn new_podman( config: PodmanComputeConfig, store: Arc, @@ -572,6 +618,18 @@ impl ComputeRuntime { .await } + #[cfg(target_os = "windows")] + pub async fn new_podman( + _config: PodmanComputeConfig, + _store: Arc, + _sandbox_index: SandboxIndex, + _sandbox_watch_bus: SandboxWatchBus, + _tracing_log_bus: TracingLogBus, + _supervisor_sessions: Arc, + ) -> Result { + Err(unsupported_compute_driver_error("Podman")) + } + #[must_use] pub fn default_image(&self) -> &str { &self.default_image @@ -5994,4 +6052,16 @@ mod tests { .unwrap(); assert_eq!(stored.object_workspace(), "alpha"); } + + #[cfg(target_os = "windows")] + #[test] + fn windows_compute_driver_stubs_report_unsupported() { + for driver in ["Docker", "Kubernetes", "Podman"] { + let message = unsupported_compute_driver_error(driver).to_string(); + assert!( + message.contains("unsupported on Windows"), + "{driver} stub message should be explicit, got: {message}" + ); + } + } } diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index be88047f33..1d3c8597d0 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -31,6 +31,7 @@ use super::AcquiredRemoteDriverEndpoint; #[cfg(unix)] +#[cfg(unix)] use super::ManagedDriverProcess; #[cfg(unix)] use hyper_util::rt::TokioIo; @@ -56,8 +57,11 @@ use tonic::transport::Endpoint; #[cfg(unix)] use tower::service_fn; +#[cfg(unix)] const DRIVER_BIN_NAME: &str = "openshell-driver-vm"; +#[cfg(unix)] const COMPUTE_DRIVER_SOCKET_RUN_DIR: &str = "run"; +#[cfg(unix)] const COMPUTE_DRIVER_SOCKET_NAME: &str = "compute-driver.sock"; /// Configuration for launching and talking to the VM compute driver. @@ -136,6 +140,7 @@ impl VmComputeConfig { 4096 } + #[cfg(unix)] #[must_use] fn default_driver_search_dirs(home: Option) -> Vec { let mut dirs = Vec::new(); @@ -186,6 +191,7 @@ pub struct VmGuestTlsPaths { /// `/usr/local/libexec/openshell`, `/usr/local/libexec`. /// 3. Sibling of the gateway's own executable (last-resort fallback so /// local development builds still work out of the box). +#[cfg(unix)] pub fn resolve_compute_driver_bin(vm_config: &VmComputeConfig) -> Result { let mut searched: Vec = Vec::new(); @@ -224,6 +230,7 @@ pub fn resolve_compute_driver_bin(vm_config: &VmComputeConfig) -> Result Vec { vm_config.driver_dir.clone().map_or_else( || { @@ -245,6 +252,7 @@ fn resolve_driver_search_dirs(vm_config: &VmComputeConfig) -> Vec { ) } +#[cfg(unix)] fn push_unique_path(paths: &mut Vec, path: PathBuf) { if !paths.iter().any(|existing| existing == &path) { paths.push(path); @@ -252,6 +260,7 @@ fn push_unique_path(paths: &mut Vec, path: PathBuf) { } /// Path of the Unix domain socket the driver will listen on. +#[cfg(unix)] pub fn compute_driver_socket_path(vm_config: &VmComputeConfig) -> PathBuf { vm_config .state_dir @@ -515,7 +524,20 @@ pub async fn spawn( )) } -#[cfg(not(unix))] +#[cfg(target_os = "windows")] +fn unsupported_vm_message() -> &'static str { + "VM compute driver is unsupported on Windows" +} + +#[cfg(target_os = "windows")] +pub async fn spawn( + _config: &Config, + _vm_config: &VmComputeConfig, +) -> Result<(Channel, std::sync::Arc)> { + Err(Error::config(unsupported_vm_message())) +} + +#[cfg(all(not(unix), not(target_os = "windows")))] pub async fn spawn( _config: &Config, _vm_config: &VmComputeConfig, @@ -866,3 +888,11 @@ mod tests { assert!(!socket_path.exists()); } } + +#[cfg(all(test, target_os = "windows"))] +mod windows_tests { + #[test] + fn windows_spawn_reports_unsupported() { + assert!(super::unsupported_vm_message().contains("unsupported on Windows")); + } +} diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index b3dbaa569a..867c9b85b0 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -64,6 +64,7 @@ struct LiveSession { /// target-open failure reported by the supervisor. type RelayStreamSender = oneshot::Sender>; +#[cfg(not(target_os = "windows"))] impl openshell_driver_docker::SupervisorReadiness for SupervisorSessionRegistry { fn is_supervisor_connected(&self, sandbox_id: &str) -> bool { Self::is_connected(self, sandbox_id) From c78c1a03f0756e374280b1abbf35bc68bb243f6c Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Mon, 1 Jun 2026 11:14:24 -0700 Subject: [PATCH 03/30] ci(windows): add MSVC mise build lane Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- .github/workflows/windows-msvc.yml | 36 +++++ tasks/scripts/windows-msvc.ps1 | 244 +++++++++++++++++++++++++++++ tasks/windows.toml | 55 +++++++ 3 files changed, 335 insertions(+) create mode 100644 .github/workflows/windows-msvc.yml create mode 100644 tasks/scripts/windows-msvc.ps1 create mode 100644 tasks/windows.toml diff --git a/.github/workflows/windows-msvc.yml b/.github/workflows/windows-msvc.yml new file mode 100644 index 0000000000..076af08a82 --- /dev/null +++ b/.github/workflows/windows-msvc.yml @@ -0,0 +1,36 @@ +name: Windows MSVC (build-only) +on: + push: + branches: [main] + pull_request: +jobs: + x64: + runs-on: windows-2025 + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v3 + with: + install: false + experimental: true + - uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-msvc + - run: mise run --skip-tools windows:check:x64 + - run: mise run --skip-tools windows:build:x64 + - run: mise run --skip-tools windows:test:x64 + - run: mise run --skip-tools windows:test:unsupported:x64 + arm64: + # TODO: provision a windows-arm64 self-hosted runner + runs-on: [self-hosted, windows-arm64] + if: false # flip to true once the runner is online + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v3 + with: + install: false + experimental: true + - uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-pc-windows-msvc + - run: mise run --skip-tools windows:check:arm64 + - run: mise run --skip-tools windows:build:arm64 diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 new file mode 100644 index 0000000000..ad67b4b067 --- /dev/null +++ b/tasks/scripts/windows-msvc.ps1 @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Windows MSVC build wrapper used by the `windows:*` mise tasks. + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [ValidateSet("check", "build", "test", "test-unsupported", "artifacts", "ci")] + [string] $Action, + + [Parameter(Position = 1)] + [ValidateSet("x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", "all")] + [string] $Target = "all", + + [string] $LogDir +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +if (-not [System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform([System.Runtime.InteropServices.OSPlatform]::Windows)) { + throw "windows-msvc.ps1 requires a Windows MSVC host." +} + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +if (-not $LogDir) { + $LogDir = $RepoRoot +} +if (-not (Test-Path $LogDir)) { + New-Item -ItemType Directory -Force -Path $LogDir | Out-Null +} +$LogDir = (Resolve-Path $LogDir).Path + +$TargetDir = $env:CARGO_TARGET_DIR +if ([string]::IsNullOrWhiteSpace($TargetDir)) { + $TargetDir = Join-Path $RepoRoot "target" +} + +$UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm" + +function Resolve-VsDevCmd { + if ($env:OPENSHELL_VSDEVCMD -and (Test-Path $env:OPENSHELL_VSDEVCMD)) { + return (Resolve-Path $env:OPENSHELL_VSDEVCMD).Path + } + + $programFilesX86 = [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + if ($programFilesX86) { + $vswhere = Join-Path $programFilesX86 "Microsoft Visual Studio\Installer\vswhere.exe" + } else { + $vswhere = $null + } + if ($vswhere -and (Test-Path $vswhere)) { + $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find "Common7\Tools\VsDevCmd.bat" | Select-Object -First 1 + if ($found -and (Test-Path $found)) { + return (Resolve-Path $found).Path + } + } + + $programFiles = @( + [Environment]::GetEnvironmentVariable("ProgramFiles"), + [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + ) | Where-Object { $_ } + $versions = @("18", "17") + $editions = @("Enterprise", "Professional", "Community", "BuildTools") + foreach ($root in $programFiles) { + foreach ($version in $versions) { + foreach ($edition in $editions) { + $candidate = Join-Path $root "Microsoft Visual Studio\$version\$edition\Common7\Tools\VsDevCmd.bat" + if (Test-Path $candidate) { + return (Resolve-Path $candidate).Path + } + } + } + } + + throw "Could not find VsDevCmd.bat. Install Visual Studio Build Tools, or set OPENSHELL_VSDEVCMD." +} + +function Get-HostArch { + switch ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString()) { + "Arm64" { "arm64" } + default { "amd64" } + } +} + +function Get-VsTargetArch([string] $RustTarget) { + switch ($RustTarget) { + "x86_64-pc-windows-msvc" { "amd64" } + "aarch64-pc-windows-msvc" { "arm64" } + default { throw "Unsupported target: $RustTarget" } + } +} + +function Get-SelectedTargets([string] $RequestedTarget) { + if ($RequestedTarget -eq "all") { + $targets = @("x86_64-pc-windows-msvc") + if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { + $targets += "aarch64-pc-windows-msvc" + } + return $targets + } + return @($RequestedTarget) +} + +function Invoke-VsCargo { + param( + [Parameter(Mandatory = $true)] [string] $RustTarget, + [Parameter(Mandatory = $true)] [string] $CargoArgs, + [Parameter(Mandatory = $true)] [string] $LogName + ) + + & rustup target add $RustTarget + if ($LASTEXITCODE -ne 0) { + throw "rustup target add $RustTarget failed" + } + + $vsDevCmd = Resolve-VsDevCmd + $targetArch = Get-VsTargetArch $RustTarget + $hostArch = Get-HostArch + $logPath = Join-Path $LogDir $LogName + $cmd = "call `"$vsDevCmd`" -arch=$targetArch -host_arch=$hostArch && set `"CARGO_TARGET_DIR=$TargetDir`" && set `"CARGO_INCREMENTAL=0`" && set `"RUSTC_WRAPPER=`" && $CargoArgs" + + Write-Host "==> $CargoArgs" + Write-Host " target: $RustTarget" + Write-Host " log: $logPath" + + $cmdWithLog = "$cmd > `"$logPath`" 2>&1" + & cmd /v:on /d /c $cmdWithLog + $exitCode = $LASTEXITCODE + if (Test-Path $logPath) { + Get-Content $logPath + } + if ($exitCode -ne 0) { + throw "Command failed with exit code $exitCode. See $logPath" + } +} + +function Invoke-Check([string] $RustTarget) { + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget" ` + -LogName "build-$RustTarget-check.log" +} + +function Invoke-Build([string] $RustTarget) { + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell" ` + -LogName "build-$RustTarget-release.log" +} + +function Invoke-Test([string] $RustTarget) { + if ($RustTarget -ne "x86_64-pc-windows-msvc") { + throw "Windows ARM64 tests require a native ARM64 runner and are not part of this build-only lane." + } + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo test --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast" ` + -LogName "test-$RustTarget.log" +} + +function Invoke-UnsupportedContractTests([string] $RustTarget) { + if ($RustTarget -ne "x86_64-pc-windows-msvc") { + throw "Unsupported-driver contract tests run only on the native x64 Windows lane." + } + + $tests = @( + "windows_compute_driver_stubs_report_unsupported", + "windows_spawn_reports_unsupported" + ) + foreach ($test in $tests) { + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo test -p openshell-server --target $RustTarget $test" ` + -LogName "test-$RustTarget-unsupported-$test.log" + } +} + +function Show-Artifacts([string[]] $RustTargets) { + $rows = @() + foreach ($rustTarget in $RustTargets) { + foreach ($binary in @("openshell-gateway.exe", "openshell.exe")) { + $path = Join-Path $TargetDir "$rustTarget\release\$binary" + if (-not (Test-Path $path)) { + continue + } + $item = Get-Item $path + $hash = Get-FileHash $path -Algorithm SHA256 + $rows += [pscustomobject]@{ + Target = $rustTarget + Binary = $binary + Size = $item.Length + SHA256 = $hash.Hash + Path = $item.FullName + } + } + } + if ($rows.Count -eq 0) { + Write-Warning "No release artifacts found under $TargetDir" + return + } + $rows | Format-Table -AutoSize +} + +$targets = Get-SelectedTargets $Target + +switch ($Action) { + "check" { + foreach ($rustTarget in $targets) { + Invoke-Check $rustTarget + } + } + "build" { + foreach ($rustTarget in $targets) { + Invoke-Build $rustTarget + } + Show-Artifacts $targets + } + "test" { + foreach ($rustTarget in $targets) { + Invoke-Test $rustTarget + } + } + "test-unsupported" { + foreach ($rustTarget in $targets) { + Invoke-UnsupportedContractTests $rustTarget + } + } + "artifacts" { + Show-Artifacts $targets + } + "ci" { + foreach ($rustTarget in $targets) { + Invoke-Check $rustTarget + } + foreach ($rustTarget in $targets) { + Invoke-Build $rustTarget + } + Invoke-Test "x86_64-pc-windows-msvc" + Invoke-UnsupportedContractTests "x86_64-pc-windows-msvc" + Show-Artifacts $targets + } +} diff --git a/tasks/windows.toml b/tasks/windows.toml new file mode 100644 index 0000000000..fa3b2b60ae --- /dev/null +++ b/tasks/windows.toml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Windows MSVC build-only tasks. These intentionally live outside the default +# Linux/macOS `ci` task so the established Linux build path remains unchanged. + +["windows:check"] +description = "Check Windows x64 and ARM64 MSVC targets" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 check all" + +["windows:check:x64"] +description = "Check Windows x64 MSVC target" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 check x86_64-pc-windows-msvc" + +["windows:check:arm64"] +description = "Check Windows ARM64 MSVC target" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 check aarch64-pc-windows-msvc" + +["windows:build"] +description = "Build Windows x64 and ARM64 release binaries" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 build all" + +["windows:build:x64"] +description = "Build Windows x64 release binaries" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 build x86_64-pc-windows-msvc" + +["windows:build:arm64"] +description = "Build Windows ARM64 release binaries" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 build aarch64-pc-windows-msvc" + +["windows:test:x64"] +description = "Run Windows x64 MSVC workspace tests" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test x86_64-pc-windows-msvc" + +["windows:test:unsupported:x64"] +description = "Run focused Windows unsupported compute-driver contract tests" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-unsupported x86_64-pc-windows-msvc" + +["windows:artifacts"] +description = "Report Windows release artifact sizes and SHA256 hashes" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 artifacts all" + +["windows:ci"] +description = "Run Windows MSVC checks, release builds, x64 tests, and unsupported-driver contract tests" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 ci all" From 2e1fc1cb82ab5d10dda5f08f9d861f156e9001ca Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Mon, 1 Jun 2026 11:15:32 -0700 Subject: [PATCH 04/30] docs(windows): document MSVC build-only design Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- docs/reference/windows-msvc-build-design.mdx | 98 ++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/reference/windows-msvc-build-design.mdx diff --git a/docs/reference/windows-msvc-build-design.mdx b/docs/reference/windows-msvc-build-design.mdx new file mode 100644 index 0000000000..f103c4cc83 --- /dev/null +++ b/docs/reference/windows-msvc-build-design.mdx @@ -0,0 +1,98 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Windows MSVC Build Design" +sidebar-title: "Windows MSVC Build" +description: "Design notes for the build-only native Windows MSVC lane." +keywords: "Windows, MSVC, OpenShell, mise, Rust, build" +position: 6 +--- + +This page records the design decisions for the native Windows MSVC build lane. +It is intentionally build-only. It does not make Windows a Docker, Kubernetes, +Podman, or VM runtime host. + +## Goals + +- Compile the OpenShell gateway and CLI for `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`. +- Keep the Linux and macOS build paths unchanged. +- Preserve gateway configuration parsing for all existing compute driver names. +- Return clear unsupported errors when a Windows gateway is configured to use Docker, Kubernetes, Podman, or VM. +- Validate the Windows lane through mise tasks that are separate from the default Linux `ci` task. + +## Non-Goals + +- Do not support Docker Desktop, WSL, Hyper-V, Podman machine, Podman Desktop, Kubernetes, or VM-backed sandbox execution on Windows. +- Do not ship Windows standalone binaries for Docker, Kubernetes, Podman, or VM drivers. +- Do not implement named-pipe driver IPC, Windows services, MSI packaging, Credential Manager integration, DPAPI integration, or MXC policy translation in this build lane. + +## Unsupported Driver Strategy + +Unsupported compute drivers use contract stubs on Windows. The stubs preserve +configuration structs and public library entry points so the gateway can parse +existing config files and reject unsupported driver selection with a clear error. + +The Windows lane does not build, release, package, or smoke-test standalone +driver binaries for Docker, Kubernetes, Podman, or VM. Those binaries are Linux +or macOS deliverables only. + +| Driver | Windows build behavior | Runtime behavior | +|---|---|---| +| Docker | Library config stub compiles as a gateway dependency. | Gateway construction returns unsupported. | +| Kubernetes | Library config stub compiles as a gateway dependency. | Gateway construction returns unsupported. | +| Podman | Library config stub compiles as a gateway dependency. | Gateway construction returns unsupported. | +| VM | Library config stub compiles as a gateway dependency. | VM spawn returns unsupported. | + +This keeps Windows behavior explicit without carrying runtime dependencies or +creating misleading Windows driver artifacts. + +## Mise Lane + +Windows validation is exposed through `tasks/windows.toml`: + +| Task | Purpose | +|---|---| +| `windows:check:x64` | Check the x64 MSVC gateway/CLI build graph. | +| `windows:check:arm64` | Check the ARM64 MSVC gateway/CLI build graph. | +| `windows:build:x64` | Build release x64 `openshell-gateway.exe` and `openshell.exe`. | +| `windows:build:arm64` | Build release ARM64 `openshell-gateway.exe` and `openshell.exe`. | +| `windows:test:x64` | Run native x64 workspace tests, excluding unsupported driver packages as top-level test targets. | +| `windows:test:unsupported:x64` | Run focused server/runtime tests for unsupported driver contracts. | +| `windows:ci` | Run check, build, test, unsupported-contract tests, and artifact reporting. | + +The Windows tasks call `tasks/scripts/windows-msvc.ps1`. The wrapper discovers +Visual Studio's `VsDevCmd.bat`, adds rustup MSVC targets, clears inherited +`RUSTC_WRAPPER`, and keeps build artifacts under the normal Cargo target tree. + +The lane uses `mise run --skip-tools windows:*` because Windows Rust comes from +rustup and linking comes from Visual Studio Build Tools. Mise orchestrates the +tasks; it does not own the Windows toolchain. + +## CI Shape + +The x64 GitHub Actions job runs on `windows-2025` and executes: + +```powershell +mise run --skip-tools windows:check:x64 +mise run --skip-tools windows:build:x64 +mise run --skip-tools windows:test:x64 +mise run --skip-tools windows:test:unsupported:x64 +``` + +The ARM64 job is scaffolded but disabled until a Windows ARM64 runner is +available. Once enabled, it should run check and release build for +`aarch64-pc-windows-msvc`. Native ARM64 tests require an ARM64 runner and are +not part of the current build-only lane. + +## Validation Contract + +A successful Windows build report should include: + +- x64 and ARM64 `cargo check` status. +- x64 and ARM64 release build status for `openshell-gateway.exe` and `openshell.exe`. +- x64 test summary. +- Focused unsupported-driver contract test status. +- Artifact size and SHA256 for each Windows binary. + +Warnings from Linux-only dead code are acceptable in this build-only phase when +they come from code paths intentionally disabled on Windows. From 80a9a119e9fda0cd2340106daa6fab8674d685ac Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Mon, 1 Jun 2026 11:16:20 -0700 Subject: [PATCH 05/30] docs(agent): add Windows MSVC build skill Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 346 ++++++++++++++++++ .../build-openshell-mxc-windows/reference.md | 184 ++++++++++ 2 files changed, 530 insertions(+) create mode 100644 .agents/skills/build-openshell-mxc-windows/SKILL.md create mode 100644 .agents/skills/build-openshell-mxc-windows/reference.md diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md new file mode 100644 index 0000000000..6dda48373b --- /dev/null +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -0,0 +1,346 @@ +--- +name: build-openshell-mxc-windows +description: Auto-executable build skill that forks OpenShell into a sibling directory and produces Windows-native x64 (x86_64-pc-windows-msvc) and ARM64 (aarch64-pc-windows-msvc) binaries on a Windows 11 26100+ host. Scope is build-only — cross-compilation, minimal Windows compatibility shims, and ARM64 CI scaffolding. Does not implement an MXC compute driver, policy translation, an MSI installer, or a supervisor port. Trigger keywords - build openshell mxc windows, openshell windows build, x64 build, arm64 build, windows msvc, openshell-mxc fork, native windows openshell, MXC build. +--- + +# Build OpenShell-MXC for Windows (x64 + ARM64) + +Auto-executable skill. Forks the OpenShell repository into a sibling directory, applies the minimum Windows compatibility shims required to compile, and produces native binaries for `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`. + +This skill covers only the build slice. It does **not** implement the MXC compute driver, policy translation, AppContainer wiring, supervisor port, MSI installer, or Windows Service registration. Those are handled by sibling skills (to be created). + +## Scope + +In scope: + +- `x86_64-pc-windows-msvc` compilation +- `aarch64-pc-windows-msvc` compilation +- Audit and cfg-gate Unix-only dependencies in `openshell-server` and shared crates so the workspace compiles to Windows MSVC +- Keep Docker, Kubernetes, Podman, and VM compute crates in the Windows build graph as stubs that return "unsupported" at runtime +- Add a Windows-specific mise task lane (`windows:*`) that wraps MSVC builds without changing the Linux `mise run ci` path +- `cargo test` compiles and passes on Windows for non-Linux-gated tests +- Scaffold an ARM64 CI workflow (job definition only; runner provisioning is operator work) + +Out of scope (defer to follow-on skills): + +- New MXC compute driver crate +- OpenShell → MXC policy translation +- Windows network and credential integration +- Sandbox supervisor Windows port +- Docker, Kubernetes, Podman, or VM runtime support on Windows +- MSI installer, Windows Service registration, WinGet manifest + +The Linux build must remain green at every commit. All Windows-specific code must be behind `#[cfg(target_os = "windows")]` and Linux-only code behind `#[cfg(target_os = "linux")]` or `#[cfg(unix)]`. + +## Prerequisites + +The skill targets a **Windows 11 host with CurrentBuild ≥ 26100**. Because the scope is compilation only, the skill warns rather than aborts when any of the items below is missing — a `cargo check` attempt can still surface useful information on a less-than-ideal host. Recommended: + +| Requirement | Check | Install hint | +|---|---|---| +| Windows 11 build ≥ 26100 | `[System.Environment]::OSVersion.Version` | OS update | +| Visual Studio 2022 Build Tools with **MSVC v143** + **Windows 11 SDK** | `where cl.exe` from a VS Developer PowerShell | https://visualstudio.microsoft.com/downloads/ | +| Rust ≥ 1.88 via rustup with MSVC targets | `rustc --version` | `winget install Rustlang.Rustup` | +| mise CLI for task orchestration only | `mise --version` | https://mise.jdx.dev/installing-mise.html | +| Git ≥ 2.40 | `git --version` | `winget install Git.Git` | +| Windows PowerShell 5.1+ or PowerShell 7+ | `$PSVersionTable.PSVersion` | Built into Windows; PowerShell 7 optional | + +The Windows mise lane shells into Visual Studio's developer environment before invoking Cargo, so commands can run from an ordinary PowerShell session if Visual Studio Build Tools are discoverable. Use rustup for the Rust toolchain and MSVC targets; mise is only the task runner for the Windows lane. In CI, prefer `mise run --skip-tools windows:*` so Linux tool installation remains untouched. + +## Inputs + +The skill reads these environment variables (PowerShell syntax): + +| Variable | Default | Purpose | +|---|---|---| +| `$env:OPENSHELL_UPSTREAM` | `https://github.com/NVIDIA/OpenShell.git` | Upstream Git remote to fork from. Override to use a local path or GitLab mirror. | +| `$env:OPENSHELL_MXC_FORK_DIR` | `C:\Users\$env:USERNAME\openshell-mxc` | Sibling directory for the fork. Must not exist. | +| `$env:OPENSHELL_WXC_EXEC_PATH` | unset | Optional path to `wxc-exec.exe`. Validated for existence and architecture match only. Not used in this build-only skill. | +| `$env:OPENSHELL_MXC_SKIP_ARM64` | `0` | Set to `1` to skip aarch64 build (e.g., for fast x64-only iterations). | +| `$env:OPENSHELL_MXC_FORK_BRANCH` | `windows-mxc-build` | Branch name created in the fork. | + +## Workflow + +The skill runs the following checklist top-to-bottom. Each step is idempotent; re-running the skill is safe once the fork exists. + +``` +[ ] Step 1: Verify host preconditions +[ ] Step 2: Install Windows MSVC Rust targets +[ ] Step 3: Fork OpenShell into the sibling directory +[ ] Step 4: Apply minimum Windows compatibility shims (cfg gating) +[ ] Step 5: Add Windows mise task lane +[ ] Step 6: mise check on x86_64-pc-windows-msvc +[ ] Step 7: mise check on aarch64-pc-windows-msvc +[ ] Step 8: mise build --release (both targets) +[ ] Step 9: mise test on x86_64-pc-windows-msvc (native run) +[ ] Step 10: Validate $env:OPENSHELL_WXC_EXEC_PATH (informational) +[ ] Step 11: Scaffold ARM64 CI workflow +[ ] Step 12: Commit and report artifacts +``` + +### Step 1: Verify host preconditions + +This step is **advisory**. It records what is and isn't available on the host, but the skill continues regardless — compilation may still succeed (or produce useful errors) on a less-than-ideal host. + +```powershell +$warnings = @() + +$os = [System.Environment]::OSVersion.Version +if ($os.Build -lt 26100) { + $warnings += "Windows build $($os.Build) < 26100 (recommended for MXC runtime; not required for compilation)." +} + +# MSVC toolchain (must be a VS 2022 Developer PowerShell to be on PATH) +foreach ($tool in 'cl.exe', 'link.exe') { + if (-not (Get-Command $tool -ErrorAction SilentlyContinue)) { + $warnings += "$tool not found on PATH — open a VS 2022 Developer PowerShell, or expect link failures in Step 5+." + } +} + +# Rust + git +foreach ($tool in 'rustup', 'cargo', 'rustc', 'git') { + if (-not (Get-Command $tool -ErrorAction SilentlyContinue)) { + $warnings += "$tool not found on PATH — Step $(if ($tool -eq 'git') { '3' } else { '2+' }) will fail." + } +} + +if (Get-Command rustc -ErrorAction SilentlyContinue) { rustc --version } + +if ($warnings.Count -gt 0) { + Write-Warning "Host preconditions not fully met:" + $warnings | ForEach-Object { Write-Warning " - $_" } + Write-Warning "Continuing anyway — re-evaluate after the next failing step." +} else { + Write-Host "Host preconditions OK." +} +``` + +Record the warning list so it can be included in the final report (Step 12). Do not attempt to install Visual Studio Build Tools or rustup automatically. + +### Step 2: Install Windows MSVC Rust targets + +```powershell +rustup target add x86_64-pc-windows-msvc +if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { + rustup target add aarch64-pc-windows-msvc +} +``` + +### Step 3: Fork OpenShell into the sibling directory + +This is the only step that creates a new on-disk repo. Confirm `$env:OPENSHELL_MXC_FORK_DIR` does not exist before cloning. + +```powershell +$fork = if ($env:OPENSHELL_MXC_FORK_DIR) { $env:OPENSHELL_MXC_FORK_DIR } else { "C:\Users\$env:USERNAME\openshell-mxc" } +$upstream = if ($env:OPENSHELL_UPSTREAM) { $env:OPENSHELL_UPSTREAM } else { "https://github.com/NVIDIA/OpenShell.git" } +$branch = if ($env:OPENSHELL_MXC_FORK_BRANCH) { $env:OPENSHELL_MXC_FORK_BRANCH } else { "windows-mxc-build" } + +if (Test-Path $fork) { throw "Fork dir already exists: $fork. Remove or set OPENSHELL_MXC_FORK_DIR." } + +git clone $upstream $fork +Set-Location $fork +git checkout -b $branch + +# Copy this skill into the fork so it can self-iterate. +New-Item -ItemType Directory -Force -Path "$fork\.claude\skills\build-openshell-mxc-windows" | Out-Null +New-Item -ItemType Directory -Force -Path "$fork\.agents\skills\build-openshell-mxc-windows" | Out-Null +Copy-Item "$PSScriptRoot\*" "$fork\.claude\skills\build-openshell-mxc-windows\" -Recurse -Force +Copy-Item "$PSScriptRoot\*" "$fork\.agents\skills\build-openshell-mxc-windows\" -Recurse -Force +``` + +From this point forward, `Set-Location $fork`. All edits land in the fork, not in the source OpenShell repo. + +### Step 4: Apply minimum Windows compatibility shims + +The goal is **the smallest patchset that makes `cargo check --target x86_64-pc-windows-msvc` succeed**, not a full Windows port. Strategy: + +1. **Cfg-gate Linux-only crates and modules.** Any module that imports `nix`, `landlock`, `libseccomp`, `caps`, or constructs Unix-domain sockets without conditional compilation must be wrapped in `#[cfg(target_os = "linux")]` or `#[cfg(unix)]`. +2. **Stub Windows-side gateway entry points.** Where Linux uses Unix domain sockets for driver IPC (gateway ↔ compute driver), provide a `#[cfg(target_os = "windows")]` stub that returns `Err(unimplemented!("Windows named-pipe transport — follow-on skill"))`. The build must compile; runtime behavior is not in scope. +3. **Disable Linux-only crates from the Windows build graph.** In each crate's `Cargo.toml`, gate platform-specific dependencies with `[target.'cfg(target_os = "linux")'.dependencies]`. Common offenders: `nix`, `landlock`, `libseccomp`, `caps`, `procfs`. +4. **Default storage paths.** Where Linux code uses `~/.local/share/openshell` or `/var/lib/openshell`, add a `#[cfg(target_os = "windows")]` branch returning `%APPDATA%\OpenShell`. + +See [reference.md](reference.md) for the concrete dependency audit (which crates and modules need gating, and the exact patterns to apply). + +Driver crates that are not supported on Windows (`openshell-driver-docker`, `openshell-driver-podman`, `openshell-driver-vm`, `openshell-driver-kubernetes`) must compile as minimal Windows library stubs. Preserve their configuration structs so existing config parsing keeps working, and make every Windows runtime entry point or constructor return an "unsupported on Windows" error. Do not build, package, ship, or smoke-test standalone driver binaries as Windows deliverables. Do not enable Docker, Kubernetes, Podman, VM, Docker Desktop, WSL, Hyper-V, Podman machine, Podman Desktop, or any VM-backed runtime in this build-only skill. + +### Step 5: Add Windows mise task lane + +Create a Windows-only task file at `tasks/windows.toml` and a PowerShell wrapper at `tasks/scripts/windows-msvc.ps1`. This lane is additive: do not change the existing Linux `build`, `test`, or `ci` tasks. The Linux build procedure remains the repo's default mise/Cargo path. + +The task file must expose these commands: + +| Task | Purpose | +|---|---| +| `windows:check:x64` | `cargo check --workspace --target x86_64-pc-windows-msvc` | +| `windows:check:arm64` | `cargo check --workspace --target aarch64-pc-windows-msvc` | +| `windows:build:x64` | Release-build `openshell-gateway` and `openshell` for x64 | +| `windows:build:arm64` | Release-build `openshell-gateway` and `openshell` for ARM64 | +| `windows:test:x64` | Native x64 `cargo test --workspace --no-fail-fast`, excluding unsupported driver packages as top-level workspace targets | +| `windows:test:unsupported:x64` | Focused server/runtime tests that assert unsupported Windows driver contracts without building standalone driver binaries | +| `windows:ci` | Ordered Windows check/build/test/unsupported-contract/artifact lane | + +The PowerShell wrapper must: + +1. Discover `VsDevCmd.bat` through `$env:OPENSHELL_VSDEVCMD`, `vswhere`, or standard Visual Studio install paths. +2. Add the requested rustup target before invoking Cargo. +3. Set `CARGO_INCREMENTAL=0` and keep `CARGO_TARGET_DIR` inside the fork unless the caller overrides it. +4. Clear inherited `RUSTC_WRAPPER` for this lane, because `mise run --skip-tools` intentionally does not provision `sccache`. +5. Exclude Docker, Kubernetes, Podman, and VM driver packages as top-level Windows workspace targets for check/test while still allowing their library stubs to compile as dependencies. +6. Emit logs for x64/ARM64 checks, builds, tests, and unsupported-contract tests. +7. Fail fast if a Windows-only task is invoked on a non-Windows host. + +Use `mise run --skip-tools windows:*` in GitHub Actions and local Windows automation. `--skip-tools` is intentional: this repo should not ask mise to install Rust on Windows because the MSVC flow relies on rustup plus Visual Studio Build Tools. + +### Step 6: mise check on x86_64-pc-windows-msvc + +```powershell +$env:CARGO_TARGET_DIR = "$fork\target" +mise run --skip-tools windows:check:x64 +``` + +This wraps `cargo check --workspace --target x86_64-pc-windows-msvc`. If it fails, the error log is the audit list. Iterate Step 4 through Step 6 until x64 succeeds. Common error patterns and their fixes are in [reference.md](reference.md#common-errors). + +### Step 7: mise check on aarch64-pc-windows-msvc + +```powershell +if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { + mise run --skip-tools windows:check:arm64 +} +``` + +ARM64 typically surfaces the same dependency issues as x64. If x64 passes but ARM64 fails, the fault is almost always a native dependency (a `*-sys` crate without ARM64 prebuilds) or an inline-asm block lacking aarch64 paths. Either find a pure-Rust replacement or add ARM64 to the existing cfg gate. + +### Step 8: mise build --release (both targets) + +```powershell +mise run --skip-tools windows:build:x64 +if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { + mise run --skip-tools windows:build:arm64 +} +``` + +Build only the binaries needed for build validation: `openshell-gateway` and `openshell` CLI. Skip `openshell-sandbox` — the supervisor Windows port is a follow-on skill. + +Verify the binaries exist and report their architecture: + +```powershell +Get-Item "$fork\target\x86_64-pc-windows-msvc\release\openshell-gateway.exe" +Get-Item "$fork\target\aarch64-pc-windows-msvc\release\openshell-gateway.exe" -ErrorAction SilentlyContinue +dumpbin /HEADERS "$fork\target\x86_64-pc-windows-msvc\release\openshell-gateway.exe" | Select-String "machine" +``` + +Expected machine values: `x64` for `x86_64-pc-windows-msvc`, `ARM64` for `aarch64-pc-windows-msvc`. + +### Step 9: mise test on x86_64-pc-windows-msvc (native run) + +```powershell +mise run --skip-tools windows:test:x64 +mise run --skip-tools windows:test:unsupported:x64 +``` + +Failures fall into three buckets: + +1. **Linux-only test** — gate with `#[cfg(not(target_os = "windows"))]` and add a short comment describing the Windows-equivalent test that will eventually replace it. +2. **Path or environment assumption** — fix with conditional `%APPDATA%` paths. +3. **Genuine bug** — fix or open a follow-on issue and gate. + +ARM64 tests are not run locally — they require a native ARM64 runner and are scaffolded in Step 11. + +### Step 10: Validate `$env:OPENSHELL_WXC_EXEC_PATH` (informational) + +This skill does **not** invoke `wxc-exec.exe`. The validation here is a forward-compatibility check so the follow-on MXC driver skill can rely on a known-good path. + +```powershell +if ($env:OPENSHELL_WXC_EXEC_PATH) { + if (-not (Test-Path $env:OPENSHELL_WXC_EXEC_PATH)) { + Write-Warning "OPENSHELL_WXC_EXEC_PATH set but file missing: $env:OPENSHELL_WXC_EXEC_PATH" + } else { + $arch = (dumpbin /HEADERS $env:OPENSHELL_WXC_EXEC_PATH | Select-String "machine").Line + Write-Host "wxc-exec.exe found: $env:OPENSHELL_WXC_EXEC_PATH ($arch)" + } +} else { + Write-Host "OPENSHELL_WXC_EXEC_PATH unset — MXC driver wiring is out of scope for this skill." +} +``` + +### Step 11: Scaffold ARM64 CI workflow + +Add `.github/workflows/windows-msvc.yml` (or its GitLab equivalent for the fork) with two jobs: `x64` and `arm64`. The x64 job runs on `windows-2025`; the arm64 job runs on a self-hosted runner labelled `windows-arm64`. Do not provision the runner from the skill — emit a TODO comment in the workflow file noting that runner setup is operator work. + +Template (skill writes this verbatim to `.github/workflows/windows-msvc.yml` in the fork): + +```yaml +name: Windows MSVC (build-only) +on: + push: + branches: [windows-mxc-build] + pull_request: +jobs: + x64: + runs-on: windows-2025 + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v3 + with: + install: false + experimental: true + - uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-msvc + - run: mise run --skip-tools windows:check:x64 + - run: mise run --skip-tools windows:build:x64 + - run: mise run --skip-tools windows:test:x64 + - run: mise run --skip-tools windows:test:unsupported:x64 + arm64: + # TODO: provision a windows-arm64 self-hosted runner + runs-on: [self-hosted, windows-arm64] + if: false # flip to true once the runner is online + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v3 + with: + install: false + experimental: true + - uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-pc-windows-msvc + - run: mise run --skip-tools windows:check:arm64 + - run: mise run --skip-tools windows:build:arm64 +``` + +### Step 12: Commit and report + +Commit each logical change as a Conventional Commit (per AGENTS.md). Suggested commit sequence: + +```powershell +git add Cargo.toml Cargo.lock +git commit -m "chore(windows): platform-gate Linux-only dependencies" + +git add crates/ +git commit -m "feat(windows): cfg-gate Linux-only modules for MSVC target" + +git add .github/workflows/windows-msvc.yml +git commit -m "ci(windows): scaffold x64 + arm64 MSVC workflow" + +git add tasks/windows.toml tasks/scripts/windows-msvc.ps1 +git commit -m "chore(windows): add mise MSVC task lane" +``` + +Final report should include: + +| Item | Value | +|---|---| +| Host preconditions | "OK" or the warning list captured in Step 1 | +| Fork directory | `$env:OPENSHELL_MXC_FORK_DIR` | +| Branch | `$env:OPENSHELL_MXC_FORK_BRANCH` | +| x64 binary | `target\x86_64-pc-windows-msvc\release\openshell-gateway.exe` (size, sha256) | +| ARM64 binary | `target\aarch64-pc-windows-msvc\release\openshell-gateway.exe` (size, sha256) or "skipped" | +| Test summary | passed / failed / gated-out from `test-x64.log` | +| Windows mise lane | status of `windows:check:*`, `windows:build:*`, `windows:test:x64`, and `windows:test:unsupported:x64` | +| Gated modules | list of crates and modules with new `#[cfg(...)]` guards | +| `wxc-exec.exe` | path and architecture, or "unset" | +| Next skills to run | follow-on driver and policy-translation skills | + +## Additional resources + +- [reference.md](reference.md) — Unix dependency audit, common cargo errors and fixes, cfg gating patterns diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md new file mode 100644 index 0000000000..d7dcc16240 --- /dev/null +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -0,0 +1,184 @@ +# Reference: Unix dependency audit and cfg gating patterns + +Companion to [SKILL.md](SKILL.md). Use these tables and patterns when Step 4 (apply minimum Windows compatibility shims) hits a specific error during `cargo check --target x86_64-pc-windows-msvc`. + +## Linux-only workspace dependencies + +These crates from the OpenShell workspace `Cargo.toml` do not build on `x86_64-pc-windows-msvc`. Gate them out of the Windows build graph in each consuming crate's `Cargo.toml`. + +| Crate | Used by | Windows fix | +|---|---|---| +| `nix` (features: signal, process, user, fs, term) | `openshell-sandbox`, `openshell-driver-vm`, `openshell-cli` | Move to `[target.'cfg(unix)'.dependencies]` in each consumer | +| `rustix` (features: process) | `openshell-server`, `openshell-sandbox` | Has Windows support — usually compiles; only gate the call sites that use Unix-only functions | +| `landlock` (if present in any crate) | `openshell-sandbox` | `[target.'cfg(target_os = "linux")'.dependencies]` | +| `libseccomp` / `seccompiler` | `openshell-sandbox` | `[target.'cfg(target_os = "linux")'.dependencies]` | +| `caps` | `openshell-sandbox` | `[target.'cfg(unix)'.dependencies]` | +| `procfs` | `openshell-sandbox`, `openshell-driver-vm` | `[target.'cfg(target_os = "linux")'.dependencies]` | +| `libkrun-sys` (transitive via VM driver) | `openshell-driver-vm` | Move Linux implementation behind non-Windows cfg and expose a Windows stub that returns unsupported | + +Compute drivers that must remain unsupported on Windows: + +- `openshell-driver-docker` - preserve config parsing, but Windows runtime construction returns unsupported +- `openshell-driver-podman` - preserve config parsing, but Windows runtime construction returns unsupported +- `openshell-driver-vm` - preserve config parsing, but Windows VM spawn returns unsupported +- `openshell-driver-kubernetes` - preserve config parsing, but Windows runtime construction returns unsupported + +The build-only slice does not need Docker, Kubernetes, Podman, or VM runtime support. Keep these crates in the Windows build graph as library stubs so config files still deserialize and the gateway can return clear unsupported errors. Do not build, package, ship, or smoke-test standalone driver binaries as Windows deliverables, and do not enable Docker Desktop, WSL, Hyper-V, Kubernetes, Podman machine, Podman Desktop, or any VM-backed runtime. + +## Common errors and fixes + +### `error[E0432]: unresolved import 'tokio::net::UnixListener'` + +Wrap the import and all use sites: + +```rust +#[cfg(unix)] +use tokio::net::{UnixListener, UnixStream}; +``` + +For the Windows side, leave a compile-time stub: + +```rust +#[cfg(target_os = "windows")] +pub fn bind_driver_socket(_path: &str) -> anyhow::Result<()> { + anyhow::bail!("named-pipe driver IPC not implemented"); +} +``` + +### `error: failed to run custom build command for 'libseccomp-sys'` + +The crate has no Windows backend. Gate it out: + +```toml +# in crates/openshell-sandbox/Cargo.toml +[target.'cfg(target_os = "linux")'.dependencies] +libseccomp = "..." +``` + +Then in `src/lib.rs`: + +```rust +#[cfg(target_os = "linux")] +mod seccomp; +``` + +### `error: linking with 'link.exe' failed: exit code: 1120` referencing `nix_*` symbols + +A `use nix::...` is reachable from the Windows build path. Either gate the `use` with `#[cfg(unix)]` or move the function into a Unix-only module. + +### `error: the trait bound 'PathBuf: From<&str>' is not satisfied` on Windows-only paths + +Usually caused by hardcoded forward-slash paths. Replace with `PathBuf::from()` and rely on `std::path::MAIN_SEPARATOR` or use `dirs::data_local_dir()` for `%APPDATA%`. + +### `error[E0599]: no method 'set_nonblocking' found` on `std::os::unix::net::UnixStream` + +Same pattern — gate with `#[cfg(unix)]`. + +### `cargo:warning=libsecret-sys ... not found` + +`libsecret` only works on Linux. If `openshell-providers` pulls it in, gate it. The build-only skill does not need credential storage on Windows; that is a follow-on skill. + +## Cfg gating patterns + +### Module-level + +```rust +// crates/openshell-sandbox/src/sandbox/mod.rs +#[cfg(target_os = "linux")] +mod linux; + +#[cfg(target_os = "windows")] +mod windows; + +#[cfg(target_os = "linux")] +pub use linux::*; + +#[cfg(target_os = "windows")] +pub use windows::*; +``` + +### Cargo dependency-level + +```toml +[target.'cfg(target_os = "linux")'.dependencies] +landlock = "0.4" +libseccomp = "0.3" + +[target.'cfg(target_os = "windows")'.dependencies] +windows-sys = { version = "0.59", features = ["Win32_Storage_FileSystem", "Win32_System_Pipes"] } +``` + +Do **not** add the `windows-sys` dependency in this build-only skill unless a specific compile error demands it. The follow-on MXC driver skill will introduce the Windows API surface. + +### Path defaults + +```rust +pub fn default_state_dir() -> PathBuf { + #[cfg(target_os = "linux")] + { + dirs::data_local_dir() + .unwrap_or_else(|| PathBuf::from("/var/lib")) + .join("openshell") + } + #[cfg(target_os = "windows")] + { + // %APPDATA%\OpenShell + dirs::config_dir() + .unwrap_or_else(|| PathBuf::from(r"C:\ProgramData")) + .join("OpenShell") + } + #[cfg(not(any(target_os = "linux", target_os = "windows")))] + { + PathBuf::from(".openshell") + } +} +``` + +### Test gating + +```rust +#[cfg(not(target_os = "windows"))] +#[test] +fn linux_landlock_smoke() { + // ... +} + +#[cfg(target_os = "windows")] +#[test] +#[ignore = "supervisor Windows port is a follow-on skill"] +fn windows_supervisor_smoke() { + unimplemented!() +} +``` + +## Linux build must stay green + +After every cfg gate change, run `cargo check --workspace` on the Linux baseline before committing. The skill's commits must not regress Linux. Use a separate clone or WSL session if running on a Windows-only host, or rely on the existing Linux CI to fail the PR. + +## Windows mise lane expectations + +The Windows build path is additive. Keep the repository's Linux `mise run ci`, default Cargo tasks, and Linux documentation unchanged. Windows automation lives in `tasks/windows.toml` and delegates to `tasks/scripts/windows-msvc.ps1`. + +| Task | Expected behavior | +|---|---| +| `mise run --skip-tools windows:check:x64` | Runs x64 MSVC `cargo check --workspace` | +| `mise run --skip-tools windows:check:arm64` | Runs ARM64 MSVC `cargo check --workspace` | +| `mise run --skip-tools windows:build:x64` | Builds release `openshell-gateway.exe` and `openshell.exe` for x64 | +| `mise run --skip-tools windows:build:arm64` | Builds release `openshell-gateway.exe` and `openshell.exe` for ARM64 | +| `mise run --skip-tools windows:test:x64` | Runs native x64 workspace tests | +| `mise run --skip-tools windows:test:unsupported:x64` | Verifies unsupported driver contracts through server/runtime tests without building standalone driver binaries | +| `mise run --skip-tools windows:ci` | Runs the full Windows lane in order | + +Use `--skip-tools` for Windows CI and automation. Rust must come from rustup with MSVC targets, and Visual Studio Build Tools must provide the linker and SDK. Because `--skip-tools` does not provision mise-managed tools, the Windows wrapper clears inherited `RUSTC_WRAPPER=sccache` before invoking Cargo. The wrapper excludes unsupported driver packages as top-level workspace targets for Windows check/test, but those library stubs still compile when the gateway depends on them. The wrapper may discover `VsDevCmd.bat`, but it must not install Visual Studio, Rust, Docker, Kubernetes, Podman, VM tooling, WSL, or Hyper-V. + +## What NOT to do in this skill + +- Do not implement named-pipe IPC. Stubs only. (Belongs to the follow-on MXC driver skill.) +- Do not add Windows Credential Manager integration. (Follow-on skill.) +- Do not implement DPAPI encryption. (Follow-on skill.) +- Do not create a new MXC driver crate. (Separate skill.) +- Do not write OpenShell → MXC JSON translation. (Separate skill.) +- Do not build MSI installers. (Follow-on skill.) +- Do not modify the `openshell-sandbox` supervisor for Windows beyond cfg-gating to compile. The full port is a follow-on skill. + +The success criterion is **the workspace compiles and basic tests pass on Windows MSVC for both architectures, with the Linux build unchanged**. Nothing more. From f411ac28dfde52bdf83a6ac634d4dc3d4d5cb787 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Mon, 8 Jun 2026 11:32:57 -0500 Subject: [PATCH 06/30] feat(windows): add Windows build support Signed-off-by: Akber Raza --- CONTRIBUTING.md | 52 ++++++++++++++++++++------------ Cargo.lock | 19 ------------ Cargo.toml | 3 +- crates/openshell-cli/Cargo.toml | 4 ++- crates/openshell-cli/src/main.rs | 29 ++++++++++++++++-- crates/openshell-cli/src/run.rs | 43 +++++++++++++------------- crates/openshell-cli/src/ssh.rs | 2 ++ crates/openshell-core/Cargo.toml | 5 --- crates/openshell-core/build.rs | 9 +++--- 9 files changed, 90 insertions(+), 76 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0465f88f7d..8d57e50c7f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,30 +130,18 @@ Project requirements: - Rust 1.90+ - Python 3.11+ - Docker (running) -- Z3 solver library (for the policy prover crate) -### macOS build tools - -Install Apple Command Line Tools before building locally: - -```bash -xcode-select --install -``` - -If Cargo fails while building `protobuf-src` with an error such as -`fatal error: 'utility' file not found`, `fatal error: 'cstdlib' file not -found`, or `A compiler with support for C++11 language features is required`, -your Command Line Tools install may not expose the libc++ headers on the -compiler's default include path. Reinstall Command Line Tools to correct the error: +The common local development workflow runs on macOS and Linux. The +`openshell-cli` crate also supports x86-64 Windows builds with the MSVC toolchain +(`x86_64-pc-windows-msvc`). Windows builds require the Visual Studio C++ build +tools and LLVM/libclang for crates that generate C bindings. -```bash -sudo rm -rf /Library/Developer/CommandLineTools -xcode-select --install -``` +The policy prover uses Z3 on supported targets. ### Z3 installation -The `openshell-prover` crate links against the system Z3 library via pkg-config. +The `openshell-prover` crate links against Z3. On macOS and Linux, install the +system Z3 development package; `z3-sys` discovers it through `pkg-config`. ```bash # macOS @@ -166,12 +154,36 @@ sudo apt install libz3-dev sudo dnf install z3-devel ``` -If you prefer not to install Z3 system-wide, you can compile it from source as a one-time step: +If you prefer not to install Z3 system-wide, use the bundled Z3 feature. This +compiles Z3 from source during the Rust build: ```bash cargo build -p openshell-prover --features bundled-z3 ``` +For x86-64 Windows MSVC builds, use one of these Z3 paths: + +- System Z3: point `Z3_LIBRARY_PATH_OVERRIDE` at the directory containing the + 64-bit MSVC Z3 library and `Z3_SYS_Z3_HEADER` at `z3.h`. +- Bundled Z3: pass `--features bundled-z3` so `z3-sys` builds Z3 from source. + +Both Windows paths still require `libclang.dll` for `bindgen`. If LLVM is not on +the default search path, set `LIBCLANG_PATH` to the directory containing +`libclang.dll`. + +```powershell +$env:LIBCLANG_PATH='C:\Program Files\Microsoft Visual Studio\2022\\VC\Tools\Llvm\x64\bin' +cargo build -p openshell-cli --target x86_64-pc-windows-msvc --features bundled-z3 +``` + +### macOS build tools + +Install Apple Command Line Tools before building locally: + +```bash +xcode-select --install +``` + ## Getting Started ```bash diff --git a/Cargo.lock b/Cargo.lock index 803b544b3e..e2ee2687dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,15 +258,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "autotools" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" -dependencies = [ - "cc", -] - [[package]] name = "aws-config" version = "1.8.15" @@ -3893,7 +3884,6 @@ dependencies = [ "nix", "prost", "prost-types", - "protobuf-src", "protoc-bin-vendored", "reqwest 0.12.28", "serde", @@ -4946,15 +4936,6 @@ dependencies = [ "prost", ] -[[package]] -name = "protobuf-src" -version = "1.1.0+21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" -dependencies = [ - "autotools", -] - [[package]] name = "protoc-bin-vendored" version = "3.2.0" diff --git a/Cargo.toml b/Cargo.toml index 40bc57ee67..139896ecfc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ tonic-prost-build = "0.14" prost = "0.14" prost-types = "0.14" prost-reflect = { version = "0.16.5", features = ["serde"] } -protoc-bin-vendored = "3.2.0" # HTTP server axum = { version = "0.8", features = ["ws"] } @@ -109,7 +108,7 @@ futures = "0.3" bytes = "1" pin-project-lite = "0.2" tokio-stream = "0.1" -protobuf-src = "1.1.0" +protoc-bin-vendored = "3.2.0" url = "2" indexmap = "2" diff --git a/crates/openshell-cli/Cargo.toml b/crates/openshell-cli/Cargo.toml index d7b8fd502c..0d1cdb531f 100644 --- a/crates/openshell-cli/Cargo.toml +++ b/crates/openshell-cli/Cargo.toml @@ -72,7 +72,6 @@ tokio-tungstenite = { workspace = true } # Streams futures = { workspace = true } tokio-stream = { workspace = true } -nix = { workspace = true } # URL parsing url = { workspace = true } @@ -84,6 +83,9 @@ tracing-subscriber = { workspace = true } [lints] workspace = true +[target.'cfg(unix)'.dependencies] +nix = { workspace = true } + [dev-dependencies] futures = { workspace = true } rcgen = { version = "0.13", features = ["crypto", "pem"] } diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 00942d79ec..9f117f7407 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -2131,9 +2131,32 @@ enum WorkspaceMemberCommands { }, } -#[tokio::main] -#[allow(clippy::large_stack_frames)] // CLI dispatch holds many futures; OK at top level. -async fn main() -> Result<()> { +#[cfg(windows)] +fn main() -> Result<()> { + std::thread::Builder::new() + .name("openshell-main".to_string()) + .stack_size(8 * 1024 * 1024) + .spawn(run_main) + .map_err(|err| miette::miette!("failed to start OpenShell main thread: {err}"))? + .join() + .map_err(|_| miette::miette!("OpenShell main thread panicked"))? +} + +#[cfg(not(windows))] +fn main() -> Result<()> { + run_main() +} + +fn run_main() -> Result<()> { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|err| miette::miette!("failed to build Tokio runtime: {err}"))? + .block_on(run_async()) +} + +#[allow(clippy::large_stack_frames)] // CLI dispatch holds many futures; run on an expanded Windows stack. +async fn run_async() -> Result<()> { // Install the rustls crypto provider before completion runs — completers may // establish TLS connections to the gateway. rustls::crypto::ring::default_provider() diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 834e9cd7fb..c8bbabbdf8 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1811,31 +1811,28 @@ async fn sandbox_exec_interactive_grpc( // spawn_blocking) so the tokio runtime shutdown doesn't wait for a // thread blocked on stdin.read(). The thread exits when the channel // closes (blocking_send returns Err) or stdin hits EOF. - #[cfg(unix)] - { - let stdin_tx = input_tx.clone(); - std::thread::spawn(move || { - let mut stdin = std::io::stdin().lock(); - let mut buf = [0u8; 4096]; - loop { - match stdin.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - if stdin_tx - .blocking_send(ExecSandboxInput { - payload: Some(exec_sandbox_input::Payload::Stdin( - buf[..n].to_vec(), - )), - }) - .is_err() - { - break; - } + let stdin_tx = input_tx.clone(); + std::thread::spawn(move || { + let mut stdin = std::io::stdin().lock(); + let mut buf = [0u8; 4096]; + loop { + match stdin.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + if stdin_tx + .blocking_send(ExecSandboxInput { + payload: Some(exec_sandbox_input::Payload::Stdin( + buf[..n].to_vec(), + )), + }) + .is_err() + { + break; } } } - }); - } + } + }); // SIGWINCH handler: forward terminal resize events. #[cfg(unix)] @@ -6930,6 +6927,8 @@ mod tests { refresh_status_row, resolve_from, sandbox_should_persist, sandbox_upload_plan, service_expose_status_error, service_url_for_gateway, }; + #[cfg(unix)] + use super::{local_upload_path_exists, local_upload_path_is_symlink}; use crate::TEST_ENV_LOCK; use crate::commands::common::progress_step_from_metadata; use crate::test_utils::EnvVarGuard; diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index f8de99a692..4b16240bcb 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -2233,7 +2233,9 @@ mod tests { #[derive(Debug)] struct UploadArchiveEntry { path: String, + #[cfg_attr(not(unix), allow(dead_code))] entry_type: tar::EntryType, + #[cfg_attr(not(unix), allow(dead_code))] link_name: Option, } diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index 1f7cbf60cb..2fd737508b 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -40,11 +40,6 @@ telemetry = ["dep:reqwest", "dep:chrono"] [build-dependencies] tonic-prost-build = { workspace = true } - -[target.'cfg(not(target_os = "windows"))'.build-dependencies] -protobuf-src = { workspace = true } - -[target.'cfg(target_os = "windows")'.build-dependencies] protoc-bin-vendored = { workspace = true } [dev-dependencies] diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 26cbedbca3..22fc371025 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -22,14 +22,15 @@ fn main() -> Result<(), Box> { // --- Protobuf compilation --- // Re-run when anything under proto/ changes (including newly added .proto files). println!("cargo:rerun-if-changed={PROTO_REL}"); - // Use a bundled protoc. The system protoc (from apt-get) does not bundle - // the well-known type includes (google/protobuf/struct.proto etc.), so the - // build script picks a vendored provider per platform. + // Use a vendored protoc binary and include tree. System protoc installs + // often omit the well-known type includes (google/protobuf/struct.proto, + // etc.), and protobuf-src requires autotools/sh which breaks MSVC builds. // SAFETY: This is run at build time in a single-threaded build script context. // No other threads are reading environment variables concurrently. #[allow(unsafe_code)] unsafe { - env::set_var("PROTOC", protoc_path()?); + env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?); + env::set_var("PROTOC_INCLUDE", protoc_bin_vendored::include_path()?); } let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); From e8a12261ef91d5bcb8cf6648b43c82c3ab73fcab Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Mon, 8 Jun 2026 16:59:15 -0500 Subject: [PATCH 07/30] refactor(windows): consolidate Windows-specific dependencies and improve build logic Signed-off-by: Akber Raza --- crates/openshell-core/build.rs | 10 - crates/openshell-prover/Cargo.toml | 5 +- crates/openshell-prover/src/lib.rs | 299 +++++++++++++++++++++++++++-- 3 files changed, 288 insertions(+), 26 deletions(-) diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 22fc371025..4a363892a3 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -65,16 +65,6 @@ fn main() -> Result<(), Box> { Ok(()) } -#[cfg(target_os = "windows")] -fn protoc_path() -> Result> { - Ok(protoc_bin_vendored::protoc_bin_path()?) -} - -#[cfg(not(target_os = "windows"))] -fn protoc_path() -> Result> { - Ok(protobuf_src::protoc()) -} - fn proto_include_dirs(proto_root: &Path) -> Result, Box> { let mut includes = vec![proto_root.to_path_buf()]; #[cfg(target_os = "windows")] diff --git a/crates/openshell-prover/Cargo.toml b/crates/openshell-prover/Cargo.toml index 3175acf36c..ee815f3a3f 100644 --- a/crates/openshell-prover/Cargo.toml +++ b/crates/openshell-prover/Cargo.toml @@ -13,10 +13,7 @@ repository.workspace = true [features] bundled-z3 = ["z3/bundled"] -[target.'cfg(target_os = "windows")'.dependencies] -miette = { workspace = true } - -[target.'cfg(not(target_os = "windows"))'.dependencies] +[dependencies] z3 = { workspace = true } serde = { workspace = true } serde_yml = { workspace = true } diff --git a/crates/openshell-prover/src/lib.rs b/crates/openshell-prover/src/lib.rs index 15e4f633f6..2267052048 100644 --- a/crates/openshell-prover/src/lib.rs +++ b/crates/openshell-prover/src/lib.rs @@ -1,18 +1,293 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +//! Formal policy verification for `OpenShell` sandboxes. +//! +//! Encodes sandbox policies, binary capabilities, and credential scopes as Z3 +//! SMT constraints, then checks reachability queries to detect data exfiltration +//! paths and write-bypass violations. -#[cfg(target_os = "windows")] +pub mod accepted_risks; +pub mod credentials; +pub mod finding; +pub mod model; +pub mod policy; +pub mod queries; +pub mod registry; +pub mod report; + +use std::path::Path; + +use miette::Result; + +use accepted_risks::{apply_accepted_risks, load_accepted_risks}; +use model::build_model; +use policy::parse_policy; +use queries::run_all_queries; +use report::{render_compact, render_report}; + +/// Run the prover end-to-end and return an exit code. +/// +/// - `0` — pass (no critical/high findings, or all accepted) +/// - `1` — fail (critical or high findings present) +/// - `2` — input error +/// +/// Binary and API capability registries are embedded at compile time. +/// Pass `registry_dir` to override with a custom filesystem registry. pub fn prove( - _policy_path: &str, - _credentials_path: &str, - _registry_dir: Option<&str>, - _accepted_risks_path: Option<&str>, - _compact: bool, -) -> miette::Result { - Err(miette::miette!( - "policy prover is not available on Windows in this build" - )) + policy_path: &str, + credentials_path: &str, + registry_dir: Option<&str>, + accepted_risks_path: Option<&str>, + compact: bool, +) -> Result { + let policy = parse_policy(Path::new(policy_path))?; + + let (credential_set, binary_registry) = match registry_dir { + Some(dir) => { + let dir = Path::new(dir); + ( + credentials::load_credential_set_from_dir(Path::new(credentials_path), dir)?, + registry::load_binary_registry_from_dir(dir)?, + ) + } + None => ( + credentials::load_credential_set_embedded(Path::new(credentials_path))?, + registry::load_embedded_binary_registry()?, + ), + }; + + let z3_model = build_model(policy, credential_set, binary_registry); + let mut findings = run_all_queries(&z3_model); + + if let Some(ar_path) = accepted_risks_path { + let accepted = load_accepted_risks(Path::new(ar_path))?; + findings = apply_accepted_risks(findings, &accepted); + } + + let exit_code = if compact { + render_compact(&findings, policy_path, credentials_path) + } else { + render_report(&findings, policy_path, credentials_path) + }; + + Ok(exit_code) +} + +// =========================================================================== +// Tests +// =========================================================================== + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn testdata_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") + } + + // 1. Parse testdata/policy.yaml, verify structure. + #[test] + fn test_parse_policy() { + let path = testdata_dir().join("policy.yaml"); + let model = parse_policy(&path).expect("failed to parse policy"); + assert_eq!(model.version, 1); + assert!(model.network_policies.contains_key("github_api")); + let rule = &model.network_policies["github_api"]; + assert_eq!(rule.name, "github-api"); + assert_eq!(rule.endpoints.len(), 2); + assert!(rule.binaries.len() >= 4); + } + + // 2. Verify readable_paths. + #[test] + fn test_filesystem_policy() { + let path = testdata_dir().join("policy.yaml"); + let model = parse_policy(&path).expect("failed to parse policy"); + let readable = model.filesystem_policy.readable_paths(); + assert!(readable.contains(&"/usr".to_owned())); + assert!(readable.contains(&"/sandbox".to_owned())); + assert!(readable.contains(&"/tmp".to_owned())); + } + + // 3. Workdir NOT included by default (matches runtime behavior). + #[test] + fn test_include_workdir_default() { + let yaml = r" +version: 1 +filesystem_policy: + read_only: + - /usr +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model.filesystem_policy.readable_paths(); + assert!(!readable.contains(&"/sandbox".to_owned())); + } + + // 4. Workdir excluded when include_workdir: false. + #[test] + fn test_include_workdir_false() { + let yaml = r" +version: 1 +filesystem_policy: + include_workdir: false + read_only: + - /usr +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model.filesystem_policy.readable_paths(); + assert!(!readable.contains(&"/sandbox".to_owned())); + } + + // 5. No duplicate when workdir already in read_write. + #[test] + fn test_include_workdir_no_duplicate() { + let yaml = r" +version: 1 +filesystem_policy: + include_workdir: true + read_write: + - /sandbox + - /tmp +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model.filesystem_policy.readable_paths(); + let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); + assert_eq!(sandbox_count, 1); + } + + // 6. End-to-end: testdata policy with a github credential in scope and a + // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. + // The prover output is categorical, not severity-graded. + #[test] + fn test_findings_for_github_policy() { + use finding::category; + + let policy_path = testdata_dir().join("policy.yaml"); + let creds_path = testdata_dir().join("credentials.yaml"); + + let pol = parse_policy(&policy_path).expect("parse policy"); + let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(pol, cred_set, bin_reg); + let findings = run_all_queries(&z3_model); + + let categories: std::collections::HashSet<&str> = + findings.iter().map(|f| f.query.as_str()).collect(); + assert!( + categories.contains(category::L7_BYPASS_CREDENTIALED), + "expected l7_bypass_credentialed finding for bypass-L7 binary with credential in scope; \ + got categories: {categories:?}" + ); + // Every emitted category must be one of the four v1 categories. + let allowed: std::collections::HashSet<&str> = [ + category::LINK_LOCAL_REACH, + category::L7_BYPASS_CREDENTIALED, + category::CREDENTIAL_REACH_EXPANSION, + category::CAPABILITY_EXPANSION, + ] + .into_iter() + .collect(); + for c in &categories { + assert!( + allowed.contains(*c), + "unexpected category {c} emitted by the prover" + ); + } + } + + #[test] + fn test_wildcard_endpoint_covering_credential_host_emits_credential_reach() { + use finding::{FindingPath, category}; + + let policy = policy::parse_policy_str( + r#" +version: 1 +network_policies: + github_wildcard: + name: github-wildcard + endpoints: + - host: "*.github.com" + port: 443 + protocol: rest + enforcement: enforce + access: read-write + binaries: + - path: /usr/bin/curl +"#, + ) + .expect("parse policy"); + let cred_set = + credentials::load_credential_set_embedded(&testdata_dir().join("credentials.yaml")) + .expect("load creds"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(policy, cred_set, bin_reg); + let findings = run_all_queries(&z3_model); + + let reach = findings + .iter() + .find(|finding| finding.query == category::CREDENTIAL_REACH_EXPANSION) + .expect("wildcard host covering api.github.com must be credentialed"); + assert!(reach.paths.iter().any(|path| matches!( + path, + FindingPath::Exfil(exfil) + if exfil.endpoint_host == "*.github.com" && exfil.binary == "/usr/bin/curl" + ))); + } + + #[test] + fn test_known_metadata_hostname_emits_link_local_finding() { + use finding::{FindingPath, category}; + + let policy = policy::parse_policy_str( + r" +version: 1 +network_policies: + metadata: + name: metadata + endpoints: + - host: metadata.google.internal + port: 80 + binaries: + - path: /usr/bin/curl +", + ) + .expect("parse policy"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(policy, credentials::CredentialSet::default(), bin_reg); + let findings = run_all_queries(&z3_model); + + let link_local = findings + .iter() + .find(|finding| finding.query == category::LINK_LOCAL_REACH) + .expect("known metadata hostname must emit link-local/metadata finding"); + assert!(link_local.paths.iter().any(|path| matches!( + path, + FindingPath::Exfil(exfil) + if exfil.endpoint_host == "metadata.google.internal" + ))); + } + + // 7. Empty policy produces no findings. + #[test] + fn test_empty_policy_no_findings() { + let policy_path = testdata_dir().join("empty-policy.yaml"); + let creds_path = testdata_dir().join("credentials.yaml"); + + let pol = parse_policy(&policy_path).expect("parse policy"); + let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); + let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); + + let z3_model = build_model(pol, cred_set, bin_reg); + let findings = run_all_queries(&z3_model); + + assert!( + findings.is_empty(), + "deny-all policy should produce no findings, got: {findings:?}" + ); + } } From cd12c155497205baf1a57babb1d83db14331f52e Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Tue, 9 Jun 2026 15:38:02 -0500 Subject: [PATCH 08/30] feat(windows): add libclang path resolution and update cargo commands with bundled Z3 features Signed-off-by: Akber Raza --- tasks/scripts/windows-msvc.ps1 | 60 +++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index ad67b4b067..129739cd5a 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -38,6 +38,8 @@ if ([string]::IsNullOrWhiteSpace($TargetDir)) { } $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm" +$BundledZ3WorkspaceFeatures = "--features openshell-cli/bundled-z3,openshell-prover/bundled-z3" +$BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-prover/bundled-z3" function Resolve-VsDevCmd { if ($env:OPENSHELL_VSDEVCMD -and (Test-Path $env:OPENSHELL_VSDEVCMD)) { @@ -77,6 +79,54 @@ function Resolve-VsDevCmd { throw "Could not find VsDevCmd.bat. Install Visual Studio Build Tools, or set OPENSHELL_VSDEVCMD." } +function Resolve-LibclangPath { + if ($env:LIBCLANG_PATH) { + $candidate = Join-Path $env:LIBCLANG_PATH "libclang.dll" + if (Test-Path $candidate) { + return (Resolve-Path $env:LIBCLANG_PATH).Path + } + throw "LIBCLANG_PATH is set but libclang.dll was not found at: $candidate" + } + + $programFilesX86 = [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + if ($programFilesX86) { + $vswhere = Join-Path $programFilesX86 "Microsoft Visual Studio\Installer\vswhere.exe" + } else { + $vswhere = $null + } + if ($vswhere -and (Test-Path $vswhere)) { + $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -find "VC\Tools\Llvm\x64\bin\libclang.dll" | Select-Object -First 1 + if ($found -and (Test-Path $found)) { + return (Split-Path -Parent (Resolve-Path $found).Path) + } + } + + $programFiles = @( + [Environment]::GetEnvironmentVariable("ProgramFiles"), + [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + ) | Where-Object { $_ } + $versions = @("2022", "18", "17") + $editions = @("Enterprise", "Professional", "Community", "BuildTools") + foreach ($root in $programFiles) { + foreach ($version in $versions) { + foreach ($edition in $editions) { + $candidateDir = Join-Path $root "Microsoft Visual Studio\$version\$edition\VC\Tools\Llvm\x64\bin" + $candidate = Join-Path $candidateDir "libclang.dll" + if (Test-Path $candidate) { + return (Resolve-Path $candidateDir).Path + } + } + } + } + + $llvmDir = "C:\Program Files\LLVM\bin" + if (Test-Path (Join-Path $llvmDir "libclang.dll")) { + return (Resolve-Path $llvmDir).Path + } + + throw "Could not find libclang.dll. Install Visual Studio C++ Clang tools, or set LIBCLANG_PATH to the directory containing libclang.dll." +} + function Get-HostArch { switch ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString()) { "Arm64" { "arm64" } @@ -139,14 +189,14 @@ function Invoke-VsCargo { function Invoke-Check([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget" ` + -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget $BundledZ3WorkspaceFeatures" ` -LogName "build-$RustTarget-check.log" } function Invoke-Build([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell" ` + -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell $BundledZ3WorkspaceFeatures" ` -LogName "build-$RustTarget-release.log" } @@ -156,7 +206,7 @@ function Invoke-Test([string] $RustTarget) { } Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast" ` + -CargoArgs "cargo test --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast $BundledZ3WorkspaceFeatures" ` -LogName "test-$RustTarget.log" } @@ -172,7 +222,7 @@ function Invoke-UnsupportedContractTests([string] $RustTarget) { foreach ($test in $tests) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test -p openshell-server --target $RustTarget $test" ` + -CargoArgs "cargo test -p openshell-server --target $RustTarget $test $BundledZ3ServerFeatures" ` -LogName "test-$RustTarget-unsupported-$test.log" } } @@ -204,6 +254,8 @@ function Show-Artifacts([string[]] $RustTargets) { } $targets = Get-SelectedTargets $Target +$env:LIBCLANG_PATH = Resolve-LibclangPath +Write-Host "==> LIBCLANG_PATH=$env:LIBCLANG_PATH" switch ($Action) { "check" { From 7541fb4b7729785a238e2dcfd18bdb8df9217ca4 Mon Sep 17 00:00:00 2001 From: Giedrius Burachas Date: Tue, 9 Jun 2026 17:09:19 -0700 Subject: [PATCH 09/30] chore(tooling): lock Windows tool artifacts Signed-off-by: Giedrius Burachas Signed-off-by: Akber Raza --- mise.lock | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/mise.lock b/mise.lock index a3864976c7..eea215b82a 100644 --- a/mise.lock +++ b/mise.lock @@ -22,6 +22,11 @@ checksum = "sha256:d5255ead3ac861a11c785bf19d4f70f16e59ac8b9519c312da61213d3d573 url = "https://github.com/EmbarkStudios/cargo-about/releases/download/0.8.4/cargo-about-0.8.4-aarch64-apple-darwin.tar.gz" url_api = "https://api.github.com/repos/EmbarkStudios/cargo-about/releases/assets/324268695" +[tools."github:EmbarkStudios/cargo-about"."platforms.windows-x64"] +checksum = "sha256:d5c38fb914bbad57c6a7d58c4847315bc3fe11efbe4fb51b3c515f997a56ceb7" +url = "https://github.com/EmbarkStudios/cargo-about/releases/download/0.8.4/cargo-about-0.8.4-x86_64-pc-windows-msvc.tar.gz" +url_api = "https://api.github.com/repos/EmbarkStudios/cargo-about/releases/assets/324269449" + [[tools."github:anchore/syft"]] version = "1.44.0" backend = "github:anchore/syft" @@ -44,6 +49,12 @@ url = "https://github.com/anchore/syft/releases/download/v1.44.0/syft_1.44.0_dar url_api = "https://api.github.com/repos/anchore/syft/releases/assets/410001187" provenance = "github-attestations" +[tools."github:anchore/syft"."platforms.windows-x64"] +checksum = "sha256:195e786eb84ec145854f20528992e86637c77d1968731dfe6ce850c90e28f47a" +url = "https://github.com/anchore/syft/releases/download/v1.44.0/syft_1.44.0_windows_amd64.zip" +url_api = "https://api.github.com/repos/anchore/syft/releases/assets/410001172" +provenance = "github-attestations" + [[tools."github:mozilla/sccache"]] version = "0.14.0" backend = "github:mozilla/sccache" @@ -63,6 +74,11 @@ checksum = "sha256:a781e8018260ab128e7690d8497736fa231b6ca895d57131d5b5b966ca987 url = "https://github.com/mozilla/sccache/releases/download/v0.14.0/sccache-v0.14.0-aarch64-apple-darwin.tar.gz" url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/353135984" +[tools."github:mozilla/sccache"."platforms.windows-x64"] +checksum = "sha256:74a3ffd4207e8e0e62af7747bd03b42deab0f6dabc7ef0a8cdd950f83f1037c8" +url = "https://github.com/mozilla/sccache/releases/download/v0.14.0/sccache-v0.14.0-x86_64-pc-windows-msvc.zip" +url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/353136140" + [[tools."github:mozilla/sccache"]] version = "0.14.0" backend = "github:mozilla/sccache" @@ -94,6 +110,11 @@ checksum = "sha256:29caf036bdbb4e6f07afea31706b6f386cb5a4db9a46a3a8b462b9b78157e url = "https://github.com/rust-cross/cargo-zigbuild/releases/download/v0.22.3/cargo-zigbuild-aarch64-apple-darwin.tar.xz" url_api = "https://api.github.com/repos/rust-cross/cargo-zigbuild/releases/assets/405676922" +[tools."github:rust-cross/cargo-zigbuild"."platforms.windows-x64"] +checksum = "sha256:675804464634cf068dc206e9fafc1bd4557ffcc0b1638a1ff246d899b776fe35" +url = "https://github.com/rust-cross/cargo-zigbuild/releases/download/v0.22.3/cargo-zigbuild-x86_64-pc-windows-msvc.zip" +url_api = "https://api.github.com/repos/rust-cross/cargo-zigbuild/releases/assets/405676944" + [[tools.helm]] version = "4.2.0" backend = "aqua:helm/helm" @@ -110,6 +131,10 @@ url = "https://get.helm.sh/helm-v4.2.0-linux-amd64.tar.gz" checksum = "sha256:f13f959015447b6bc309f9fd506509926543988a39035c088b52522ec95e2acb" url = "https://get.helm.sh/helm-v4.2.0-darwin-arm64.tar.gz" +[tools.helm."platforms.windows-x64"] +checksum = "sha256:34bf9659f8f04f3841a60131183b8e2acc44e260db4c93b889ff09718cacca6f" +url = "https://get.helm.sh/helm-v4.2.0-windows-amd64.tar.gz" + [[tools.helm-docs]] version = "1.14.2" backend = "aqua:norwoodj/helm-docs" @@ -126,6 +151,10 @@ url = "https://github.com/norwoodj/helm-docs/releases/download/v1.14.2/helm-docs checksum = "sha256:2d8399db5b33d240d5f8985241bcf5483563150b968e3229823822979f3e4b8b" url = "https://github.com/norwoodj/helm-docs/releases/download/v1.14.2/helm-docs_1.14.2_Darwin_arm64.tar.gz" +[tools.helm-docs."platforms.windows-x64"] +checksum = "sha256:3c54fd78d99e2769cf83a0faf12cf6cd4de1cac6ed7bee8d908a2c8fc23f538c" +url = "https://github.com/norwoodj/helm-docs/releases/download/v1.14.2/helm-docs_1.14.2_Windows_x86_64.tar.gz" + [[tools.k3d]] version = "5.8.3" backend = "aqua:k3d-io/k3d" @@ -158,6 +187,10 @@ url = "https://dl.k8s.io/v1.36.1/bin/linux/amd64/kubectl" checksum = "sha256:9092778abaef3079449da4cd70ded0e4be112480c93efcdeace3155968d1d133" url = "https://dl.k8s.io/v1.36.1/bin/darwin/arm64/kubectl" +[tools.kubectl."platforms.windows-x64"] +checksum = "sha256:538f4229eee91a17b34724da7daade7687393d6988e33b723c6c306572c13900" +url = "https://dl.k8s.io/v1.36.1/bin/windows/amd64/kubectl.exe" + [[tools.node]] version = "24.15.0" backend = "core:node" @@ -174,6 +207,10 @@ url = "https://nodejs.org/dist/v24.15.0/node-v24.15.0-linux-x64.tar.gz" checksum = "sha256:372331b969779ab5d15b949884fc6eaf88d5afe87bde8ba881d6400b9100ffc4" url = "https://nodejs.org/dist/v24.15.0/node-v24.15.0-darwin-arm64.tar.gz" +[tools.node."platforms.windows-x64"] +checksum = "sha256:cc5149eabd53779ce1e7bdc5401643622d0c7e6800ade18928a767e940bb0e62" +url = "https://nodejs.org/dist/v24.15.0/node-v24.15.0-win-x64.zip" + [[tools."npm:markdownlint-cli2"]] version = "0.22.0" backend = "npm:markdownlint-cli2" @@ -194,6 +231,10 @@ url = "https://github.com/protocolbuffers/protobuf/releases/download/v29.6/proto checksum = "sha256:b9576b5fa1a1ef3fe13a8c91d9d8204b46545759bea5ae155cd6ba2ea4cdaeed" url = "https://github.com/protocolbuffers/protobuf/releases/download/v29.6/protoc-29.6-osx-aarch_64.zip" +[tools.protoc."platforms.windows-x64"] +checksum = "sha256:1ebd7c87baffb9f1c47169b640872bf5fb1e4408079c691af527be9561d8f6f7" +url = "https://github.com/protocolbuffers/protobuf/releases/download/v29.6/protoc-29.6-win64.zip" + [[tools.python]] version = "3.14.5" backend = "core:python" @@ -216,6 +257,11 @@ checksum = "sha256:3a0373cc39fefd494754ef555267f245c720cddbaaabf63a7c9a4269f1e56 url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260602/cpython-3.14.5+20260602-aarch64-apple-darwin-install_only_stripped.tar.gz" provenance = "github-attestations" +[tools.python."platforms.windows-x64"] +checksum = "blake3:1af86eafd814227465499d996373e89f56f750b8044eb6188621615d8ff4f8fb" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260602/cpython-3.14.5+20260602-x86_64-pc-windows-msvc-install_only_stripped.tar.gz" +provenance = "github-attestations" + [[tools.rust]] version = "1.95.0" backend = "core:rust" @@ -234,6 +280,10 @@ url = "https://storage.googleapis.com/skaffold/releases/v2.20.0/skaffold-linux-a [tools.skaffold."platforms.macos-arm64"] url = "https://storage.googleapis.com/skaffold/releases/v2.20.0/skaffold-darwin-arm64" +[tools.skaffold."platforms.windows-x64"] +checksum = "blake3:f6db036671e29118d0fdc4457dc563e99d7bff8a39f0f2f72e3e5d857d9013a3" +url = "https://storage.googleapis.com/skaffold/releases/v2.20.0/skaffold-windows-amd64.exe" + [[tools.uv]] version = "0.10.12" backend = "aqua:astral-sh/uv" @@ -253,6 +303,11 @@ checksum = "sha256:ae738b5661a900579ec621d3918c0ef17bdec0da2a8a6d8b161137cd15f25 url = "https://github.com/astral-sh/uv/releases/download/0.10.12/uv-aarch64-apple-darwin.tar.gz" provenance = "github-attestations" +[tools.uv."platforms.windows-x64"] +checksum = "sha256:4c1d55501869b3330d4aabf45ad6024ce2367e0f3af83344395702d272c22e88" +url = "https://github.com/astral-sh/uv/releases/download/0.10.12/uv-x86_64-pc-windows-msvc.zip" +provenance = "github-attestations" + [[tools.zig]] version = "0.14.1" backend = "core:zig" @@ -268,3 +323,8 @@ url = "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz" [tools.zig."platforms.macos-arm64"] checksum = "sha256:39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa" url = "https://ziglang.org/download/0.14.1/zig-aarch64-macos-0.14.1.tar.xz" + +[tools.zig."platforms.windows-x64"] +checksum = "sha256:554f5378228923ffd558eac35e21af020c73789d87afeabf4bfd16f2e6feed2c" +url = "https://ziglang.org/download/0.14.1/zig-x86_64-windows-0.14.1.zip" +provenance = "minisign" From de33dedc418e5e067985c5cf578b7fac083cd126 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Tue, 9 Jun 2026 20:46:16 -0500 Subject: [PATCH 10/30] feat(windows): enhance libclang path resolution to support architecture-specific subdirectories Signed-off-by: Akber Raza --- tasks/scripts/windows-msvc.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 129739cd5a..1c44824e91 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -79,7 +79,13 @@ function Resolve-VsDevCmd { throw "Could not find VsDevCmd.bat. Install Visual Studio Build Tools, or set OPENSHELL_VSDEVCMD." } +function Get-LibclangBinSubdir { + return ([System.Runtime.InteropServices.RuntimeInformation, mscorlib]::OSArchitecture.ToString()) +} + function Resolve-LibclangPath { + $subdir = Get-LibclangBinSubdir + if ($env:LIBCLANG_PATH) { $candidate = Join-Path $env:LIBCLANG_PATH "libclang.dll" if (Test-Path $candidate) { @@ -95,7 +101,7 @@ function Resolve-LibclangPath { $vswhere = $null } if ($vswhere -and (Test-Path $vswhere)) { - $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -find "VC\Tools\Llvm\x64\bin\libclang.dll" | Select-Object -First 1 + $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -find "VC\Tools\Llvm\$subdir\bin\libclang.dll" | Select-Object -First 1 if ($found -and (Test-Path $found)) { return (Split-Path -Parent (Resolve-Path $found).Path) } @@ -110,7 +116,7 @@ function Resolve-LibclangPath { foreach ($root in $programFiles) { foreach ($version in $versions) { foreach ($edition in $editions) { - $candidateDir = Join-Path $root "Microsoft Visual Studio\$version\$edition\VC\Tools\Llvm\x64\bin" + $candidateDir = Join-Path $root "Microsoft Visual Studio\$version\$edition\VC\Tools\Llvm\$subdir\bin" $candidate = Join-Path $candidateDir "libclang.dll" if (Test-Path $candidate) { return (Resolve-Path $candidateDir).Path From 5baaa3fc2317a50d0c0e6d177d6a811890135296 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Wed, 10 Jun 2026 19:18:30 -0700 Subject: [PATCH 11/30] Fix Windows dependency gating after sync merge Signed-off-by: Akber Raza --- crates/openshell-bootstrap/Cargo.toml | 4 +++- .../openshell-bootstrap/src/build_windows.rs | 21 +++++++++++++++++++ crates/openshell-bootstrap/src/lib.rs | 4 ++++ crates/openshell-core/src/driver_mounts.rs | 13 +++++++----- 4 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 crates/openshell-bootstrap/src/build_windows.rs diff --git a/crates/openshell-bootstrap/Cargo.toml b/crates/openshell-bootstrap/Cargo.toml index c860cb1384..96a7985085 100644 --- a/crates/openshell-bootstrap/Cargo.toml +++ b/crates/openshell-bootstrap/Cargo.toml @@ -11,7 +11,6 @@ rust-version.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } -bollard = "0.20" bytes = { workspace = true } futures = { workspace = true } miette = { workspace = true } @@ -24,6 +23,9 @@ tempfile = "3" tokio = { workspace = true } tracing = { workspace = true } +[target.'cfg(not(target_os = "windows"))'.dependencies] +bollard = "0.20" + [dev-dependencies] [lints] diff --git a/crates/openshell-bootstrap/src/build_windows.rs b/crates/openshell-bootstrap/src/build_windows.rs new file mode 100644 index 0000000000..6337919b21 --- /dev/null +++ b/crates/openshell-bootstrap/src/build_windows.rs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Windows stub for local Dockerfile image builds. + +use std::collections::HashMap; +use std::path::Path; + +use miette::Result; + +pub async fn build_local_image( + _dockerfile_path: &Path, + _tag: &str, + _context_dir: &Path, + _build_args: &HashMap, + _on_log: &mut impl FnMut(String), +) -> Result<()> { + Err(miette::miette!( + "local Dockerfile sandbox sources are unsupported on Windows" + )) +} diff --git a/crates/openshell-bootstrap/src/lib.rs b/crates/openshell-bootstrap/src/lib.rs index aa0532260e..5598d524ee 100644 --- a/crates/openshell-bootstrap/src/lib.rs +++ b/crates/openshell-bootstrap/src/lib.rs @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#[cfg(not(target_os = "windows"))] +pub mod build; +#[cfg(target_os = "windows")] +#[path = "build_windows.rs"] pub mod build; pub mod edge_token; pub mod jwt; diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 086d992c37..6860be7eb7 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -67,11 +67,14 @@ pub fn validate_mount_subpath(subpath: &str) -> Result<(), String> { return Err("mount subpath must not contain NUL bytes".to_string()); } let path = Path::new(subpath); - if path.is_absolute() - || path - .components() - .any(|component| matches!(component, std::path::Component::ParentDir)) - { + if path.components().any(|component| { + matches!( + component, + std::path::Component::Prefix(_) + | std::path::Component::RootDir + | std::path::Component::ParentDir + ) + }) { return Err("mount subpath must be relative and must not contain '..'".to_string()); } Ok(()) From 3669d7960fb79da04840d007876db2294d1f7140 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Thu, 11 Jun 2026 11:15:34 -0500 Subject: [PATCH 12/30] fix(z3): update Z3 header path requirements in Windows build documentation and scripts Signed-off-by: Akber Raza --- CONTRIBUTING.md | 12 +++- docs/reference/windows-msvc-build-design.mdx | 4 ++ tasks/scripts/windows-msvc.ps1 | 64 ++++++++++++++++++-- 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d57e50c7f..f23a761441 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -164,7 +164,9 @@ cargo build -p openshell-prover --features bundled-z3 For x86-64 Windows MSVC builds, use one of these Z3 paths: - System Z3: point `Z3_LIBRARY_PATH_OVERRIDE` at the directory containing the - 64-bit MSVC Z3 library and `Z3_SYS_Z3_HEADER` at `z3.h`. + 64-bit MSVC Z3 library and `Z3_SYS_Z3_HEADER` at the full path to `z3.h`. + The `windows:*` tasks use this path automatically when `Z3_LIBRARY_PATH_OVERRIDE` + is set. - Bundled Z3: pass `--features bundled-z3` so `z3-sys` builds Z3 from source. Both Windows paths still require `libclang.dll` for `bindgen`. If LLVM is not on @@ -176,6 +178,14 @@ $env:LIBCLANG_PATH='C:\Program Files\Microsoft Visual Studio\2022\\VC\T cargo build -p openshell-cli --target x86_64-pc-windows-msvc --features bundled-z3 ``` +To use a local x64 Z3 release with the Windows task wrapper: + +```powershell +$env:Z3_LIBRARY_PATH_OVERRIDE='C:\path\to\z3-4.16.0-x64-win\bin' +$env:Z3_SYS_Z3_HEADER='C:\path\to\z3-4.16.0-x64-win\include\z3.h' +mise run --skip-tools windows:build:x64 +``` + ### macOS build tools Install Apple Command Line Tools before building locally: diff --git a/docs/reference/windows-msvc-build-design.mdx b/docs/reference/windows-msvc-build-design.mdx index f103c4cc83..6a3be4560a 100644 --- a/docs/reference/windows-msvc-build-design.mdx +++ b/docs/reference/windows-msvc-build-design.mdx @@ -63,6 +63,10 @@ Windows validation is exposed through `tasks/windows.toml`: The Windows tasks call `tasks/scripts/windows-msvc.ps1`. The wrapper discovers Visual Studio's `VsDevCmd.bat`, adds rustup MSVC targets, clears inherited `RUSTC_WRAPPER`, and keeps build artifacts under the normal Cargo target tree. +By default it enables bundled Z3 for reproducible Windows builds. When +`Z3_LIBRARY_PATH_OVERRIDE` points at a directory containing `libz3.lib`, the +wrapper uses that system Z3 instead and requires `Z3_SYS_Z3_HEADER` to point at +the full path to `z3.h`. The lane uses `mise run --skip-tools windows:*` because Windows Rust comes from rustup and linking comes from Visual Studio Build Tools. Mise orchestrates the diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 1c44824e91..d298663c7f 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -40,6 +40,8 @@ if ([string]::IsNullOrWhiteSpace($TargetDir)) { $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm" $BundledZ3WorkspaceFeatures = "--features openshell-cli/bundled-z3,openshell-prover/bundled-z3" $BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-prover/bundled-z3" +$Z3WorkspaceFeatures = $BundledZ3WorkspaceFeatures +$Z3ServerFeatures = $BundledZ3ServerFeatures function Resolve-VsDevCmd { if ($env:OPENSHELL_VSDEVCMD -and (Test-Path $env:OPENSHELL_VSDEVCMD)) { @@ -159,6 +161,57 @@ function Get-SelectedTargets([string] $RequestedTarget) { return @($RequestedTarget) } +function Resolve-Z3HeaderPath([string] $HeaderPath) { + if ([string]::IsNullOrWhiteSpace($HeaderPath)) { + throw "Z3_LIBRARY_PATH_OVERRIDE is set. Set Z3_SYS_Z3_HEADER to the full path of z3.h." + } + + if (-not (Test-Path $HeaderPath -PathType Leaf)) { + throw "Z3_SYS_Z3_HEADER is set but z3.h was not found at: $HeaderPath" + } + if ((Split-Path -Leaf $HeaderPath) -ne "z3.h") { + throw "Z3_SYS_Z3_HEADER must point to z3.h. Got: $HeaderPath" + } + + return (Resolve-Path $HeaderPath).Path +} + +function Configure-Z3 { + if ([string]::IsNullOrWhiteSpace($env:Z3_LIBRARY_PATH_OVERRIDE)) { + Write-Host "==> Z3: bundled" + return [pscustomobject]@{ + WorkspaceFeatures = $BundledZ3WorkspaceFeatures + ServerFeatures = $BundledZ3ServerFeatures + } + } + + if (-not (Test-Path $env:Z3_LIBRARY_PATH_OVERRIDE -PathType Container)) { + throw "Z3_LIBRARY_PATH_OVERRIDE is set but the directory does not exist: $env:Z3_LIBRARY_PATH_OVERRIDE" + } + + $libDir = (Resolve-Path $env:Z3_LIBRARY_PATH_OVERRIDE).Path + $importLib = Join-Path $libDir "libz3.lib" + if (-not (Test-Path $importLib -PathType Leaf)) { + throw "Z3_LIBRARY_PATH_OVERRIDE is set but libz3.lib was not found at: $importLib" + } + + $env:Z3_LIBRARY_PATH_OVERRIDE = $libDir + $env:Z3_SYS_Z3_HEADER = Resolve-Z3HeaderPath $env:Z3_SYS_Z3_HEADER + + if (($env:PATH -split ";") -notcontains $libDir) { + $env:PATH = "$libDir;$env:PATH" + } + + Write-Host "==> Z3: system" + Write-Host " Z3_LIBRARY_PATH_OVERRIDE=$env:Z3_LIBRARY_PATH_OVERRIDE" + Write-Host " Z3_SYS_Z3_HEADER=$env:Z3_SYS_Z3_HEADER" + + return [pscustomobject]@{ + WorkspaceFeatures = "" + ServerFeatures = "" + } +} + function Invoke-VsCargo { param( [Parameter(Mandatory = $true)] [string] $RustTarget, @@ -195,14 +248,14 @@ function Invoke-VsCargo { function Invoke-Check([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget $BundledZ3WorkspaceFeatures" ` + -CargoArgs "cargo check --workspace $UnsupportedDriverPackageExcludes --target $RustTarget $Z3WorkspaceFeatures" ` -LogName "build-$RustTarget-check.log" } function Invoke-Build([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell $BundledZ3WorkspaceFeatures" ` + -CargoArgs "cargo build --release --target $RustTarget --bin openshell-gateway --bin openshell $Z3WorkspaceFeatures" ` -LogName "build-$RustTarget-release.log" } @@ -212,7 +265,7 @@ function Invoke-Test([string] $RustTarget) { } Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast $BundledZ3WorkspaceFeatures" ` + -CargoArgs "cargo test --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast $Z3WorkspaceFeatures" ` -LogName "test-$RustTarget.log" } @@ -228,7 +281,7 @@ function Invoke-UnsupportedContractTests([string] $RustTarget) { foreach ($test in $tests) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test -p openshell-server --target $RustTarget $test $BundledZ3ServerFeatures" ` + -CargoArgs "cargo test -p openshell-server --target $RustTarget $test $Z3ServerFeatures" ` -LogName "test-$RustTarget-unsupported-$test.log" } } @@ -260,6 +313,9 @@ function Show-Artifacts([string[]] $RustTargets) { } $targets = Get-SelectedTargets $Target +$z3Features = Configure-Z3 +$Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures +$Z3ServerFeatures = $z3Features.ServerFeatures $env:LIBCLANG_PATH = Resolve-LibclangPath Write-Host "==> LIBCLANG_PATH=$env:LIBCLANG_PATH" From bbf271f880eec9193a55d280df2e21613a4d93cc Mon Sep 17 00:00:00 2001 From: shailendras Date: Mon, 22 Jun 2026 22:15:15 +0000 Subject: [PATCH 13/30] docs(windows): relocate Windows MSVC build design to architecture/ Why: windows-msvc-build-design.mdx is a design document ("design decisions for the native Windows MSVC build lane"), but it lived in the published, user-facing docs/reference/ tree. Per AGENTS.md (Documentation) and architecture/README.md ("rfc/ vs architecture/"), design content belongs in architecture/ (or rfc/), not in published reference. It also shared Fern sidebar "position: 6" with the MXC compute-driver design page, colliding in the Reference nav ordering. What: - Move docs/reference/windows-msvc-build-design.mdx -> architecture/windows-msvc-build.md. - Strip the Fern publish frontmatter and add a plain H1, matching the other architecture docs. - Register it in the architecture doc index in architecture/README.md. - Repoint the inbound references (build-openshell-mxc-windows skill + reference, implement-openshell-mxc-driver skill) to the new path. With both design pages moved out of docs/reference/, the duplicate position-6 sidebar collision is resolved. Signed-off-by: Akber Raza --- architecture/README.md | 1 + .../windows-msvc-build.md | 10 +--------- 2 files changed, 2 insertions(+), 9 deletions(-) rename docs/reference/windows-msvc-build-design.mdx => architecture/windows-msvc-build.md (92%) diff --git a/architecture/README.md b/architecture/README.md index 3fc72afd25..9453d59826 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -167,6 +167,7 @@ that crate's `README.md`. | [Compute Runtimes](compute-runtimes.md) | Docker, Podman, Kubernetes, VM, sandbox images, and runtime-specific responsibilities. | | [Build](build.md) | Build artifacts, CI/E2E, docs site validation, and release packaging. | | [Google Vertex AI Provider](google-vertex-ai-provider.md) | Implementation reference for the `google-vertex-ai` provider, from CLI through gateway to sandbox. | +| [Windows MSVC Build](windows-msvc-build.md) | Build-only native Windows MSVC lane (x64/ARM64) and unsupported-runtime behavior on Windows. | ## `rfc/` vs `architecture/` diff --git a/docs/reference/windows-msvc-build-design.mdx b/architecture/windows-msvc-build.md similarity index 92% rename from docs/reference/windows-msvc-build-design.mdx rename to architecture/windows-msvc-build.md index 6a3be4560a..c83cd8f2b2 100644 --- a/docs/reference/windows-msvc-build-design.mdx +++ b/architecture/windows-msvc-build.md @@ -1,12 +1,4 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Windows MSVC Build Design" -sidebar-title: "Windows MSVC Build" -description: "Design notes for the build-only native Windows MSVC lane." -keywords: "Windows, MSVC, OpenShell, mise, Rust, build" -position: 6 ---- +# Windows MSVC Build Design This page records the design decisions for the native Windows MSVC build lane. It is intentionally build-only. It does not make Windows a Docker, Kubernetes, From d95df2f8b09e7a26198fa5243191444992cf6bd9 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Tue, 14 Jul 2026 19:27:13 -0500 Subject: [PATCH 14/30] remove openshell-supervisor-network from unsupported driver package test exclusion list Signed-off-by: Akber Raza # Conflicts: # tasks/scripts/windows-msvc.ps1 From 849807576908b00c0ad37ce824c0aad4b2971f7f Mon Sep 17 00:00:00 2001 From: Jamie King Date: Wed, 15 Jul 2026 20:14:19 -0600 Subject: [PATCH 15/30] fix(interceptors): gate unix-only imports so the crate builds on Windows openshell-gateway-interceptors failed to compile on Windows (E0432: no UnixStream in tokio::net), breaking any Windows build of openshell-server (which depends on it unconditionally). The connect_unix_endpoint fn was already #[cfg(unix)]-gated, but the imports it uses (UnixStream, TokioIo, Uri, service_fn) were left ungated. Gate those four imports with #[cfg(unix)] too. No behavior change on unix; Windows now compiles (no errors, no unused-import warnings). Signed-off-by: Akber Raza --- crates/openshell-gateway-interceptors/src/plan.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/openshell-gateway-interceptors/src/plan.rs b/crates/openshell-gateway-interceptors/src/plan.rs index b65d5612fd..36e7aa3115 100644 --- a/crates/openshell-gateway-interceptors/src/plan.rs +++ b/crates/openshell-gateway-interceptors/src/plan.rs @@ -7,6 +7,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; use std::time::Duration; +#[cfg(unix)] use hyper_util::rt::TokioIo; use openshell_core::config::{ GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, @@ -16,10 +17,13 @@ use openshell_core::proto::gateway_interceptor::v1::{ DescribeRequest, GatewayInterceptorPhase, InterceptorBinding, InterceptorSelector, gateway_interceptor_client::GatewayInterceptorClient, }; +#[cfg(unix)] use tokio::net::UnixStream; use tonic::Request; +#[cfg(unix)] use tonic::codegen::http::Uri; use tonic::transport::{Channel, Endpoint}; +#[cfg(unix)] use tower::service_fn; use tracing::{info, warn}; From 7809c78ba97637d9fff720f7ebe26627f4882fea Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Thu, 16 Jul 2026 20:46:24 -0700 Subject: [PATCH 16/30] feat(windows): add native ARM64 test support Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 24 +++++++--- .../build-openshell-mxc-windows/reference.md | 6 +++ architecture/windows-msvc-build.md | 18 ++++++-- tasks/scripts/windows-msvc.ps1 | 46 ++++++++++++++----- tasks/windows.toml | 10 ++++ 5 files changed, 82 insertions(+), 22 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 6dda48373b..4d546e6aeb 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -18,7 +18,7 @@ In scope: - Audit and cfg-gate Unix-only dependencies in `openshell-server` and shared crates so the workspace compiles to Windows MSVC - Keep Docker, Kubernetes, Podman, and VM compute crates in the Windows build graph as stubs that return "unsupported" at runtime - Add a Windows-specific mise task lane (`windows:*`) that wraps MSVC builds without changing the Linux `mise run ci` path -- `cargo test` compiles and passes on Windows for non-Linux-gated tests +- `cargo test` compiles and passes on a native x64 or ARM64 Windows host for non-Linux-gated tests - Scaffold an ARM64 CI workflow (job definition only; runner provisioning is operator work) Out of scope (defer to follow-on skills): @@ -39,7 +39,8 @@ The skill targets a **Windows 11 host with CurrentBuild ≥ 26100**. Because the | Requirement | Check | Install hint | |---|---|---| | Windows 11 build ≥ 26100 | `[System.Environment]::OSVersion.Version` | OS update | -| Visual Studio 2022 Build Tools with **MSVC v143** + **Windows 11 SDK** | `where cl.exe` from a VS Developer PowerShell | https://visualstudio.microsoft.com/downloads/ | +| Visual Studio 2022 or newer with **MSVC v143** + **Windows 11 SDK** | `where.exe cl.exe` from a VS Developer PowerShell | Include the x64/x86 and ARM64 C++ tools. | +| Visual C++ Clang and CMake tools | `where.exe clang.exe`; `where.exe cmake.exe` | `bindgen` needs host-native `libclang.dll`; bundled Z3 uses CMake and MSBuild. | | Rust ≥ 1.88 via rustup with MSVC targets | `rustc --version` | `winget install Rustlang.Rustup` | | mise CLI for task orchestration only | `mise --version` | https://mise.jdx.dev/installing-mise.html | | Git ≥ 2.40 | `git --version` | `winget install Git.Git` | @@ -72,7 +73,7 @@ The skill runs the following checklist top-to-bottom. Each step is idempotent; r [ ] Step 6: mise check on x86_64-pc-windows-msvc [ ] Step 7: mise check on aarch64-pc-windows-msvc [ ] Step 8: mise build --release (both targets) -[ ] Step 9: mise test on x86_64-pc-windows-msvc (native run) +[ ] Step 9: mise test on the host's native MSVC target [ ] Step 10: Validate $env:OPENSHELL_WXC_EXEC_PATH (informational) [ ] Step 11: Scaffold ARM64 CI workflow [ ] Step 12: Commit and report artifacts @@ -176,7 +177,9 @@ The task file must expose these commands: | `windows:build:x64` | Release-build `openshell-gateway` and `openshell` for x64 | | `windows:build:arm64` | Release-build `openshell-gateway` and `openshell` for ARM64 | | `windows:test:x64` | Native x64 `cargo test --workspace --no-fail-fast`, excluding unsupported driver packages as top-level workspace targets | +| `windows:test:arm64` | Native ARM64 `cargo test --workspace --no-fail-fast` with the same exclusions | | `windows:test:unsupported:x64` | Focused server/runtime tests that assert unsupported Windows driver contracts without building standalone driver binaries | +| `windows:test:unsupported:arm64` | The same focused contracts on a native ARM64 host | | `windows:ci` | Ordered Windows check/build/test/unsupported-contract/artifact lane | The PowerShell wrapper must: @@ -231,13 +234,22 @@ dumpbin /HEADERS "$fork\target\x86_64-pc-windows-msvc\release\openshell-gateway. Expected machine values: `x64` for `x86_64-pc-windows-msvc`, `ARM64` for `aarch64-pc-windows-msvc`. -### Step 9: mise test on x86_64-pc-windows-msvc (native run) +### Step 9: mise test on the native Windows architecture ```powershell -mise run --skip-tools windows:test:x64 -mise run --skip-tools windows:test:unsupported:x64 +$arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture +if ($arch -eq [System.Runtime.InteropServices.Architecture]::Arm64) { + mise run --skip-tools windows:test:arm64 + mise run --skip-tools windows:test:unsupported:arm64 +} else { + mise run --skip-tools windows:test:x64 + mise run --skip-tools windows:test:unsupported:x64 +} ``` +The wrapper rejects test targets that do not match the host architecture. This +keeps ARM64 results native and avoids reporting x64 emulation as ARM64 coverage. + Failures fall into three buckets: 1. **Linux-only test** — gate with `#[cfg(not(target_os = "windows"))]` and add a short comment describing the Windows-equivalent test that will eventually replace it. diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index d7dcc16240..c3148160b1 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -166,11 +166,17 @@ The Windows build path is additive. Keep the repository's Linux `mise run ci`, d | `mise run --skip-tools windows:build:x64` | Builds release `openshell-gateway.exe` and `openshell.exe` for x64 | | `mise run --skip-tools windows:build:arm64` | Builds release `openshell-gateway.exe` and `openshell.exe` for ARM64 | | `mise run --skip-tools windows:test:x64` | Runs native x64 workspace tests | +| `mise run --skip-tools windows:test:arm64` | Runs workspace tests on a native ARM64 Windows host | | `mise run --skip-tools windows:test:unsupported:x64` | Verifies unsupported driver contracts through server/runtime tests without building standalone driver binaries | +| `mise run --skip-tools windows:test:unsupported:arm64` | Verifies the same contracts on a native ARM64 Windows host | | `mise run --skip-tools windows:ci` | Runs the full Windows lane in order | Use `--skip-tools` for Windows CI and automation. Rust must come from rustup with MSVC targets, and Visual Studio Build Tools must provide the linker and SDK. Because `--skip-tools` does not provision mise-managed tools, the Windows wrapper clears inherited `RUSTC_WRAPPER=sccache` before invoking Cargo. The wrapper excludes unsupported driver packages as top-level workspace targets for Windows check/test, but those library stubs still compile when the gateway depends on them. The wrapper may discover `VsDevCmd.bat`, but it must not install Visual Studio, Rust, Docker, Kubernetes, Podman, VM tooling, WSL, or Hyper-V. +ARM64 validation requires the Visual Studio ARM64 C++ tools, host-native +`libclang.dll`, CMake tools for bundled Z3, and an ARM64-capable Windows SDK. +Test tasks reject a target that does not match the Windows host architecture. + ## What NOT to do in this skill - Do not implement named-pipe IPC. Stubs only. (Belongs to the follow-on MXC driver skill.) diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index c83cd8f2b2..ca2993ffa1 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -48,13 +48,17 @@ Windows validation is exposed through `tasks/windows.toml`: | `windows:check:arm64` | Check the ARM64 MSVC gateway/CLI build graph. | | `windows:build:x64` | Build release x64 `openshell-gateway.exe` and `openshell.exe`. | | `windows:build:arm64` | Build release ARM64 `openshell-gateway.exe` and `openshell.exe`. | -| `windows:test:x64` | Run native x64 workspace tests, excluding unsupported driver packages as top-level test targets. | +| `windows:test:x64` | Run native x64 workspace tests, excluding unsupported Windows packages as top-level test targets. | +| `windows:test:arm64` | Run native ARM64 workspace tests with the same package exclusions. | | `windows:test:unsupported:x64` | Run focused server/runtime tests for unsupported driver contracts. | +| `windows:test:unsupported:arm64` | Run the same focused contracts natively on ARM64. | | `windows:ci` | Run check, build, test, unsupported-contract tests, and artifact reporting. | The Windows tasks call `tasks/scripts/windows-msvc.ps1`. The wrapper discovers Visual Studio's `VsDevCmd.bat`, adds rustup MSVC targets, clears inherited `RUSTC_WRAPPER`, and keeps build artifacts under the normal Cargo target tree. +Test tasks require the Rust target architecture to match the Windows host, so +an ARM64 test result is native coverage rather than x64 emulation coverage. By default it enables bundled Z3 for reproducible Windows builds. When `Z3_LIBRARY_PATH_OVERRIDE` points at a directory containing `libz3.lib`, the wrapper uses that system Z3 instead and requires `Z3_SYS_Z3_HEADER` to point at @@ -64,6 +68,11 @@ The lane uses `mise run --skip-tools windows:*` because Windows Rust comes from rustup and linking comes from Visual Studio Build Tools. Mise orchestrates the tasks; it does not own the Windows toolchain. +ARM64 validation requires the Visual Studio ARM64 MSVC tools, host-native Clang +tools for `libclang.dll`, CMake tools for bundled Z3, and an ARM64-capable +Windows SDK. Artifact hashing uses .NET SHA256 directly because module +autoloading in the mise-launched Windows PowerShell process is not guaranteed. + ## CI Shape The x64 GitHub Actions job runs on `windows-2025` and executes: @@ -76,9 +85,9 @@ mise run --skip-tools windows:test:unsupported:x64 ``` The ARM64 job is scaffolded but disabled until a Windows ARM64 runner is -available. Once enabled, it should run check and release build for -`aarch64-pc-windows-msvc`. Native ARM64 tests require an ARM64 runner and are -not part of the current build-only lane. +available. Once enabled, it should run check, release build, native workspace +tests, and the focused unsupported-driver contracts for +`aarch64-pc-windows-msvc`. ## Validation Contract @@ -87,6 +96,7 @@ A successful Windows build report should include: - x64 and ARM64 `cargo check` status. - x64 and ARM64 release build status for `openshell-gateway.exe` and `openshell.exe`. - x64 test summary. +- Native ARM64 test summary when validation runs on an ARM64 host. - Focused unsupported-driver contract test status. - Artifact size and SHA256 for each Windows binary. diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index d298663c7f..cb53f57a8a 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -43,7 +43,7 @@ $BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-pro $Z3WorkspaceFeatures = $BundledZ3WorkspaceFeatures $Z3ServerFeatures = $BundledZ3ServerFeatures -function Resolve-VsDevCmd { +function Resolve-VsDevCmd([string] $RustTarget) { if ($env:OPENSHELL_VSDEVCMD -and (Test-Path $env:OPENSHELL_VSDEVCMD)) { return (Resolve-Path $env:OPENSHELL_VSDEVCMD).Path } @@ -55,7 +55,12 @@ function Resolve-VsDevCmd { $vswhere = $null } if ($vswhere -and (Test-Path $vswhere)) { - $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find "Common7\Tools\VsDevCmd.bat" | Select-Object -First 1 + $requiredComponent = switch ($RustTarget) { + "x86_64-pc-windows-msvc" { "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" } + "aarch64-pc-windows-msvc" { "Microsoft.VisualStudio.Component.VC.Tools.ARM64" } + default { throw "Unsupported target: $RustTarget" } + } + $found = & $vswhere -latest -products * -requires $requiredComponent -find "Common7\Tools\VsDevCmd.bat" | Select-Object -First 1 if ($found -and (Test-Path $found)) { return (Resolve-Path $found).Path } @@ -65,7 +70,7 @@ function Resolve-VsDevCmd { [Environment]::GetEnvironmentVariable("ProgramFiles"), [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") ) | Where-Object { $_ } - $versions = @("18", "17") + $versions = @("18", "2022", "17") $editions = @("Enterprise", "Professional", "Community", "BuildTools") foreach ($root in $programFiles) { foreach ($version in $versions) { @@ -150,6 +155,14 @@ function Get-VsTargetArch([string] $RustTarget) { } } +function Assert-NativeTestTarget([string] $RustTarget) { + $targetArch = Get-VsTargetArch $RustTarget + $hostArch = Get-HostArch + if ($targetArch -ne $hostArch) { + throw "Windows tests require a native runner. Target $RustTarget maps to $targetArch, but the host is $hostArch." + } +} + function Get-SelectedTargets([string] $RequestedTarget) { if ($RequestedTarget -eq "all") { $targets = @("x86_64-pc-windows-msvc") @@ -224,7 +237,7 @@ function Invoke-VsCargo { throw "rustup target add $RustTarget failed" } - $vsDevCmd = Resolve-VsDevCmd + $vsDevCmd = Resolve-VsDevCmd $RustTarget $targetArch = Get-VsTargetArch $RustTarget $hostArch = Get-HostArch $logPath = Join-Path $LogDir $LogName @@ -260,9 +273,7 @@ function Invoke-Build([string] $RustTarget) { } function Invoke-Test([string] $RustTarget) { - if ($RustTarget -ne "x86_64-pc-windows-msvc") { - throw "Windows ARM64 tests require a native ARM64 runner and are not part of this build-only lane." - } + Assert-NativeTestTarget $RustTarget Invoke-VsCargo ` -RustTarget $RustTarget ` -CargoArgs "cargo test --workspace $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast $Z3WorkspaceFeatures" ` @@ -270,9 +281,7 @@ function Invoke-Test([string] $RustTarget) { } function Invoke-UnsupportedContractTests([string] $RustTarget) { - if ($RustTarget -ne "x86_64-pc-windows-msvc") { - throw "Unsupported-driver contract tests run only on the native x64 Windows lane." - } + Assert-NativeTestTarget $RustTarget $tests = @( "windows_compute_driver_stubs_report_unsupported", @@ -286,6 +295,20 @@ function Invoke-UnsupportedContractTests([string] $RustTarget) { } } +function Get-Sha256([string] $Path) { + $stream = [System.IO.File]::OpenRead($Path) + try { + $sha256 = [System.Security.Cryptography.SHA256]::Create() + try { + return [BitConverter]::ToString($sha256.ComputeHash($stream)).Replace("-", "") + } finally { + $sha256.Dispose() + } + } finally { + $stream.Dispose() + } +} + function Show-Artifacts([string[]] $RustTargets) { $rows = @() foreach ($rustTarget in $RustTargets) { @@ -295,12 +318,11 @@ function Show-Artifacts([string[]] $RustTargets) { continue } $item = Get-Item $path - $hash = Get-FileHash $path -Algorithm SHA256 $rows += [pscustomobject]@{ Target = $rustTarget Binary = $binary Size = $item.Length - SHA256 = $hash.Hash + SHA256 = Get-Sha256 $item.FullName Path = $item.FullName } } diff --git a/tasks/windows.toml b/tasks/windows.toml index fa3b2b60ae..c7a2e51f99 100644 --- a/tasks/windows.toml +++ b/tasks/windows.toml @@ -39,11 +39,21 @@ description = "Run Windows x64 MSVC workspace tests" run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test x86_64-pc-windows-msvc" +["windows:test:arm64"] +description = "Run Windows ARM64 MSVC workspace tests on a native ARM64 host" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test aarch64-pc-windows-msvc" + ["windows:test:unsupported:x64"] description = "Run focused Windows unsupported compute-driver contract tests" run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-unsupported x86_64-pc-windows-msvc" +["windows:test:unsupported:arm64"] +description = "Run focused Windows unsupported compute-driver contract tests on a native ARM64 host" +run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-unsupported aarch64-pc-windows-msvc" + ["windows:artifacts"] description = "Report Windows release artifact sizes and SHA256 hashes" run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1" From 78651169cae189127676a97f94d31bc41dcbb8e9 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Thu, 16 Jul 2026 21:05:41 -0700 Subject: [PATCH 17/30] fix(mise): skip Skaffold on Windows Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise.toml b/mise.toml index a5d9682108..b4c00e1d8e 100644 --- a/mise.toml +++ b/mise.toml @@ -27,7 +27,7 @@ uv = "0.10.12" protoc = "29.6" helm = "4.2.0" helm-docs = "1.14.2" -skaffold = "2.20.0" +skaffold = { version = "2.20.0", os = ["linux", "macos"] } # Keep k3d out of Linux CI images until upstream ships a release rebuilt with # patched Go/container dependencies. Linux Kubernetes E2E uses kind or an # externally provided cluster context. From a8e78bb8405bfd2e31eba405604467d91507b5fc Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Sat, 18 Jul 2026 17:44:47 -0700 Subject: [PATCH 18/30] fix(windows): harden ARM64 toolchain discovery Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 10 +- .../build-openshell-mxc-windows/reference.md | 10 +- architecture/windows-msvc-build.md | 21 +- tasks/scripts/windows-msvc.ps1 | 236 +++++++++++++++--- 4 files changed, 230 insertions(+), 47 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 4d546e6aeb..0defc1fbf9 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -39,8 +39,9 @@ The skill targets a **Windows 11 host with CurrentBuild ≥ 26100**. Because the | Requirement | Check | Install hint | |---|---|---| | Windows 11 build ≥ 26100 | `[System.Environment]::OSVersion.Version` | OS update | -| Visual Studio 2022 or newer with **MSVC v143** + **Windows 11 SDK** | `where.exe cl.exe` from a VS Developer PowerShell | Include the x64/x86 and ARM64 C++ tools. | -| Visual C++ Clang and CMake tools | `where.exe clang.exe`; `where.exe cmake.exe` | `bindgen` needs host-native `libclang.dll`; bundled Z3 uses CMake and MSBuild. | +| Visual Studio 2022 or newer with **MSVC v143** + **Windows 11 SDK** | `where.exe cl.exe` from a VS Developer PowerShell | Include the x64/x86 and ARM64 C++ tools. The wrapper discovers installed release directories such as `18` and `2022`. | +| Visual C++ ARM64 Spectre-mitigated libraries | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre -property installationPath` | Required by ARM64 dependencies that select the Spectre runtime. | +| Visual C++ Clang and CMake tools | `where.exe clang-cl.exe`; `where.exe cmake.exe` | `bindgen` needs host-native `libclang.dll`; ARM64 crypto crates use `clang-cl`; bundled Z3 uses CMake with native MSVC and Ninja for x64-to-ARM64 builds. | | Rust ≥ 1.88 via rustup with MSVC targets | `rustc --version` | `winget install Rustlang.Rustup` | | mise CLI for task orchestration only | `mise --version` | https://mise.jdx.dev/installing-mise.html | | Git ≥ 2.40 | `git --version` | `winget install Git.Git` | @@ -213,6 +214,11 @@ if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { ARM64 typically surfaces the same dependency issues as x64. If x64 passes but ARM64 fails, the fault is almost always a native dependency (a `*-sys` crate without ARM64 prebuilds) or an inline-asm block lacking aarch64 paths. Either find a pure-Rust replacement or add ARM64 to the existing cfg gate. +On an x64 host this is a cross-build. The wrapper validates the ARM64 compiler +and Spectre libraries, adds host-native LLVM and Ninja to `PATH`, lets ARM64 +crypto crates select `clang-cl`, and keeps bundled Z3 on native MSVC `cl.exe`. +Use a short absolute `CARGO_TARGET_DIR` if Windows path-length limits are hit. + ### Step 8: mise build --release (both targets) ```powershell diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index c3148160b1..433162dc33 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -173,9 +173,13 @@ The Windows build path is additive. Keep the repository's Linux `mise run ci`, d Use `--skip-tools` for Windows CI and automation. Rust must come from rustup with MSVC targets, and Visual Studio Build Tools must provide the linker and SDK. Because `--skip-tools` does not provision mise-managed tools, the Windows wrapper clears inherited `RUSTC_WRAPPER=sccache` before invoking Cargo. The wrapper excludes unsupported driver packages as top-level workspace targets for Windows check/test, but those library stubs still compile when the gateway depends on them. The wrapper may discover `VsDevCmd.bat`, but it must not install Visual Studio, Rust, Docker, Kubernetes, Podman, VM tooling, WSL, or Hyper-V. -ARM64 validation requires the Visual Studio ARM64 C++ tools, host-native -`libclang.dll`, CMake tools for bundled Z3, and an ARM64-capable Windows SDK. -Test tasks reject a target that does not match the Windows host architecture. +ARM64 validation requires the Visual Studio ARM64 C++ tools, ARM64 +Spectre-mitigated libraries, host-native `libclang.dll` and `clang-cl.exe`, +CMake tools, Ninja, and an ARM64-capable Windows SDK. During x64-to-ARM64 +check/build, ARM64 crypto crates use `clang-cl` while bundled Z3 stays on +native MSVC `cl.exe` with Ninja. Use a short `CARGO_TARGET_DIR` if Windows +path-length limits are reached. Test tasks reject a target that does not match +the Windows host architecture. ## What NOT to do in this skill diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index ca2993ffa1..fa0e2a54e6 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -55,8 +55,10 @@ Windows validation is exposed through `tasks/windows.toml`: | `windows:ci` | Run check, build, test, unsupported-contract tests, and artifact reporting. | The Windows tasks call `tasks/scripts/windows-msvc.ps1`. The wrapper discovers -Visual Studio's `VsDevCmd.bat`, adds rustup MSVC targets, clears inherited -`RUSTC_WRAPPER`, and keeps build artifacts under the normal Cargo target tree. +Visual Studio's `VsDevCmd.bat` with `vswhere` or by enumerating installed +release directories, validates the requested compiler and ARM64 Spectre +libraries, adds rustup MSVC targets, clears inherited `RUSTC_WRAPPER`, and +keeps build artifacts under the normal Cargo target tree. Test tasks require the Rust target architecture to match the Windows host, so an ARM64 test result is native coverage rather than x64 emulation coverage. By default it enables bundled Z3 for reproducible Windows builds. When @@ -68,10 +70,14 @@ The lane uses `mise run --skip-tools windows:*` because Windows Rust comes from rustup and linking comes from Visual Studio Build Tools. Mise orchestrates the tasks; it does not own the Windows toolchain. -ARM64 validation requires the Visual Studio ARM64 MSVC tools, host-native Clang -tools for `libclang.dll`, CMake tools for bundled Z3, and an ARM64-capable -Windows SDK. Artifact hashing uses .NET SHA256 directly because module -autoloading in the mise-launched Windows PowerShell process is not guaranteed. +ARM64 validation requires the Visual Studio ARM64 MSVC tools, ARM64 +Spectre-mitigated libraries, host-native Clang tools, CMake tools, and an +ARM64-capable Windows SDK. Clang provides `libclang.dll` for `bindgen` and +`clang-cl.exe` for ARM64 crypto dependencies. During x64-to-ARM64 check/build, +the wrapper uses Ninja with native MSVC `cl.exe` for bundled Z3 so the Z3 build +does not inherit the crypto crates' compiler requirement. Artifact hashing uses +.NET SHA256 directly because module autoloading in the mise-launched Windows +PowerShell process is not guaranteed. ## CI Shape @@ -84,6 +90,9 @@ mise run --skip-tools windows:test:x64 mise run --skip-tools windows:test:unsupported:x64 ``` +The ARM64 check and release build in this x64 job are cross-builds. Native +ARM64 tests remain exclusive to an ARM64 runner. + The ARM64 job is scaffolded but disabled until a Windows ARM64 runner is available. Once enabled, it should run check, release build, native workspace tests, and the focused unsupported-driver contracts for diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index cb53f57a8a..2b1ba3a0df 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -43,6 +43,99 @@ $BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-pro $Z3WorkspaceFeatures = $BundledZ3WorkspaceFeatures $Z3ServerFeatures = $BundledZ3ServerFeatures +function Get-VsInstallRoots { + $programFiles = @( + [Environment]::GetEnvironmentVariable("ProgramFiles"), + [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + ) | Where-Object { $_ } + $editions = @("Enterprise", "Professional", "Community", "BuildTools") + $candidates = @() + + foreach ($programFilesRoot in $programFiles) { + $vsRoot = Join-Path $programFilesRoot "Microsoft Visual Studio" + if (-not (Test-Path $vsRoot -PathType Container)) { + continue + } + foreach ($releaseDir in Get-ChildItem $vsRoot -Directory) { + foreach ($edition in $editions) { + $installRoot = Join-Path $releaseDir.FullName $edition + $vsDevCmd = Join-Path $installRoot "Common7\Tools\VsDevCmd.bat" + if (-not (Test-Path $vsDevCmd -PathType Leaf)) { + continue + } + + $toolsetVersion = [version] "0.0" + $versionFile = Join-Path $installRoot "VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt" + if (Test-Path $versionFile -PathType Leaf) { + try { + $toolsetVersion = [version] ((Get-Content $versionFile -Raw).Trim()) + } catch { + $toolsetVersion = [version] "0.0" + } + } + $candidates += [pscustomobject]@{ + Root = $installRoot + ToolsetVersion = $toolsetVersion + } + } + } + } + + return @($candidates | Sort-Object ToolsetVersion -Descending | Select-Object -ExpandProperty Root -Unique) +} + +function Get-DefaultMsvcToolsetRoot([string] $VsInstallRoot) { + $versionFile = Join-Path $VsInstallRoot "VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt" + if (Test-Path $versionFile -PathType Leaf) { + $version = (Get-Content $versionFile -Raw).Trim() + $toolsetRoot = Join-Path $VsInstallRoot "VC\Tools\MSVC\$version" + if (Test-Path $toolsetRoot -PathType Container) { + return (Resolve-Path $toolsetRoot).Path + } + } + + $toolsetsRoot = Join-Path $VsInstallRoot "VC\Tools\MSVC" + if (Test-Path $toolsetsRoot -PathType Container) { + $toolset = Get-ChildItem $toolsetsRoot -Directory | + Sort-Object { try { [version] $_.Name } catch { [version] "0.0" } } -Descending | + Select-Object -First 1 + if ($toolset) { + return $toolset.FullName + } + } + + return $null +} + +function Test-VsInstanceSupportsTarget([string] $VsInstallRoot, [string] $RustTarget) { + $toolsetRoot = Get-DefaultMsvcToolsetRoot $VsInstallRoot + if (-not $toolsetRoot) { + return $false + } + + $hostToolsDir = switch (Get-HostArch) { + "arm64" { "Hostarm64" } + default { "Hostx64" } + } + $targetToolsDir = switch (Get-VsTargetArch $RustTarget) { + "arm64" { "arm64" } + default { "x64" } + } + $compiler = Join-Path $toolsetRoot "bin\$hostToolsDir\$targetToolsDir\cl.exe" + if (-not (Test-Path $compiler -PathType Leaf)) { + return $false + } + + if ($RustTarget -eq "aarch64-pc-windows-msvc") { + $spectreLibs = Join-Path $toolsetRoot "lib\spectre\arm64" + if (-not (Test-Path $spectreLibs -PathType Container)) { + return $false + } + } + + return $true +} + function Resolve-VsDevCmd([string] $RustTarget) { if ($env:OPENSHELL_VSDEVCMD -and (Test-Path $env:OPENSHELL_VSDEVCMD)) { return (Resolve-Path $env:OPENSHELL_VSDEVCMD).Path @@ -55,35 +148,37 @@ function Resolve-VsDevCmd([string] $RustTarget) { $vswhere = $null } if ($vswhere -and (Test-Path $vswhere)) { - $requiredComponent = switch ($RustTarget) { - "x86_64-pc-windows-msvc" { "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" } - "aarch64-pc-windows-msvc" { "Microsoft.VisualStudio.Component.VC.Tools.ARM64" } + $requiredComponents = switch ($RustTarget) { + "x86_64-pc-windows-msvc" { @("Microsoft.VisualStudio.Component.VC.Tools.x86.x64") } + "aarch64-pc-windows-msvc" { + @( + "Microsoft.VisualStudio.Component.VC.Tools.ARM64", + "Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre" + ) + } default { throw "Unsupported target: $RustTarget" } } - $found = & $vswhere -latest -products * -requires $requiredComponent -find "Common7\Tools\VsDevCmd.bat" | Select-Object -First 1 + $found = & $vswhere -latest -products * -requires $requiredComponents -find "Common7\Tools\VsDevCmd.bat" | Select-Object -First 1 if ($found -and (Test-Path $found)) { - return (Resolve-Path $found).Path + $resolved = (Resolve-Path $found).Path + $installRoot = (Resolve-Path (Join-Path (Split-Path -Parent $resolved) "..\..")).Path + if (Test-VsInstanceSupportsTarget $installRoot $RustTarget) { + return $resolved + } } } - $programFiles = @( - [Environment]::GetEnvironmentVariable("ProgramFiles"), - [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") - ) | Where-Object { $_ } - $versions = @("18", "2022", "17") - $editions = @("Enterprise", "Professional", "Community", "BuildTools") - foreach ($root in $programFiles) { - foreach ($version in $versions) { - foreach ($edition in $editions) { - $candidate = Join-Path $root "Microsoft Visual Studio\$version\$edition\Common7\Tools\VsDevCmd.bat" - if (Test-Path $candidate) { - return (Resolve-Path $candidate).Path - } - } + foreach ($installRoot in Get-VsInstallRoots) { + if (Test-VsInstanceSupportsTarget $installRoot $RustTarget) { + $candidate = Join-Path $installRoot "Common7\Tools\VsDevCmd.bat" + return (Resolve-Path $candidate).Path } } - throw "Could not find VsDevCmd.bat. Install Visual Studio Build Tools, or set OPENSHELL_VSDEVCMD." + if ($RustTarget -eq "aarch64-pc-windows-msvc") { + throw "Could not find a Visual Studio instance with the ARM64 compiler and ARM64 Spectre-mitigated libraries. Install Microsoft.VisualStudio.Component.VC.Tools.ARM64 and Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre, or set OPENSHELL_VSDEVCMD." + } + throw "Could not find a Visual Studio instance with the x64 compiler. Install Microsoft.VisualStudio.Component.VC.Tools.x86.x64, or set OPENSHELL_VSDEVCMD." } function Get-LibclangBinSubdir { @@ -114,21 +209,11 @@ function Resolve-LibclangPath { } } - $programFiles = @( - [Environment]::GetEnvironmentVariable("ProgramFiles"), - [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") - ) | Where-Object { $_ } - $versions = @("2022", "18", "17") - $editions = @("Enterprise", "Professional", "Community", "BuildTools") - foreach ($root in $programFiles) { - foreach ($version in $versions) { - foreach ($edition in $editions) { - $candidateDir = Join-Path $root "Microsoft Visual Studio\$version\$edition\VC\Tools\Llvm\$subdir\bin" - $candidate = Join-Path $candidateDir "libclang.dll" - if (Test-Path $candidate) { - return (Resolve-Path $candidateDir).Path - } - } + foreach ($installRoot in Get-VsInstallRoots) { + $candidateDir = Join-Path $installRoot "VC\Tools\Llvm\$subdir\bin" + $candidate = Join-Path $candidateDir "libclang.dll" + if (Test-Path $candidate -PathType Leaf) { + return (Resolve-Path $candidateDir).Path } } @@ -140,6 +225,62 @@ function Resolve-LibclangPath { throw "Could not find libclang.dll. Install Visual Studio C++ Clang tools, or set LIBCLANG_PATH to the directory containing libclang.dll." } +function Resolve-NinjaPath { + $fromPath = Get-Command ninja.exe -ErrorAction SilentlyContinue + if ($fromPath) { + return $fromPath.Source + } + + $programFilesX86 = [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + if ($programFilesX86) { + $vswhere = Join-Path $programFilesX86 "Microsoft Visual Studio\Installer\vswhere.exe" + } else { + $vswhere = $null + } + if ($vswhere -and (Test-Path $vswhere)) { + $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.CMake.Project -find "Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" | Select-Object -First 1 + if ($found -and (Test-Path $found -PathType Leaf)) { + return (Resolve-Path $found).Path + } + } + + foreach ($installRoot in Get-VsInstallRoots) { + $candidate = Join-Path $installRoot "Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + if (Test-Path $candidate -PathType Leaf) { + return (Resolve-Path $candidate).Path + } + } + + throw "Could not find ninja.exe. Install Microsoft.VisualStudio.Component.VC.CMake.Project." +} + +function Add-PathEntry([string] $Directory) { + if (($env:PATH -split ";") -notcontains $Directory) { + $env:PATH = "$Directory;$env:PATH" + } +} + +function Configure-Arm64CrossBuild([string[]] $RustTargets) { + if ((Get-HostArch) -ne "amd64" -or $RustTargets -notcontains "aarch64-pc-windows-msvc") { + return + } + + $clangCl = Join-Path $env:LIBCLANG_PATH "clang-cl.exe" + if (-not (Test-Path $clangCl -PathType Leaf)) { + throw "ARM64 cross-compilation requires host-native clang-cl.exe next to libclang.dll. Install Microsoft.VisualStudio.Component.VC.Llvm.Clang." + } + Add-PathEntry $env:LIBCLANG_PATH + + $ninja = Resolve-NinjaPath + Add-PathEntry (Split-Path -Parent $ninja) + [Environment]::SetEnvironmentVariable("CMAKE_GENERATOR_aarch64_pc_windows_msvc", "Ninja", "Process") + + Write-Host "==> ARM64 cross-build toolchain" + Write-Host " clang-cl: $clangCl" + Write-Host " ninja: $ninja" + Write-Host " Z3: MSVC cl.exe with Ninja" +} + function Get-HostArch { switch ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString()) { "Arm64" { "arm64" } @@ -241,7 +382,24 @@ function Invoke-VsCargo { $targetArch = Get-VsTargetArch $RustTarget $hostArch = Get-HostArch $logPath = Join-Path $LogDir $LogName - $cmd = "call `"$vsDevCmd`" -arch=$targetArch -host_arch=$hostArch && set `"CARGO_TARGET_DIR=$TargetDir`" && set `"CARGO_INCREMENTAL=0`" && set `"RUSTC_WRAPPER=`" && $CargoArgs" + $environmentSetup = @( + "set `"CARGO_TARGET_DIR=$TargetDir`"", + "set `"CARGO_INCREMENTAL=0`"", + "set `"RUSTC_WRAPPER=`"" + ) + if ($hostArch -eq "amd64" -and $RustTarget -eq "aarch64-pc-windows-msvc") { + # Let cmake-rs select MSVC cl.exe for bundled Z3. AWS-LC selects + # clang-cl inside its own ARM64 build script. + $environmentSetup += @( + "set `"CC=`"", + "set `"CXX=`"", + "set `"CC_aarch64-pc-windows-msvc=`"", + "set `"CXX_aarch64-pc-windows-msvc=`"", + "set `"CC_aarch64_pc_windows_msvc=`"", + "set `"CXX_aarch64_pc_windows_msvc=`"" + ) + } + $cmd = "call `"$vsDevCmd`" -arch=$targetArch -host_arch=$hostArch && $($environmentSetup -join ' && ') && $CargoArgs" Write-Host "==> $CargoArgs" Write-Host " target: $RustTarget" @@ -334,12 +492,18 @@ function Show-Artifacts([string[]] $RustTargets) { $rows | Format-Table -AutoSize } +if ($Action -eq "ci" -and (Get-HostArch) -ne "amd64") { + throw "windows:ci is an x64-host contract. On ARM64, run windows:check:arm64, windows:build:arm64, windows:test:arm64, windows:test:unsupported:arm64, and windows:artifacts explicitly." +} + $targets = Get-SelectedTargets $Target $z3Features = Configure-Z3 $Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures $Z3ServerFeatures = $z3Features.ServerFeatures $env:LIBCLANG_PATH = Resolve-LibclangPath +Add-PathEntry $env:LIBCLANG_PATH Write-Host "==> LIBCLANG_PATH=$env:LIBCLANG_PATH" +Configure-Arm64CrossBuild $targets switch ($Action) { "check" { From 7ee6ebb2f648a442a31e6597591caf8e16d38e4a Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Sat, 18 Jul 2026 18:01:14 -0700 Subject: [PATCH 19/30] fix(windows): scope ARM64 toolchain preflight Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- tasks/scripts/windows-msvc.ps1 | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 2b1ba3a0df..590955a806 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -497,13 +497,21 @@ if ($Action -eq "ci" -and (Get-HostArch) -ne "amd64") { } $targets = Get-SelectedTargets $Target -$z3Features = Configure-Z3 -$Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures -$Z3ServerFeatures = $z3Features.ServerFeatures -$env:LIBCLANG_PATH = Resolve-LibclangPath -Add-PathEntry $env:LIBCLANG_PATH -Write-Host "==> LIBCLANG_PATH=$env:LIBCLANG_PATH" -Configure-Arm64CrossBuild $targets +if ($Action -in @("test", "test-unsupported")) { + foreach ($rustTarget in $targets) { + Assert-NativeTestTarget $rustTarget + } +} + +if ($Action -in @("check", "build", "test", "test-unsupported", "ci")) { + $z3Features = Configure-Z3 + $Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures + $Z3ServerFeatures = $z3Features.ServerFeatures + $env:LIBCLANG_PATH = Resolve-LibclangPath + Add-PathEntry $env:LIBCLANG_PATH + Write-Host "==> LIBCLANG_PATH=$env:LIBCLANG_PATH" + Configure-Arm64CrossBuild $targets +} switch ($Action) { "check" { From e5f59869e64701aecba681ab8833450ca1a33b97 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Mon, 20 Jul 2026 13:57:02 -0700 Subject: [PATCH 20/30] fix(windows): restore compatibility after GitHub sync Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- tasks/scripts/windows-msvc.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 590955a806..e0cb0ecbef 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -38,7 +38,7 @@ if ([string]::IsNullOrWhiteSpace($TargetDir)) { } $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm" -$BundledZ3WorkspaceFeatures = "--features openshell-cli/bundled-z3,openshell-prover/bundled-z3" +$BundledZ3WorkspaceFeatures = "--features openshell-prover/bundled-z3" $BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-prover/bundled-z3" $Z3WorkspaceFeatures = $BundledZ3WorkspaceFeatures $Z3ServerFeatures = $BundledZ3ServerFeatures From 170bf6f09f3db1eb0685c7e9e8a5b75265b9ebee Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Wed, 22 Jul 2026 09:28:33 -0700 Subject: [PATCH 21/30] fix(windows): avoid rate-limited Z3 source lookup Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 6 ++ .../build-openshell-mxc-windows/reference.md | 11 +++ architecture/windows-msvc-build.md | 8 +- tasks/scripts/windows-msvc.ps1 | 82 +++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 0defc1fbf9..f1965c50fd 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -60,6 +60,7 @@ The skill reads these environment variables (PowerShell syntax): | `$env:OPENSHELL_WXC_EXEC_PATH` | unset | Optional path to `wxc-exec.exe`. Validated for existence and architecture match only. Not used in this build-only skill. | | `$env:OPENSHELL_MXC_SKIP_ARM64` | `0` | Set to `1` to skip aarch64 build (e.g., for fast x64-only iterations). | | `$env:OPENSHELL_MXC_FORK_BRANCH` | `windows-mxc-build` | Branch name created in the fork. | +| `$env:Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned source cached under `CARGO_TARGET_DIR` | Reuse an existing Z3 source tree containing `src/api/z3.h`; otherwise the wrapper fetches and caches the pinned revision automatically. | ## Workflow @@ -195,6 +196,11 @@ The PowerShell wrapper must: Use `mise run --skip-tools windows:*` in GitHub Actions and local Windows automation. `--skip-tools` is intentional: this repo should not ask mise to install Rust on Windows because the MSVC flow relies on rustup plus Visual Studio Build Tools. +For bundled Z3, the wrapper fetches the revision pinned by `z3-sys` through +Git, caches it under `CARGO_TARGET_DIR`, and sets +`Z3_SYS_BUNDLED_DIR_OVERRIDE`. This bypasses the unauthenticated GitHub +Contents API lookup that can fail with HTTP 403 on shared networks. + ### Step 6: mise check on x86_64-pc-windows-msvc ```powershell diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index 433162dc33..8768f89702 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -78,6 +78,12 @@ Same pattern — gate with `#[cfg(unix)]`. `libsecret` only works on Linux. If `openshell-providers` pulls it in, gate it. The build-only skill does not need credential storage on Windows; that is a follow-on skill. +### Bundled Z3 fails with HTTP 403 + +The wrapper should fetch the pinned Z3 revision through Git before Cargo starts. +If that prefetch fails, inspect the reported partial checkout and verify that +Git can reach `https://github.com/Z3Prover/z3.git`. + ## Cfg gating patterns ### Module-level @@ -181,6 +187,11 @@ native MSVC `cl.exe` with Ninja. Use a short `CARGO_TARGET_DIR` if Windows path-length limits are reached. Test tasks reject a target that does not match the Windows host architecture. +Bundled Z3 source is pinned by revision, fetched through Git, and cached under +`CARGO_TARGET_DIR`. The wrapper sets `Z3_SYS_BUNDLED_DIR_OVERRIDE` so `z3-sys` +does not make an unauthenticated GitHub Contents API call. An explicit override +must point to a source tree containing `src/api/z3.h`. + ## What NOT to do in this skill - Do not implement named-pipe IPC. Stubs only. (Belongs to the follow-on MXC driver skill.) diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index fa0e2a54e6..3049555daf 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -64,7 +64,13 @@ an ARM64 test result is native coverage rather than x64 emulation coverage. By default it enables bundled Z3 for reproducible Windows builds. When `Z3_LIBRARY_PATH_OVERRIDE` points at a directory containing `libz3.lib`, the wrapper uses that system Z3 instead and requires `Z3_SYS_Z3_HEADER` to point at -the full path to `z3.h`. +the full path to `z3.h`. For bundled builds, the wrapper fetches the Z3 source +revision pinned by `z3-sys` through Git, caches it under `CARGO_TARGET_DIR`, and +sets `Z3_SYS_BUNDLED_DIR_OVERRIDE`. This avoids the unauthenticated GitHub API +lookup in the `z3-sys` build script, which can fail with HTTP 403 when a shared +runner or developer network exhausts its API rate limit. An explicitly set +`Z3_SYS_BUNDLED_DIR_OVERRIDE` remains supported and must contain +`src/api/z3.h`. The lane uses `mise run --skip-tools windows:*` because Windows Rust comes from rustup and linking comes from Visual Studio Build Tools. Mise orchestrates the diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index e0cb0ecbef..c27e4f4bd2 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -40,6 +40,10 @@ if ([string]::IsNullOrWhiteSpace($TargetDir)) { $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm" $BundledZ3WorkspaceFeatures = "--features openshell-prover/bundled-z3" $BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-prover/bundled-z3" +$BundledZ3Repository = "https://github.com/Z3Prover/z3.git" +$BundledZ3SysVersion = "0.10.9" +# This is the matching z3-sys submodule revision. Update both pins together. +$BundledZ3Revision = "ddb49568d3520e99799e364fb22f35fc67d887b1" $Z3WorkspaceFeatures = $BundledZ3WorkspaceFeatures $Z3ServerFeatures = $BundledZ3ServerFeatures @@ -330,9 +334,87 @@ function Resolve-Z3HeaderPath([string] $HeaderPath) { return (Resolve-Path $HeaderPath).Path } +function Assert-BundledZ3Source([string] $SourcePath, [string] $ExpectedRevision) { + if (-not (Test-Path $SourcePath -PathType Container)) { + throw "Bundled Z3 source directory does not exist: $SourcePath" + } + + $header = Join-Path $SourcePath "src\api\z3.h" + if (-not (Test-Path $header -PathType Leaf)) { + throw "Bundled Z3 source directory does not contain src\api\z3.h: $SourcePath" + } + + if (-not [string]::IsNullOrWhiteSpace($ExpectedRevision)) { + $actualRevision = (& git -C $SourcePath rev-parse HEAD 2>$null) + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($actualRevision)) { + throw "Could not verify the bundled Z3 source revision at: $SourcePath" + } + if ($actualRevision.Trim() -ne $ExpectedRevision) { + throw "Bundled Z3 source revision mismatch at ${SourcePath}: expected $ExpectedRevision, found $($actualRevision.Trim())" + } + } + + return (Resolve-Path $SourcePath).Path +} + +function Resolve-BundledZ3Source { + if (-not [string]::IsNullOrWhiteSpace($env:Z3_SYS_BUNDLED_DIR_OVERRIDE)) { + return Assert-BundledZ3Source $env:Z3_SYS_BUNDLED_DIR_OVERRIDE "" + } + + $cargoLock = Get-Content (Join-Path $RepoRoot "Cargo.lock") -Raw + $packagePattern = '(?ms)^\[\[package\]\]\s+name = "z3-sys"\s+version = "([^"]+)"' + $packageMatches = [regex]::Matches($cargoLock, $packagePattern) + if ($packageMatches.Count -ne 1 -or $packageMatches[0].Groups[1].Value -ne $BundledZ3SysVersion) { + throw "Bundled Z3 source pin expects z3-sys $BundledZ3SysVersion. Update the version and revision pins for the z3-sys version in Cargo.lock." + } + + $revisionPrefix = $BundledZ3Revision.Substring(0, 12) + $sourcePath = Join-Path $TargetDir "z3-source-$revisionPrefix" + if (Test-Path $sourcePath) { + return Assert-BundledZ3Source $sourcePath $BundledZ3Revision + } + + if (-not (Get-Command git.exe -ErrorAction SilentlyContinue)) { + throw "Bundled Z3 source preparation requires git.exe on PATH." + } + if (-not (Test-Path $TargetDir -PathType Container)) { + New-Item -ItemType Directory -Force -Path $TargetDir | Out-Null + } + + $stagingPath = "$sourcePath.partial-$([guid]::NewGuid().ToString('N'))" + Write-Host "==> Fetching bundled Z3 source" + Write-Host " repository: $BundledZ3Repository" + Write-Host " revision: $BundledZ3Revision" + Write-Host " cache: $sourcePath" + + & git init --quiet $stagingPath + if ($LASTEXITCODE -ne 0) { + throw "git init failed while preparing bundled Z3 source at: $stagingPath" + } + & git -C $stagingPath remote add origin $BundledZ3Repository + if ($LASTEXITCODE -ne 0) { + throw "git remote add failed while preparing bundled Z3 source at: $stagingPath" + } + & git -C $stagingPath fetch --quiet --depth 1 origin $BundledZ3Revision + if ($LASTEXITCODE -ne 0) { + throw "git fetch failed for bundled Z3 revision $BundledZ3Revision. Partial source remains at: $stagingPath" + } + & git -C $stagingPath checkout --quiet --detach FETCH_HEAD + if ($LASTEXITCODE -ne 0) { + throw "git checkout failed for bundled Z3 revision $BundledZ3Revision. Partial source remains at: $stagingPath" + } + + Assert-BundledZ3Source $stagingPath $BundledZ3Revision | Out-Null + Move-Item -Path $stagingPath -Destination $sourcePath + return Assert-BundledZ3Source $sourcePath $BundledZ3Revision +} + function Configure-Z3 { if ([string]::IsNullOrWhiteSpace($env:Z3_LIBRARY_PATH_OVERRIDE)) { Write-Host "==> Z3: bundled" + $env:Z3_SYS_BUNDLED_DIR_OVERRIDE = Resolve-BundledZ3Source + Write-Host " Z3_SYS_BUNDLED_DIR_OVERRIDE=$env:Z3_SYS_BUNDLED_DIR_OVERRIDE" return [pscustomobject]@{ WorkspaceFeatures = $BundledZ3WorkspaceFeatures ServerFeatures = $BundledZ3ServerFeatures From dc4d03f990dcda4b2b1582ea75eb4d9eef6e14b3 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Wed, 22 Jul 2026 09:56:51 -0700 Subject: [PATCH 22/30] fix(mise): skip Helm checks on Windows Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- tasks/helm.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tasks/helm.toml b/tasks/helm.toml index 24b6667b1d..dd5128ef64 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -20,6 +20,7 @@ run = """ exit 1 fi """ +run_windows = "echo Skipping helm:docs:check: Helm validation is not part of the native Windows lane." hide = true ["helm:lint"] @@ -38,6 +39,7 @@ run = """ done echo "All variants passed." """ +run_windows = "echo Skipping helm:lint: Helm validation is not part of the native Windows lane." ["helm:test"] description = "Run Helm chart unit tests" @@ -49,6 +51,7 @@ run = """ helm dependency build deploy/helm/openshell helm unittest deploy/helm/openshell """ +run_windows = "echo Skipping helm:test: Helm validation is not part of the native Windows lane." ["helm:skaffold:dev"] description = "Run skaffold dev for deploy/helm/openshell (iterative deploy)" From 0677ab17a2af18eb64bbfb81120b660ccd2d8169 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Wed, 22 Jul 2026 12:12:22 -0700 Subject: [PATCH 23/30] fix(windows): support repository pre-commit checks Signed-off-by: Shailendra Singh Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 7 +++ .../build-openshell-mxc-windows/reference.md | 7 +++ architecture/windows-msvc-build.md | 12 ++++- .../openshell-bootstrap/src/build_windows.rs | 2 + crates/openshell-server/src/config_file.rs | 1 + .../src/identity.rs | 39 +++++++++----- python/openshell/sandbox.py | 12 ++++- scripts/update_license_headers.py | 2 +- tasks/markdown.toml | 1 + tasks/rust.toml | 2 + tasks/scripts/windows-msvc.ps1 | 51 +++++++++++++++++-- tasks/test.toml | 3 ++ 12 files changed, 119 insertions(+), 20 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index f1965c50fd..4fa79b6c92 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -201,6 +201,13 @@ Git, caches it under `CARGO_TARGET_DIR`, and sets `Z3_SYS_BUNDLED_DIR_OVERRIDE`. This bypasses the unauthenticated GitHub Contents API lookup that can fail with HTTP 403 on shared networks. +The repository-wide `mise run pre-commit` task is supported on Windows. +`tasks/rust.toml` and `tasks/test.toml` route compiler-bearing checks through +the wrapper for the native host target, while `tasks/markdown.toml` provides a +Windows-safe dependency setup command. Keep existing Unix `run` bodies +unchanged when adding `run_windows` behavior. Linux installer and packaging +asset tests skip explicitly; cross-platform checks continue to run. + ### Step 6: mise check on x86_64-pc-windows-msvc ```powershell diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index 8768f89702..2857448208 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -165,6 +165,13 @@ After every cfg gate change, run `cargo check --workspace` on the Linux baseline The Windows build path is additive. Keep the repository's Linux `mise run ci`, default Cargo tasks, and Linux documentation unchanged. Windows automation lives in `tasks/windows.toml` and delegates to `tasks/scripts/windows-msvc.ps1`. +On Windows, `mise run pre-commit` routes `rust:check`, `rust:lint`, and +`test:rust` through the same wrapper for the host-native target. Shared task +definitions retain their Unix commands. Only Linux installer and +service/RPM-packaging tests skip. The Windows Clippy command allows unused +imports, dead code, and unused async functions caused by cfg-gated stubs; other +warnings remain errors. + | Task | Expected behavior | |---|---| | `mise run --skip-tools windows:check:x64` | Runs x64 MSVC `cargo check --workspace` | diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 3049555daf..c27a556e54 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -10,7 +10,9 @@ Podman, or VM runtime host. - Keep the Linux and macOS build paths unchanged. - Preserve gateway configuration parsing for all existing compute driver names. - Return clear unsupported errors when a Windows gateway is configured to use Docker, Kubernetes, Podman, or VM. -- Validate the Windows lane through mise tasks that are separate from the default Linux `ci` task. +- Keep dedicated `windows:*` validation tasks while allowing the repository-wide + `pre-commit` task to delegate compiler-bearing Rust checks to the native + Windows MSVC environment. ## Non-Goals @@ -59,6 +61,14 @@ Visual Studio's `VsDevCmd.bat` with `vswhere` or by enumerating installed release directories, validates the requested compiler and ARM64 Spectre libraries, adds rustup MSVC targets, clears inherited `RUSTC_WRAPPER`, and keeps build artifacts under the normal Cargo target tree. +On Windows, the generic `rust:check`, `rust:lint`, and `test:rust` tasks call +the same wrapper with the host-native MSVC target. The wrapper preserves the +Unix Cargo commands on Linux and macOS, excludes unsupported Windows runtime +packages, and runs the server test-support suite separately. Windows Clippy +continues to deny all warnings except unused imports, dead code, and unused +async functions caused by cfg-gated Windows stubs. Repository-wide pre-commit +skips only Linux-specific installer and packaging-asset tests; its +cross-platform Python, Markdown, license, and documentation checks still run. Test tasks require the Rust target architecture to match the Windows host, so an ARM64 test result is native coverage rather than x64 emulation coverage. By default it enables bundled Z3 for reproducible Windows builds. When diff --git a/crates/openshell-bootstrap/src/build_windows.rs b/crates/openshell-bootstrap/src/build_windows.rs index 6337919b21..93af1345d3 100644 --- a/crates/openshell-bootstrap/src/build_windows.rs +++ b/crates/openshell-bootstrap/src/build_windows.rs @@ -8,6 +8,8 @@ use std::path::Path; use miette::Result; +// Keep this stub's signature aligned with the supported-platform implementation. +#[allow(clippy::implicit_hasher)] pub async fn build_local_image( _dockerfile_path: &Path, _tag: &str, diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index c4e0cbc959..8d3b346141 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -239,6 +239,7 @@ pub enum ConfigFileError { /// /// Returns `Ok(ConfigFile::default())` for an empty file (the gateway then /// falls back entirely to CLI/env/built-in defaults). +#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] pub fn load(path: &Path) -> Result { let contents = std::fs::read_to_string(path).map_err(|source| ConfigFileError::Io { path: path.to_path_buf(), diff --git a/crates/openshell-supervisor-network/src/identity.rs b/crates/openshell-supervisor-network/src/identity.rs index 5e89c35031..4f90888378 100644 --- a/crates/openshell-supervisor-network/src/identity.rs +++ b/crates/openshell-supervisor-network/src/identity.rs @@ -21,10 +21,8 @@ use tracing::debug; #[derive(Clone)] struct FileFingerprint { len: u64, - mtime_sec: i64, - mtime_nsec: i64, - ctime_sec: i64, - ctime_nsec: i64, + mtime: Option<(i64, i64)>, + ctime: Option<(i64, i64)>, #[cfg(unix)] dev: u64, #[cfg(unix)] @@ -33,12 +31,20 @@ struct FileFingerprint { impl FileFingerprint { fn from_metadata(metadata: &Metadata) -> Self { + #[cfg(unix)] + let (mtime, ctime) = ( + Some((metadata.mtime(), metadata.mtime_nsec())), + Some((metadata.ctime(), metadata.ctime_nsec())), + ); + #[cfg(not(unix))] + let (mtime, ctime) = ( + metadata.modified().ok().and_then(system_time_parts), + metadata.created().ok().and_then(system_time_parts), + ); Self { len: metadata.len(), - mtime_sec: metadata.mtime(), - mtime_nsec: metadata.mtime_nsec(), - ctime_sec: metadata.ctime(), - ctime_nsec: metadata.ctime_nsec(), + mtime, + ctime, #[cfg(unix)] dev: metadata.dev(), #[cfg(unix)] @@ -47,13 +53,22 @@ impl FileFingerprint { } } +#[cfg(not(unix))] +fn system_time_parts(time: std::time::SystemTime) -> Option<(i64, i64)> { + let duration = time.duration_since(std::time::UNIX_EPOCH).ok()?; + let seconds = i64::try_from(duration.as_secs()).ok()?; + Some((seconds, i64::from(duration.subsec_nanos()))) +} + impl PartialEq for FileFingerprint { fn eq(&self, other: &Self) -> bool { self.len == other.len - && self.mtime_sec == other.mtime_sec - && self.mtime_nsec == other.mtime_nsec - && self.ctime_sec == other.ctime_sec - && self.ctime_nsec == other.ctime_nsec + && self.mtime.is_some() + && other.mtime.is_some() + && self.mtime == other.mtime + && self.ctime.is_some() + && other.ctime.is_some() + && self.ctime == other.ctime && { #[cfg(unix)] { diff --git a/python/openshell/sandbox.py b/python/openshell/sandbox.py index b62a9c8855..db0db85d85 100644 --- a/python/openshell/sandbox.py +++ b/python/openshell/sandbox.py @@ -1531,7 +1531,17 @@ def _write_to_disk(self, bundle: dict) -> None: f.write(payload) with contextlib.suppress(OSError): tmp_path.chmod(0o600) - tmp_path.replace(path) + for attempt in range(20): + try: + tmp_path.replace(path) + break + except PermissionError: + # Concurrent MoveFileExW calls can briefly lock the + # destination on Windows. Retry that transient sharing + # violation while preserving POSIX rename behavior. + if os.name != "nt" or attempt == 19: + raise + time.sleep(0.01) except BaseException: # Clean up our tmp on failure so we don't leave orphaned # `.oidc_token..tmp` files lying around. The replace diff --git a/scripts/update_license_headers.py b/scripts/update_license_headers.py index f56dbc2935..0f2d87ddd5 100755 --- a/scripts/update_license_headers.py +++ b/scripts/update_license_headers.py @@ -101,7 +101,7 @@ def find_repo_root() -> Path: def is_excluded(rel: Path) -> bool: """Return True if a path should be skipped.""" - rel_str = str(rel) + rel_str = rel.as_posix() # Exact filename exclusions. if rel.name in EXCLUDE_FILES: diff --git a/tasks/markdown.toml b/tasks/markdown.toml index fb070fd683..d0cba9d492 100644 --- a/tasks/markdown.toml +++ b/tasks/markdown.toml @@ -9,6 +9,7 @@ run = """ cd scripts/lint-mermaid npm ci --no-audit --no-fund --silent """ +run_windows = "npm --prefix scripts/lint-mermaid ci --no-audit --no-fund --silent" hide = true sources = ["scripts/lint-mermaid/package.json", "scripts/lint-mermaid/package-lock.json"] outputs = ["scripts/lint-mermaid/node_modules/.package-lock.json"] diff --git a/tasks/rust.toml b/tasks/rust.toml index f035193fd8..37099675b8 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -6,6 +6,7 @@ ["rust:check"] description = "Check all Rust crates for errors" run = "cargo check --workspace" +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 check native" hide = true ["rust:lint"] @@ -14,6 +15,7 @@ run = [ "cargo clippy --workspace --all-targets -- -D warnings", "cargo clippy --manifest-path e2e/rust/Cargo.toml --all-targets -- -D warnings", ] +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 lint native" hide = true ["rust:format"] diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index c27e4f4bd2..1331d5d123 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -6,11 +6,11 @@ [CmdletBinding()] param( [Parameter(Mandatory = $true, Position = 0)] - [ValidateSet("check", "build", "test", "test-unsupported", "artifacts", "ci")] + [ValidateSet("check", "lint", "build", "test", "test-precommit", "test-unsupported", "artifacts", "ci")] [string] $Action, [Parameter(Position = 1)] - [ValidateSet("x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", "all")] + [ValidateSet("x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", "native", "all")] [string] $Target = "all", [string] $LogDir @@ -37,7 +37,9 @@ if ([string]::IsNullOrWhiteSpace($TargetDir)) { $TargetDir = Join-Path $RepoRoot "target" } -$UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm" +$UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm --exclude openshell-supervisor-process" +$WindowsClippyPackageExcludes = $UnsupportedDriverPackageExcludes +$WindowsClippyLintArgs = "-D warnings -A dead-code -A unused-imports -A clippy::unused-async" $BundledZ3WorkspaceFeatures = "--features openshell-prover/bundled-z3" $BundledZ3ServerFeatures = "--features openshell-server/bundled-z3,openshell-prover/bundled-z3" $BundledZ3Repository = "https://github.com/Z3Prover/z3.git" @@ -309,6 +311,12 @@ function Assert-NativeTestTarget([string] $RustTarget) { } function Get-SelectedTargets([string] $RequestedTarget) { + if ($RequestedTarget -eq "native") { + switch (Get-HostArch) { + "arm64" { return @("aarch64-pc-windows-msvc") } + default { return @("x86_64-pc-windows-msvc") } + } + } if ($RequestedTarget -eq "all") { $targets = @("x86_64-pc-windows-msvc") if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { @@ -505,6 +513,17 @@ function Invoke-Check([string] $RustTarget) { -LogName "build-$RustTarget-check.log" } +function Invoke-Lint([string] $RustTarget) { + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo clippy --workspace --all-targets --no-deps $WindowsClippyPackageExcludes --target $RustTarget $Z3WorkspaceFeatures -- $WindowsClippyLintArgs" ` + -LogName "lint-$RustTarget-workspace.log" + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo clippy --manifest-path e2e/rust/Cargo.toml --all-targets --no-deps --target $RustTarget -- $WindowsClippyLintArgs" ` + -LogName "lint-$RustTarget-e2e.log" +} + function Invoke-Build([string] $RustTarget) { Invoke-VsCargo ` -RustTarget $RustTarget ` @@ -520,6 +539,18 @@ function Invoke-Test([string] $RustTarget) { -LogName "test-$RustTarget.log" } +function Invoke-PreCommitTest([string] $RustTarget) { + Assert-NativeTestTarget $RustTarget + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo test --workspace --exclude openshell-server $UnsupportedDriverPackageExcludes --target $RustTarget --no-fail-fast $Z3WorkspaceFeatures" ` + -LogName "test-$RustTarget-precommit-workspace.log" + Invoke-VsCargo ` + -RustTarget $RustTarget ` + -CargoArgs "cargo test -p openshell-server --features test-support --target $RustTarget --no-fail-fast $Z3ServerFeatures" ` + -LogName "test-$RustTarget-precommit-server.log" +} + function Invoke-UnsupportedContractTests([string] $RustTarget) { Assert-NativeTestTarget $RustTarget @@ -579,13 +610,13 @@ if ($Action -eq "ci" -and (Get-HostArch) -ne "amd64") { } $targets = Get-SelectedTargets $Target -if ($Action -in @("test", "test-unsupported")) { +if ($Action -in @("test", "test-precommit", "test-unsupported")) { foreach ($rustTarget in $targets) { Assert-NativeTestTarget $rustTarget } } -if ($Action -in @("check", "build", "test", "test-unsupported", "ci")) { +if ($Action -in @("check", "lint", "build", "test", "test-precommit", "test-unsupported", "ci")) { $z3Features = Configure-Z3 $Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures $Z3ServerFeatures = $z3Features.ServerFeatures @@ -601,6 +632,11 @@ switch ($Action) { Invoke-Check $rustTarget } } + "lint" { + foreach ($rustTarget in $targets) { + Invoke-Lint $rustTarget + } + } "build" { foreach ($rustTarget in $targets) { Invoke-Build $rustTarget @@ -612,6 +648,11 @@ switch ($Action) { Invoke-Test $rustTarget } } + "test-precommit" { + foreach ($rustTarget in $targets) { + Invoke-PreCommitTest $rustTarget + } + } "test-unsupported" { foreach ($rustTarget in $targets) { Invoke-UnsupportedContractTests $rustTarget diff --git a/tasks/test.toml b/tasks/test.toml index 96dde276ce..55fa2b5d45 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -21,6 +21,7 @@ hide = true ["test:install-sh"] description = "Run focused install.sh shell tests" run = "tasks/scripts/test-install-sh.sh" +run_windows = "echo Skipping test:install-sh: Linux glibc installer tests do not apply on Windows." hide = true ["test:build-env"] @@ -31,6 +32,7 @@ hide = true ["test:packaging-assets"] description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh" +run_windows = "echo Skipping test:packaging-assets: Linux service and RPM assets do not apply on Windows." hide = true [e2e] @@ -53,6 +55,7 @@ run = [ "cargo test --workspace --exclude openshell-server", "cargo test -p openshell-server --features test-support", ] +run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native" hide = true ["test:python"] From 7928f06a1c3e5965a4cfef4c13d5ff64e0ffcbc6 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Fri, 24 Jul 2026 02:35:48 -0500 Subject: [PATCH 24/30] fix(windows): stabilize native MSVC validation Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 15 ++-- .../build-openshell-mxc-windows/reference.md | 23 +++-- architecture/windows-msvc-build.md | 14 ++- crates/openshell-cli/src/run.rs | 4 +- crates/openshell-core/build.rs | 6 +- crates/openshell-core/src/paths.rs | 13 ++- crates/openshell-server/src/compute/mod.rs | 17 ++-- crates/openshell-server/src/compute/vm.rs | 8 +- crates/openshell-server/src/grpc/policy.rs | 9 ++ tasks/scripts/windows-msvc.ps1 | 88 ++++++++++--------- tasks/test.toml | 1 + 11 files changed, 115 insertions(+), 83 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 4fa79b6c92..0bd2ab4474 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -41,7 +41,7 @@ The skill targets a **Windows 11 host with CurrentBuild ≥ 26100**. Because the | Windows 11 build ≥ 26100 | `[System.Environment]::OSVersion.Version` | OS update | | Visual Studio 2022 or newer with **MSVC v143** + **Windows 11 SDK** | `where.exe cl.exe` from a VS Developer PowerShell | Include the x64/x86 and ARM64 C++ tools. The wrapper discovers installed release directories such as `18` and `2022`. | | Visual C++ ARM64 Spectre-mitigated libraries | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre -property installationPath` | Required by ARM64 dependencies that select the Spectre runtime. | -| Visual C++ Clang and CMake tools | `where.exe clang-cl.exe`; `where.exe cmake.exe` | `bindgen` needs host-native `libclang.dll`; ARM64 crypto crates use `clang-cl`; bundled Z3 uses CMake with native MSVC and Ninja for x64-to-ARM64 builds. | +| Visual C++ Clang and CMake tools | `where.exe clang-cl.exe`; `where.exe cmake.exe` | `bindgen` needs host-native `libclang.dll`; ARM64 crypto crates use `clang-cl`; bundled Z3 uses CMake's Visual Studio generator with native MSVC for x64-to-ARM64 builds. | | Rust ≥ 1.88 via rustup with MSVC targets | `rustc --version` | `winget install Rustlang.Rustup` | | mise CLI for task orchestration only | `mise --version` | https://mise.jdx.dev/installing-mise.html | | Git ≥ 2.40 | `git --version` | `winget install Git.Git` | @@ -205,8 +205,12 @@ The repository-wide `mise run pre-commit` task is supported on Windows. `tasks/rust.toml` and `tasks/test.toml` route compiler-bearing checks through the wrapper for the native host target, while `tasks/markdown.toml` provides a Windows-safe dependency setup command. Keep existing Unix `run` bodies -unchanged when adding `run_windows` behavior. Linux installer and packaging -asset tests skip explicitly; cross-platform checks continue to run. +unchanged when adding `run_windows` behavior. Linux installer, +build-environment shell-helper, and packaging asset tests skip explicitly; +cross-platform checks continue to run. The wrapper serializes its Cargo +commands and limits Cargo/MSVC compilation to four jobs by default so +concurrent pre-commit tasks do not exhaust Windows process resources. Set +`OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override the limit. ### Step 6: mise check on x86_64-pc-windows-msvc @@ -228,8 +232,9 @@ if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { ARM64 typically surfaces the same dependency issues as x64. If x64 passes but ARM64 fails, the fault is almost always a native dependency (a `*-sys` crate without ARM64 prebuilds) or an inline-asm block lacking aarch64 paths. Either find a pure-Rust replacement or add ARM64 to the existing cfg gate. On an x64 host this is a cross-build. The wrapper validates the ARM64 compiler -and Spectre libraries, adds host-native LLVM and Ninja to `PATH`, lets ARM64 -crypto crates select `clang-cl`, and keeps bundled Z3 on native MSVC `cl.exe`. +and Spectre libraries, adds host-native LLVM to `PATH`, lets ARM64 crypto crates +select `clang-cl`, and keeps bundled Z3 on native MSVC `cl.exe` through the +Visual Studio generator. Use a short absolute `CARGO_TARGET_DIR` if Windows path-length limits are hit. ### Step 8: mise build --release (both targets) diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index 2857448208..5d2089f597 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -167,10 +167,14 @@ The Windows build path is additive. Keep the repository's Linux `mise run ci`, d On Windows, `mise run pre-commit` routes `rust:check`, `rust:lint`, and `test:rust` through the same wrapper for the host-native target. Shared task -definitions retain their Unix commands. Only Linux installer and -service/RPM-packaging tests skip. The Windows Clippy command allows unused -imports, dead code, and unused async functions caused by cfg-gated stubs; other -warnings remain errors. +definitions retain their Unix commands. Only Linux installer, +build-environment shell-helper, and service/RPM-packaging tests skip. The +Windows Clippy command allows unused imports, dead code, and unused async +functions caused by cfg-gated stubs; other warnings remain errors. +Wrapper-owned Cargo commands are serialized across processes and default to +four Cargo/MSVC jobs to avoid multiplying bundled-Z3 compilation across +parallel pre-commit tasks. Override the limit with a positive integer in +`OPENSHELL_WINDOWS_BUILD_JOBS`. | Task | Expected behavior | |---|---| @@ -188,11 +192,12 @@ Use `--skip-tools` for Windows CI and automation. Rust must come from rustup wit ARM64 validation requires the Visual Studio ARM64 C++ tools, ARM64 Spectre-mitigated libraries, host-native `libclang.dll` and `clang-cl.exe`, -CMake tools, Ninja, and an ARM64-capable Windows SDK. During x64-to-ARM64 -check/build, ARM64 crypto crates use `clang-cl` while bundled Z3 stays on -native MSVC `cl.exe` with Ninja. Use a short `CARGO_TARGET_DIR` if Windows -path-length limits are reached. Test tasks reject a target that does not match -the Windows host architecture. +CMake tools, and an ARM64-capable Windows SDK. During x64-to-ARM64 check/build, +ARM64 crypto crates use `clang-cl` while bundled Z3 stays on native MSVC +`cl.exe` through CMake's Visual Studio generator. The generator must accept the +MSBuild `-m` argument emitted by `z3-sys`. Use a short `CARGO_TARGET_DIR` if +Windows path-length limits are reached. Test tasks reject a target that does +not match the Windows host architecture. Bundled Z3 source is pinned by revision, fetched through Git, and cached under `CARGO_TARGET_DIR`. The wrapper sets `Z3_SYS_BUNDLED_DIR_OVERRIDE` so `z3-sys` diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index c27a556e54..863198088f 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -67,7 +67,8 @@ Unix Cargo commands on Linux and macOS, excludes unsupported Windows runtime packages, and runs the server test-support suite separately. Windows Clippy continues to deny all warnings except unused imports, dead code, and unused async functions caused by cfg-gated Windows stubs. Repository-wide pre-commit -skips only Linux-specific installer and packaging-asset tests; its +skips only Linux-specific installer, build-environment shell-helper, and +packaging-asset tests; its cross-platform Python, Markdown, license, and documentation checks still run. Test tasks require the Rust target architecture to match the Windows host, so an ARM64 test result is native coverage rather than x64 emulation coverage. @@ -90,11 +91,18 @@ ARM64 validation requires the Visual Studio ARM64 MSVC tools, ARM64 Spectre-mitigated libraries, host-native Clang tools, CMake tools, and an ARM64-capable Windows SDK. Clang provides `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies. During x64-to-ARM64 check/build, -the wrapper uses Ninja with native MSVC `cl.exe` for bundled Z3 so the Z3 build -does not inherit the crypto crates' compiler requirement. Artifact hashing uses +the wrapper lets `cmake-rs` select the Visual Studio ARM64 generator with native +MSVC `cl.exe` for bundled Z3 so the Z3 build does not inherit the crypto crates' +compiler requirement. The Visual Studio generator is also compatible with the +MSBuild `-m` argument emitted by `z3-sys`; Ninja is not. Artifact hashing uses .NET SHA256 directly because module autoloading in the mise-launched Windows PowerShell process is not guaranteed. +The wrapper defaults Cargo and MSVC compilation to four jobs. Set +`OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override that limit. +A host-local mutex serializes wrapper-owned Cargo commands so concurrent +pre-commit tasks do not multiply the process count while bundled Z3 compiles. + ## CI Shape The x64 GitHub Actions job runs on `windows-2025` and executes: diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index c8bbabbdf8..8cdb22e5fc 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1821,9 +1821,7 @@ async fn sandbox_exec_interactive_grpc( Ok(n) => { if stdin_tx .blocking_send(ExecSandboxInput { - payload: Some(exec_sandbox_input::Payload::Stdin( - buf[..n].to_vec(), - )), + payload: Some(exec_sandbox_input::Payload::Stdin(buf[..n].to_vec())), }) .is_err() { diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 4a363892a3..ba0a05c7be 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -36,10 +36,6 @@ fn main() -> Result<(), Box> { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); let proto_includes = proto_include_dirs(&proto_root)?; - let proto_include_refs = proto_includes - .iter() - .map(PathBuf::as_path) - .collect::>(); let mut proto_files = Vec::new(); collect_proto_files(&proto_root, &mut proto_files)?; @@ -55,7 +51,7 @@ fn main() -> Result<(), Box> { // Emit a binary FileDescriptorSet so the server can enumerate every // RPC at runtime (used by the per-handler auth exhaustiveness test). .file_descriptor_set_path(&descriptor_path) - .compile_protos(&proto_files, &proto_include_refs)?; + .compile_protos(&proto_files, &proto_includes)?; println!( "cargo:rustc-env=OPENSHELL_DESCRIPTOR_PATH={}", diff --git a/crates/openshell-core/src/paths.rs b/crates/openshell-core/src/paths.rs index c1c327122e..df9deca8b7 100644 --- a/crates/openshell-core/src/paths.rs +++ b/crates/openshell-core/src/paths.rs @@ -145,7 +145,8 @@ pub fn is_file_permissions_too_open(path: &Path) -> bool { /// /// This is a lexical normalization only — it does NOT resolve symlinks or /// check the filesystem. `..` components are preserved verbatim; callers that -/// need to reject parent traversal must validate separately. +/// need to reject parent traversal must validate separately. The normalized +/// representation always uses `/` so sandbox policy paths are host-independent. pub fn normalize_path(path: &str) -> String { use std::path::Component; @@ -164,7 +165,15 @@ pub fn normalize_path(path: &str) -> String { Component::Normal(c) => normalized.push(c), } } - normalized.to_string_lossy().to_string() + let normalized = normalized.to_string_lossy(); + #[cfg(windows)] + { + normalized.replace('\\', "/") + } + #[cfg(not(windows))] + { + normalized.into_owned() + } } #[cfg(test)] diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index e0b0121633..5895f9debe 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -22,6 +22,7 @@ use crate::sandbox_watch::SandboxWatchBus; use crate::supervisor_session::SupervisorSessionRegistry; use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; +#[cfg(unix)] use hyper_util::rt::TokioIo; use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ @@ -40,19 +41,12 @@ use openshell_core::proto::{ use openshell_core::{ObjectLabels, ObjectWorkspace}; #[cfg(not(target_os = "windows"))] use openshell_driver_docker::DockerComputeDriver; -#[cfg(target_os = "windows")] -use openshell_driver_kubernetes::KubernetesComputeConfig; #[cfg(not(target_os = "windows"))] use openshell_driver_kubernetes::{ - ComputeDriverService as KubernetesDriverService, KubernetesComputeConfig, - KubernetesComputeDriver, + ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, }; -#[cfg(target_os = "windows")] -use openshell_driver_podman::PodmanComputeConfig; #[cfg(not(target_os = "windows"))] -use openshell_driver_podman::{ - ComputeDriverService as PodmanDriverService, PodmanComputeConfig, PodmanComputeDriver, -}; +use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; use std::collections::HashMap; use std::fmt; @@ -64,8 +58,11 @@ use std::time::Duration; #[cfg(unix)] use tokio::net::UnixStream; use tokio::sync::{Mutex, watch}; -use tonic::transport::{Channel, Endpoint}; +use tonic::transport::Channel; +#[cfg(unix)] +use tonic::transport::Endpoint; use tonic::{Code, Request, Status}; +#[cfg(unix)] use tower::service_fn; use tracing::{debug, info, warn}; diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index 1d3c8597d0..3afb8003fd 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -31,15 +31,16 @@ use super::AcquiredRemoteDriverEndpoint; #[cfg(unix)] -#[cfg(unix)] use super::ManagedDriverProcess; #[cfg(unix)] use hyper_util::rt::TokioIo; #[cfg(unix)] +use openshell_core::ComputeDriverKind; +#[cfg(unix)] use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{ComputeDriverKind, Config, Error, Result}; +use openshell_core::{Config, Error, Result}; #[cfg(unix)] use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; #[cfg(unix)] @@ -51,6 +52,7 @@ use std::{io::ErrorKind, process::Stdio, sync::Arc, time::Duration}; use tokio::net::UnixStream; #[cfg(unix)] use tokio::process::Command; +#[cfg(unix)] use tonic::transport::Channel; #[cfg(unix)] use tonic::transport::Endpoint; @@ -533,7 +535,7 @@ fn unsupported_vm_message() -> &'static str { pub async fn spawn( _config: &Config, _vm_config: &VmComputeConfig, -) -> Result<(Channel, std::sync::Arc)> { +) -> Result { Err(Error::config(unsupported_vm_message())) } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 5cb816ee62..f8f6895ca6 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -941,6 +941,7 @@ struct AutoApproveChunkContext<'a> { source: &'a str, resolved_from: &'a str, current_policy: &'a ProtoSandboxPolicy, + #[cfg(not(target_os = "windows"))] credential_set: &'a CredentialSet, } @@ -979,12 +980,19 @@ async fn auto_approve_chunk( // stored rule instead of trusting the incoming proposal's verdict. let rule = decode_draft_chunk_rule(&chunk)? .ok_or_else(|| Status::failed_precondition("draft chunk has no proposed rule"))?; + #[cfg(not(target_os = "windows"))] let validation_result = validation_result_for_agent_proposal( context.current_policy.clone(), &chunk.rule_name, &rule, context.credential_set, ); + #[cfg(target_os = "windows")] + let validation_result = validation_result_for_agent_proposal( + context.current_policy.clone(), + &chunk.rule_name, + &rule, + ); if validation_result != "prover: no new findings" { info!( sandbox_id = %sandbox_id, @@ -2878,6 +2886,7 @@ pub(super) async fn handle_submit_policy_analysis( source: &req.analysis_mode, resolved_from, current_policy: ¤t_policy, + #[cfg(not(target_os = "windows"))] credential_set: &credential_set, }, ) diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 1331d5d123..1ef311a981 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -37,6 +37,20 @@ if ([string]::IsNullOrWhiteSpace($TargetDir)) { $TargetDir = Join-Path $RepoRoot "target" } +$BuildJobsValue = $env:OPENSHELL_WINDOWS_BUILD_JOBS +if ([string]::IsNullOrWhiteSpace($BuildJobsValue)) { + $BuildJobsValue = $env:CARGO_BUILD_JOBS +} +if ([string]::IsNullOrWhiteSpace($BuildJobsValue)) { + $BuildJobsValue = "4" +} +[int] $WindowsBuildJobs = 0 +if (-not [int]::TryParse($BuildJobsValue, [ref] $WindowsBuildJobs) -or $WindowsBuildJobs -lt 1) { + throw "OPENSHELL_WINDOWS_BUILD_JOBS or CARGO_BUILD_JOBS must be a positive integer." +} +$ClOptions = "$($env:_CL_) /MP$WindowsBuildJobs".Trim() +$WindowsCargoMutex = [System.Threading.Mutex]::new($false, "Local\OpenShellWindowsMsvcCargo") + $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm --exclude openshell-supervisor-process" $WindowsClippyPackageExcludes = $UnsupportedDriverPackageExcludes $WindowsClippyLintArgs = "-D warnings -A dead-code -A unused-imports -A clippy::unused-async" @@ -231,35 +245,6 @@ function Resolve-LibclangPath { throw "Could not find libclang.dll. Install Visual Studio C++ Clang tools, or set LIBCLANG_PATH to the directory containing libclang.dll." } -function Resolve-NinjaPath { - $fromPath = Get-Command ninja.exe -ErrorAction SilentlyContinue - if ($fromPath) { - return $fromPath.Source - } - - $programFilesX86 = [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") - if ($programFilesX86) { - $vswhere = Join-Path $programFilesX86 "Microsoft Visual Studio\Installer\vswhere.exe" - } else { - $vswhere = $null - } - if ($vswhere -and (Test-Path $vswhere)) { - $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.CMake.Project -find "Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" | Select-Object -First 1 - if ($found -and (Test-Path $found -PathType Leaf)) { - return (Resolve-Path $found).Path - } - } - - foreach ($installRoot in Get-VsInstallRoots) { - $candidate = Join-Path $installRoot "Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" - if (Test-Path $candidate -PathType Leaf) { - return (Resolve-Path $candidate).Path - } - } - - throw "Could not find ninja.exe. Install Microsoft.VisualStudio.Component.VC.CMake.Project." -} - function Add-PathEntry([string] $Directory) { if (($env:PATH -split ";") -notcontains $Directory) { $env:PATH = "$Directory;$env:PATH" @@ -277,14 +262,9 @@ function Configure-Arm64CrossBuild([string[]] $RustTargets) { } Add-PathEntry $env:LIBCLANG_PATH - $ninja = Resolve-NinjaPath - Add-PathEntry (Split-Path -Parent $ninja) - [Environment]::SetEnvironmentVariable("CMAKE_GENERATOR_aarch64_pc_windows_msvc", "Ninja", "Process") - Write-Host "==> ARM64 cross-build toolchain" Write-Host " clang-cl: $clangCl" - Write-Host " ninja: $ninja" - Write-Host " Z3: MSVC cl.exe with Ninja" + Write-Host " Z3: MSVC cl.exe with the Visual Studio generator" } function Get-HostArch { @@ -474,7 +454,9 @@ function Invoke-VsCargo { $logPath = Join-Path $LogDir $LogName $environmentSetup = @( "set `"CARGO_TARGET_DIR=$TargetDir`"", + "set `"CARGO_BUILD_JOBS=$WindowsBuildJobs`"", "set `"CARGO_INCREMENTAL=0`"", + "set `"_CL_=$ClOptions`"", "set `"RUSTC_WRAPPER=`"" ) if ($hostArch -eq "amd64" -and $RustTarget -eq "aarch64-pc-windows-msvc") { @@ -495,14 +477,34 @@ function Invoke-VsCargo { Write-Host " target: $RustTarget" Write-Host " log: $logPath" - $cmdWithLog = "$cmd > `"$logPath`" 2>&1" - & cmd /v:on /d /c $cmdWithLog - $exitCode = $LASTEXITCODE - if (Test-Path $logPath) { - Get-Content $logPath - } - if ($exitCode -ne 0) { - throw "Command failed with exit code $exitCode. See $logPath" + $lockAcquired = $false + try { + try { + $lockAcquired = $WindowsCargoMutex.WaitOne(0) + if (-not $lockAcquired) { + Write-Host " waiting for another Windows Cargo task" + $lockAcquired = $WindowsCargoMutex.WaitOne([TimeSpan]::FromHours(2)) + } + } catch [System.Threading.AbandonedMutexException] { + $lockAcquired = $true + } + if (-not $lockAcquired) { + throw "Timed out waiting for another Windows Cargo task to finish." + } + + $cmdWithLog = "$cmd > `"$logPath`" 2>&1" + & cmd /v:on /d /c $cmdWithLog + $exitCode = $LASTEXITCODE + if (Test-Path $logPath) { + Get-Content $logPath + } + if ($exitCode -ne 0) { + throw "Command failed with exit code $exitCode. See $logPath" + } + } finally { + if ($lockAcquired) { + $WindowsCargoMutex.ReleaseMutex() + } } } diff --git a/tasks/test.toml b/tasks/test.toml index 55fa2b5d45..8c0d0c99fd 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -27,6 +27,7 @@ hide = true ["test:build-env"] description = "Run build-env.sh helper shell tests" run = "tasks/scripts/test-build-env.sh" +run_windows = "echo Skipping test:build-env: the Unix build-env.sh helper does not apply on Windows." hide = true ["test:packaging-assets"] From 463e4eee3ed2e86bd8324b7cee5cfb158a910fea Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Thu, 23 Jul 2026 12:12:40 -0700 Subject: [PATCH 25/30] fix(windows): harden shared Z3 source cache Signed-off-by: Shailendra Singh --- .../build-openshell-mxc-windows/SKILL.md | 11 +++-- architecture/windows-msvc-build.md | 16 ++++--- tasks/scripts/windows-msvc.ps1 | 42 ++++++++++++++++--- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 0bd2ab4474..264d85d921 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -60,7 +60,7 @@ The skill reads these environment variables (PowerShell syntax): | `$env:OPENSHELL_WXC_EXEC_PATH` | unset | Optional path to `wxc-exec.exe`. Validated for existence and architecture match only. Not used in this build-only skill. | | `$env:OPENSHELL_MXC_SKIP_ARM64` | `0` | Set to `1` to skip aarch64 build (e.g., for fast x64-only iterations). | | `$env:OPENSHELL_MXC_FORK_BRANCH` | `windows-mxc-build` | Branch name created in the fork. | -| `$env:Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned source cached under `CARGO_TARGET_DIR` | Reuse an existing Z3 source tree containing `src/api/z3.h`; otherwise the wrapper fetches and caches the pinned revision automatically. | +| `$env:Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned source cached under `CARGO_TARGET_DIR` when explicit, otherwise `%LOCALAPPDATA%\OpenShell\cache\z3` | Reuse an existing Z3 source tree containing `src/api/z3.h`; otherwise the wrapper fetches and caches the pinned revision automatically. | ## Workflow @@ -197,9 +197,12 @@ The PowerShell wrapper must: Use `mise run --skip-tools windows:*` in GitHub Actions and local Windows automation. `--skip-tools` is intentional: this repo should not ask mise to install Rust on Windows because the MSVC flow relies on rustup plus Visual Studio Build Tools. For bundled Z3, the wrapper fetches the revision pinned by `z3-sys` through -Git, caches it under `CARGO_TARGET_DIR`, and sets -`Z3_SYS_BUNDLED_DIR_OVERRIDE`. This bypasses the unauthenticated GitHub -Contents API lookup that can fail with HTTP 403 on shared networks. +Git and sets `Z3_SYS_BUNDLED_DIR_OVERRIDE`. It caches under an explicitly +configured `CARGO_TARGET_DIR`, or under the current user's local application +data directory when Cargo uses its default target tree. Concurrent commands +publish the validated source through an atomic directory rename so x64 and +ARM64 validation can share the cache safely. This bypasses the unauthenticated +GitHub Contents API lookup that can fail with HTTP 403 on shared networks. The repository-wide `mise run pre-commit` task is supported on Windows. `tasks/rust.toml` and `tasks/test.toml` route compiler-bearing checks through diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 863198088f..19719b3ccd 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -76,12 +76,16 @@ By default it enables bundled Z3 for reproducible Windows builds. When `Z3_LIBRARY_PATH_OVERRIDE` points at a directory containing `libz3.lib`, the wrapper uses that system Z3 instead and requires `Z3_SYS_Z3_HEADER` to point at the full path to `z3.h`. For bundled builds, the wrapper fetches the Z3 source -revision pinned by `z3-sys` through Git, caches it under `CARGO_TARGET_DIR`, and -sets `Z3_SYS_BUNDLED_DIR_OVERRIDE`. This avoids the unauthenticated GitHub API -lookup in the `z3-sys` build script, which can fail with HTTP 403 when a shared -runner or developer network exhausts its API rate limit. An explicitly set -`Z3_SYS_BUNDLED_DIR_OVERRIDE` remains supported and must contain -`src/api/z3.h`. +revision pinned by `z3-sys` through Git and sets +`Z3_SYS_BUNDLED_DIR_OVERRIDE`. When `CARGO_TARGET_DIR` is explicit, the wrapper +uses it for the source cache. Otherwise, it caches under the current user's +local application data directory, outside the checkout. Publishing uses an +atomic directory rename so concurrent x64 and ARM64 commands can share the +cache safely. This keeps downloaded sources outside the checkout by default and +avoids the unauthenticated GitHub API lookup in the `z3-sys` build script, which +can fail with HTTP 403 when a shared runner or developer network exhausts its +API rate limit. An explicitly set `Z3_SYS_BUNDLED_DIR_OVERRIDE` remains +supported and must contain `src/api/z3.h`. The lane uses `mise run --skip-tools windows:*` because Windows Rust comes from rustup and linking comes from Visual Studio Build Tools. Mise orchestrates the diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index 1ef311a981..fc265b684a 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -32,11 +32,24 @@ if (-not (Test-Path $LogDir)) { } $LogDir = (Resolve-Path $LogDir).Path +$TargetDirWasConfigured = -not [string]::IsNullOrWhiteSpace($env:CARGO_TARGET_DIR) $TargetDir = $env:CARGO_TARGET_DIR -if ([string]::IsNullOrWhiteSpace($TargetDir)) { +if (-not $TargetDirWasConfigured) { $TargetDir = Join-Path $RepoRoot "target" } +$BundledZ3CacheRoot = $TargetDir +if (-not $TargetDirWasConfigured) { + $userCacheRoot = [Environment]::GetFolderPath([Environment+SpecialFolder]::LocalApplicationData) + if ([string]::IsNullOrWhiteSpace($userCacheRoot)) { + $userCacheRoot = $env:LOCALAPPDATA + } + if ([string]::IsNullOrWhiteSpace($userCacheRoot)) { + $userCacheRoot = [IO.Path]::GetTempPath() + } + $BundledZ3CacheRoot = Join-Path $userCacheRoot "OpenShell\cache\z3" +} + $BuildJobsValue = $env:OPENSHELL_WINDOWS_BUILD_JOBS if ([string]::IsNullOrWhiteSpace($BuildJobsValue)) { $BuildJobsValue = $env:CARGO_BUILD_JOBS @@ -358,7 +371,7 @@ function Resolve-BundledZ3Source { } $revisionPrefix = $BundledZ3Revision.Substring(0, 12) - $sourcePath = Join-Path $TargetDir "z3-source-$revisionPrefix" + $sourcePath = Join-Path $BundledZ3CacheRoot "z3-source-$revisionPrefix" if (Test-Path $sourcePath) { return Assert-BundledZ3Source $sourcePath $BundledZ3Revision } @@ -366,8 +379,8 @@ function Resolve-BundledZ3Source { if (-not (Get-Command git.exe -ErrorAction SilentlyContinue)) { throw "Bundled Z3 source preparation requires git.exe on PATH." } - if (-not (Test-Path $TargetDir -PathType Container)) { - New-Item -ItemType Directory -Force -Path $TargetDir | Out-Null + if (-not (Test-Path $BundledZ3CacheRoot -PathType Container)) { + New-Item -ItemType Directory -Force -Path $BundledZ3CacheRoot | Out-Null } $stagingPath = "$sourcePath.partial-$([guid]::NewGuid().ToString('N'))" @@ -394,7 +407,26 @@ function Resolve-BundledZ3Source { } Assert-BundledZ3Source $stagingPath $BundledZ3Revision | Out-Null - Move-Item -Path $stagingPath -Destination $sourcePath + try { + # Directory.Move is an atomic rename on the same volume and, unlike + # Move-Item, fails when the destination already exists. A concurrent + # x64/ARM64 invocation can therefore win publication without the loser + # nesting its staging directory inside the shared cache. + [IO.Directory]::Move($stagingPath, $sourcePath) + } catch { + if (-not (Test-Path $sourcePath -PathType Container)) { + throw + } + Write-Host "==> Reusing bundled Z3 source published by another process" + } finally { + if (Test-Path $stagingPath -PathType Container) { + try { + Remove-Item -LiteralPath $stagingPath -Recurse -Force + } catch { + Write-Warning "Could not remove redundant bundled Z3 staging directory: $stagingPath" + } + } + } return Assert-BundledZ3Source $sourcePath $BundledZ3Revision } From 20f357215aaadf926b4ffc76fb13cdb99492141b Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Fri, 24 Jul 2026 21:58:24 -0500 Subject: [PATCH 26/30] fix(windows): avoid leaking MSVC flags into clang-cl Signed-off-by: Akber Raza --- .agents/skills/build-openshell-mxc-windows/SKILL.md | 5 ++++- .agents/skills/build-openshell-mxc-windows/reference.md | 5 ++++- architecture/windows-msvc-build.md | 5 ++++- tasks/scripts/windows-msvc.ps1 | 2 -- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 264d85d921..5094c2fc98 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -211,9 +211,12 @@ Windows-safe dependency setup command. Keep existing Unix `run` bodies unchanged when adding `run_windows` behavior. Linux installer, build-environment shell-helper, and packaging asset tests skip explicitly; cross-platform checks continue to run. The wrapper serializes its Cargo -commands and limits Cargo/MSVC compilation to four jobs by default so +commands and limits Cargo compilation to four jobs by default so concurrent pre-commit tasks do not exhaust Windows process resources. Set `OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override the limit. +It deliberately leaves `CL` and `_CL_` unset because `clang-cl` consumes those +variables too. Injecting MSVC-only options such as `/MP` can make ARM64 crypto +dependency builds treat the option as an input file. ### Step 6: mise check on x86_64-pc-windows-msvc diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index 5d2089f597..b0c221a0e6 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -172,9 +172,12 @@ build-environment shell-helper, and service/RPM-packaging tests skip. The Windows Clippy command allows unused imports, dead code, and unused async functions caused by cfg-gated stubs; other warnings remain errors. Wrapper-owned Cargo commands are serialized across processes and default to -four Cargo/MSVC jobs to avoid multiplying bundled-Z3 compilation across +four Cargo jobs to avoid multiplying bundled-Z3 compilation across parallel pre-commit tasks. Override the limit with a positive integer in `OPENSHELL_WINDOWS_BUILD_JOBS`. +The wrapper leaves `CL` and `_CL_` unset because `clang-cl` consumes those +variables too. Injecting MSVC-only options such as `/MP` can make ARM64 crypto +dependency builds treat the option as an input file. | Task | Expected behavior | |---|---| diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 19719b3ccd..7f5de5a848 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -102,10 +102,13 @@ MSBuild `-m` argument emitted by `z3-sys`; Ninja is not. Artifact hashing uses .NET SHA256 directly because module autoloading in the mise-launched Windows PowerShell process is not guaranteed. -The wrapper defaults Cargo and MSVC compilation to four jobs. Set +The wrapper defaults Cargo compilation to four jobs. Set `OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override that limit. A host-local mutex serializes wrapper-owned Cargo commands so concurrent pre-commit tasks do not multiply the process count while bundled Z3 compiles. +The wrapper deliberately leaves `CL` and `_CL_` unset because `clang-cl` +consumes those variables too. Injecting MSVC-only options such as `/MP` can +make ARM64 crypto dependency builds treat the option as an input file. ## CI Shape diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index fc265b684a..ba7d5a7558 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -61,7 +61,6 @@ if ([string]::IsNullOrWhiteSpace($BuildJobsValue)) { if (-not [int]::TryParse($BuildJobsValue, [ref] $WindowsBuildJobs) -or $WindowsBuildJobs -lt 1) { throw "OPENSHELL_WINDOWS_BUILD_JOBS or CARGO_BUILD_JOBS must be a positive integer." } -$ClOptions = "$($env:_CL_) /MP$WindowsBuildJobs".Trim() $WindowsCargoMutex = [System.Threading.Mutex]::new($false, "Local\OpenShellWindowsMsvcCargo") $UnsupportedDriverPackageExcludes = "--exclude openshell-driver-docker --exclude openshell-driver-kubernetes --exclude openshell-driver-podman --exclude openshell-driver-vm --exclude openshell-supervisor-process" @@ -488,7 +487,6 @@ function Invoke-VsCargo { "set `"CARGO_TARGET_DIR=$TargetDir`"", "set `"CARGO_BUILD_JOBS=$WindowsBuildJobs`"", "set `"CARGO_INCREMENTAL=0`"", - "set `"_CL_=$ClOptions`"", "set `"RUSTC_WRAPPER=`"" ) if ($hostArch -eq "amd64" -and $RustTarget -eq "aarch64-pc-windows-msvc") { From 58567665e9d64a08481c6e252c853dcc9298a8b7 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Fri, 24 Jul 2026 17:37:09 -0500 Subject: [PATCH 27/30] fix(windows): complete ARM64 migration audit Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 618 ++++++++---------- .../build-openshell-mxc-windows/reference.md | 359 +++++----- CONTRIBUTING.md | 1 + architecture/windows-msvc-build.md | 6 +- 4 files changed, 467 insertions(+), 517 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 5094c2fc98..a36f75745b 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -1,388 +1,332 @@ --- name: build-openshell-mxc-windows -description: Auto-executable build skill that forks OpenShell into a sibling directory and produces Windows-native x64 (x86_64-pc-windows-msvc) and ARM64 (aarch64-pc-windows-msvc) binaries on a Windows 11 26100+ host. Scope is build-only — cross-compilation, minimal Windows compatibility shims, and ARM64 CI scaffolding. Does not implement an MXC compute driver, policy translation, an MSI installer, or a supervisor port. Trigger keywords - build openshell mxc windows, openshell windows build, x64 build, arm64 build, windows msvc, openshell-mxc fork, native windows openshell, MXC build. +description: Maintain and validate OpenShell's build-only Windows MSVC lane for x64 and ARM64. Use when working on Windows compilation, `windows:*` mise tasks, unsupported Windows compute-driver contracts, or Windows build reports. This skill does not implement Docker, Kubernetes, Podman, VM, MXC driver, policy translation, MSI, service, or supervisor runtime support on Windows. --- -# Build OpenShell-MXC for Windows (x64 + ARM64) +# Build OpenShell-MXC for Windows -Auto-executable skill. Forks the OpenShell repository into a sibling directory, applies the minimum Windows compatibility shims required to compile, and produces native binaries for `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`. +This skill maintains the existing native Windows MSVC build lane in the +OpenShell repository. The Windows lane is already present in `main`; do not +treat this skill as a first-time porting recipe unless the user explicitly asks +for a new fork or a from-scratch bring-up. -This skill covers only the build slice. It does **not** implement the MXC compute driver, policy translation, AppContainer wiring, supervisor port, MSI installer, or Windows Service registration. Those are handled by sibling skills (to be created). +The lane is build-only. It validates that OpenShell can compile and test on +Windows MSVC for the supported deliverables: -## Scope - -In scope: - -- `x86_64-pc-windows-msvc` compilation -- `aarch64-pc-windows-msvc` compilation -- Audit and cfg-gate Unix-only dependencies in `openshell-server` and shared crates so the workspace compiles to Windows MSVC -- Keep Docker, Kubernetes, Podman, and VM compute crates in the Windows build graph as stubs that return "unsupported" at runtime -- Add a Windows-specific mise task lane (`windows:*`) that wraps MSVC builds without changing the Linux `mise run ci` path -- `cargo test` compiles and passes on a native x64 or ARM64 Windows host for non-Linux-gated tests -- Scaffold an ARM64 CI workflow (job definition only; runner provisioning is operator work) - -Out of scope (defer to follow-on skills): - -- New MXC compute driver crate -- OpenShell → MXC policy translation -- Windows network and credential integration -- Sandbox supervisor Windows port -- Docker, Kubernetes, Podman, or VM runtime support on Windows -- MSI installer, Windows Service registration, WinGet manifest - -The Linux build must remain green at every commit. All Windows-specific code must be behind `#[cfg(target_os = "windows")]` and Linux-only code behind `#[cfg(target_os = "linux")]` or `#[cfg(unix)]`. - -## Prerequisites - -The skill targets a **Windows 11 host with CurrentBuild ≥ 26100**. Because the scope is compilation only, the skill warns rather than aborts when any of the items below is missing — a `cargo check` attempt can still surface useful information on a less-than-ideal host. Recommended: - -| Requirement | Check | Install hint | -|---|---|---| -| Windows 11 build ≥ 26100 | `[System.Environment]::OSVersion.Version` | OS update | -| Visual Studio 2022 or newer with **MSVC v143** + **Windows 11 SDK** | `where.exe cl.exe` from a VS Developer PowerShell | Include the x64/x86 and ARM64 C++ tools. The wrapper discovers installed release directories such as `18` and `2022`. | -| Visual C++ ARM64 Spectre-mitigated libraries | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre -property installationPath` | Required by ARM64 dependencies that select the Spectre runtime. | -| Visual C++ Clang and CMake tools | `where.exe clang-cl.exe`; `where.exe cmake.exe` | `bindgen` needs host-native `libclang.dll`; ARM64 crypto crates use `clang-cl`; bundled Z3 uses CMake's Visual Studio generator with native MSVC for x64-to-ARM64 builds. | -| Rust ≥ 1.88 via rustup with MSVC targets | `rustc --version` | `winget install Rustlang.Rustup` | -| mise CLI for task orchestration only | `mise --version` | https://mise.jdx.dev/installing-mise.html | -| Git ≥ 2.40 | `git --version` | `winget install Git.Git` | -| Windows PowerShell 5.1+ or PowerShell 7+ | `$PSVersionTable.PSVersion` | Built into Windows; PowerShell 7 optional | - -The Windows mise lane shells into Visual Studio's developer environment before invoking Cargo, so commands can run from an ordinary PowerShell session if Visual Studio Build Tools are discoverable. Use rustup for the Rust toolchain and MSVC targets; mise is only the task runner for the Windows lane. In CI, prefer `mise run --skip-tools windows:*` so Linux tool installation remains untouched. - -## Inputs - -The skill reads these environment variables (PowerShell syntax): - -| Variable | Default | Purpose | -|---|---|---| -| `$env:OPENSHELL_UPSTREAM` | `https://github.com/NVIDIA/OpenShell.git` | Upstream Git remote to fork from. Override to use a local path or GitLab mirror. | -| `$env:OPENSHELL_MXC_FORK_DIR` | `C:\Users\$env:USERNAME\openshell-mxc` | Sibling directory for the fork. Must not exist. | -| `$env:OPENSHELL_WXC_EXEC_PATH` | unset | Optional path to `wxc-exec.exe`. Validated for existence and architecture match only. Not used in this build-only skill. | -| `$env:OPENSHELL_MXC_SKIP_ARM64` | `0` | Set to `1` to skip aarch64 build (e.g., for fast x64-only iterations). | -| `$env:OPENSHELL_MXC_FORK_BRANCH` | `windows-mxc-build` | Branch name created in the fork. | -| `$env:Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned source cached under `CARGO_TARGET_DIR` when explicit, otherwise `%LOCALAPPDATA%\OpenShell\cache\z3` | Reuse an existing Z3 source tree containing `src/api/z3.h`; otherwise the wrapper fetches and caches the pinned revision automatically. | - -## Workflow - -The skill runs the following checklist top-to-bottom. Each step is idempotent; re-running the skill is safe once the fork exists. - -``` -[ ] Step 1: Verify host preconditions -[ ] Step 2: Install Windows MSVC Rust targets -[ ] Step 3: Fork OpenShell into the sibling directory -[ ] Step 4: Apply minimum Windows compatibility shims (cfg gating) -[ ] Step 5: Add Windows mise task lane -[ ] Step 6: mise check on x86_64-pc-windows-msvc -[ ] Step 7: mise check on aarch64-pc-windows-msvc -[ ] Step 8: mise build --release (both targets) -[ ] Step 9: mise test on the host's native MSVC target -[ ] Step 10: Validate $env:OPENSHELL_WXC_EXEC_PATH (informational) -[ ] Step 11: Scaffold ARM64 CI workflow -[ ] Step 12: Commit and report artifacts -``` - -### Step 1: Verify host preconditions +- `openshell-gateway.exe` +- `openshell.exe` -This step is **advisory**. It records what is and isn't available on the host, but the skill continues regardless — compilation may still succeed (or produce useful errors) on a less-than-ideal host. +It intentionally does not make Windows a Docker, Kubernetes, Podman, or VM +runtime host. -```powershell -$warnings = @() +## Current Repository Shape -$os = [System.Environment]::OSVersion.Version -if ($os.Build -lt 26100) { - $warnings += "Windows build $($os.Build) < 26100 (recommended for MXC runtime; not required for compilation)." -} +The Windows build lane is implemented by these tracked files: -# MSVC toolchain (must be a VS 2022 Developer PowerShell to be on PATH) -foreach ($tool in 'cl.exe', 'link.exe') { - if (-not (Get-Command $tool -ErrorAction SilentlyContinue)) { - $warnings += "$tool not found on PATH — open a VS 2022 Developer PowerShell, or expect link failures in Step 5+." - } -} - -# Rust + git -foreach ($tool in 'rustup', 'cargo', 'rustc', 'git') { - if (-not (Get-Command $tool -ErrorAction SilentlyContinue)) { - $warnings += "$tool not found on PATH — Step $(if ($tool -eq 'git') { '3' } else { '2+' }) will fail." - } -} +| Path | Purpose | +|---|---| +| `tasks/windows.toml` | Mise task entry points for `windows:*` commands. | +| `tasks/rust.toml`, `tasks/test.toml`, and `tasks/markdown.toml` | Windows routing for compiler-bearing checks, explicit Unix-only test skips, and Markdown dependency setup. | +| `tasks/scripts/windows-msvc.ps1` | PowerShell wrapper that enters the Visual Studio developer environment and invokes Cargo. | +| `.github/workflows/windows-msvc.yml` | GitHub Actions scaffold for x64 and future ARM64 Windows validation. | +| `architecture/windows-msvc-build.md` | Design notes and validation contract. | +| `.agents/skills/build-openshell-mxc-windows/` | This skill and companion reference material. | -if (Get-Command rustc -ErrorAction SilentlyContinue) { rustc --version } +Use the code that is already in the repo. Do not generate a parallel Windows +build system, duplicate the wrapper, or add repository automation that the user +did not request. -if ($warnings.Count -gt 0) { - Write-Warning "Host preconditions not fully met:" - $warnings | ForEach-Object { Write-Warning " - $_" } - Write-Warning "Continuing anyway — re-evaluate after the next failing step." -} else { - Write-Host "Host preconditions OK." -} -``` +## Scope -Record the warning list so it can be included in the final report (Step 12). Do not attempt to install Visual Studio Build Tools or rustup automatically. +In scope: -### Step 2: Install Windows MSVC Rust targets +- Refreshing a local checkout to the latest GitLab `main`. +- Maintaining `tasks/windows.toml` and `tasks/scripts/windows-msvc.ps1`. +- Running x64 and ARM64 MSVC checks. +- Building x64 and ARM64 release binaries for `openshell-gateway` and + `openshell`. +- Running workspace tests on a native x64 or ARM64 host. +- Running focused unsupported-driver contract tests. +- Reporting test counts, skipped/gated areas, warnings, artifacts, and logs. +- Keeping Linux and macOS build paths unchanged. +- Keeping unsupported Windows compute drivers explicit and testable. + +Out of scope: + +- Docker Desktop support on Windows. +- Kubernetes support on Windows. +- Podman, Podman machine, or Podman Desktop support on Windows. +- VM, Hyper-V, WSL, libkrun, or VM-backed sandbox execution on Windows. +- New MXC compute driver crate. +- OpenShell to MXC policy translation. +- Windows named-pipe driver IPC. +- Windows Credential Manager or DPAPI integration. +- MSI, WinGet, Windows service registration, or installer work. +- Windows supervisor runtime port. + +## Hard Rules + +- Do not enable Docker, Kubernetes, Podman, or VM runtimes on Windows. +- Do not build, package, ship, or smoke-test standalone Windows binaries for + unsupported compute drivers. +- Keep unsupported drivers in the Windows build graph only as library/config + stubs when needed by the gateway. +- Unsupported Windows runtime entry points must return a clear unsupported + error. +- Keep Windows-specific code behind `#[cfg(target_os = "windows")]`. +- Keep Unix/Linux-only code behind `#[cfg(unix)]` or + `#[cfg(target_os = "linux")]`. +- Do not modify the default Linux `mise run ci` path unless the user explicitly + asks for it. +- Use `mise run --skip-tools windows:*` for Windows validation. The Windows + toolchain is rustup plus Visual Studio Build Tools, not mise-provisioned Rust. +- Keep Unix `run` bodies unchanged when adding `run_windows` behavior to shared + pre-commit tasks. + +## Recommended Checkout Flow + +From the OpenShell checkout root, use: ```powershell -rustup target add x86_64-pc-windows-msvc -if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { - rustup target add aarch64-pc-windows-msvc -} +git fetch gitlab main +git switch main +git merge --ff-only gitlab/main +git branch --set-upstream-to=gitlab/main main +git status --short --branch ``` -### Step 3: Fork OpenShell into the sibling directory - -This is the only step that creates a new on-disk repo. Confirm `$env:OPENSHELL_MXC_FORK_DIR` does not exist before cloning. +If there are local changes, preserve or resolve them before refreshing. Do not +discard user work unless the user explicitly asks to clean the checkout. -```powershell -$fork = if ($env:OPENSHELL_MXC_FORK_DIR) { $env:OPENSHELL_MXC_FORK_DIR } else { "C:\Users\$env:USERNAME\openshell-mxc" } -$upstream = if ($env:OPENSHELL_UPSTREAM) { $env:OPENSHELL_UPSTREAM } else { "https://github.com/NVIDIA/OpenShell.git" } -$branch = if ($env:OPENSHELL_MXC_FORK_BRANCH) { $env:OPENSHELL_MXC_FORK_BRANCH } else { "windows-mxc-build" } +For automated GitHub-to-GitLab sync work, use a dedicated sync checkout outside +the user's active working repo. Accept the checkout path from the user or an +environment variable instead of hardcoding a local machine path, for example: -if (Test-Path $fork) { throw "Fork dir already exists: $fork. Remove or set OPENSHELL_MXC_FORK_DIR." } - -git clone $upstream $fork -Set-Location $fork -git checkout -b $branch - -# Copy this skill into the fork so it can self-iterate. -New-Item -ItemType Directory -Force -Path "$fork\.claude\skills\build-openshell-mxc-windows" | Out-Null -New-Item -ItemType Directory -Force -Path "$fork\.agents\skills\build-openshell-mxc-windows" | Out-Null -Copy-Item "$PSScriptRoot\*" "$fork\.claude\skills\build-openshell-mxc-windows\" -Recurse -Force -Copy-Item "$PSScriptRoot\*" "$fork\.agents\skills\build-openshell-mxc-windows\" -Recurse -Force +```text + ``` -From this point forward, `Set-Location $fork`. All edits land in the fork, not in the source OpenShell repo. - -### Step 4: Apply minimum Windows compatibility shims - -The goal is **the smallest patchset that makes `cargo check --target x86_64-pc-windows-msvc` succeed**, not a full Windows port. Strategy: - -1. **Cfg-gate Linux-only crates and modules.** Any module that imports `nix`, `landlock`, `libseccomp`, `caps`, or constructs Unix-domain sockets without conditional compilation must be wrapped in `#[cfg(target_os = "linux")]` or `#[cfg(unix)]`. -2. **Stub Windows-side gateway entry points.** Where Linux uses Unix domain sockets for driver IPC (gateway ↔ compute driver), provide a `#[cfg(target_os = "windows")]` stub that returns `Err(unimplemented!("Windows named-pipe transport — follow-on skill"))`. The build must compile; runtime behavior is not in scope. -3. **Disable Linux-only crates from the Windows build graph.** In each crate's `Cargo.toml`, gate platform-specific dependencies with `[target.'cfg(target_os = "linux")'.dependencies]`. Common offenders: `nix`, `landlock`, `libseccomp`, `caps`, `procfs`. -4. **Default storage paths.** Where Linux code uses `~/.local/share/openshell` or `/var/lib/openshell`, add a `#[cfg(target_os = "windows")]` branch returning `%APPDATA%\OpenShell`. +## Prerequisites -See [reference.md](reference.md) for the concrete dependency audit (which crates and modules need gating, and the exact patterns to apply). +The lane targets a Windows host with Visual Studio Build Tools and rustup. -Driver crates that are not supported on Windows (`openshell-driver-docker`, `openshell-driver-podman`, `openshell-driver-vm`, `openshell-driver-kubernetes`) must compile as minimal Windows library stubs. Preserve their configuration structs so existing config parsing keeps working, and make every Windows runtime entry point or constructor return an "unsupported on Windows" error. Do not build, package, ship, or smoke-test standalone driver binaries as Windows deliverables. Do not enable Docker, Kubernetes, Podman, VM, Docker Desktop, WSL, Hyper-V, Podman machine, Podman Desktop, or any VM-backed runtime in this build-only skill. +| Requirement | Check | Notes | +|---|---|---| +| Windows 11 | `[System.Environment]::OSVersion.Version` | Build 26100+ is recommended for MXC-adjacent validation, but compilation can still surface useful errors on older hosts. | +| Visual Studio 2022 or newer | `where.exe cl.exe` from a Developer PowerShell | Build Tools, Community, Professional, and Enterprise editions work when the target C++ components are installed. The wrapper discovers `VsDevCmd.bat` through `OPENSHELL_VSDEVCMD`, `vswhere`, or installed release directories such as `18` and `2022`. | +| Visual C++ ARM64 tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.ARM64 -property installationPath` | Required for native ARM64 check, build, and tests and for x64-to-ARM64 check/build. Tests always require a native runner. | +| Visual C++ ARM64 Spectre-mitigated libraries | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre -property installationPath` | Required by `regorus` through `msvc_spectre_libs`; the build fails when the selected MSVC toolset lacks `lib\spectre\arm64`. | +| Visual C++ Clang tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -property installationPath` | Provides host-native `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies such as `ring` and `aws-lc-sys`. On ARM64, the wrapper uses `VC\Tools\Llvm\Arm64\bin`. | +| Visual C++ CMake tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.CMake.Project -property installationPath` | Required for bundled Z3. The x64-to-ARM64 path uses CMake's Visual Studio ARM64 generator with native MSVC `cl.exe` so Z3 does not inherit the crypto crates' `clang-cl` requirement. | +| Windows SDK | `where.exe rc.exe` from a Developer PowerShell | Install an SDK containing target libraries and ARM64 tools. | +| Rust via rustup | `rustc --version` | Add each target being validated: `x86_64-pc-windows-msvc` and/or `aarch64-pc-windows-msvc`. The wrapper also adds the selected target. | +| mise | `mise --version` | Used as a task runner only. | +| Git | `git --version` | Needed for checkout and sync work. | +| PowerShell | `$PSVersionTable.PSVersion` | Windows PowerShell 5.1 works; PowerShell 7 is quieter with mise shell hooks. | + +Do not install Visual Studio, Rust, Docker, Kubernetes, Podman, WSL, or Hyper-V +from this skill. + +## Environment Variables -### Step 5: Add Windows mise task lane +| Variable | Default | Purpose | +|---|---|---| +| `OPENSHELL_VSDEVCMD` | unset | Optional explicit path to `VsDevCmd.bat`. | +| `OPENSHELL_MXC_SKIP_ARM64` | `0` | Set to `1` to skip ARM64 when using `all` tasks. | +| `OPENSHELL_WINDOWS_BUILD_JOBS` | `CARGO_BUILD_JOBS`, then `4` | Positive Cargo job limit used by the wrapper. | +| `CARGO_TARGET_DIR` | `target` under repo root | Override Cargo output location. Use a short absolute path when x64-to-ARM64 builds approach Windows path-length limits. | +| `Z3_LIBRARY_PATH_OVERRIDE` | unset | Directory containing an x64 system `libz3.lib`; not valid for ARM64. | +| `Z3_SYS_Z3_HEADER` | unset | Full `z3.h` path required with a system Z3 library. | +| `Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned cached source | Existing Z3 source tree containing `src/api/z3.h`. | +| `RUSTC_WRAPPER` | cleared by wrapper | The wrapper clears inherited values because `--skip-tools` does not provision `sccache`. | -Create a Windows-only task file at `tasks/windows.toml` and a PowerShell wrapper at `tasks/scripts/windows-msvc.ps1`. This lane is additive: do not change the existing Linux `build`, `test`, or `ci` tasks. The Linux build procedure remains the repo's default mise/Cargo path. +Legacy fork variables such as `OPENSHELL_UPSTREAM`, +`OPENSHELL_MXC_FORK_DIR`, and `OPENSHELL_MXC_FORK_BRANCH` are no longer part +of the normal maintenance workflow. Use them only if the user explicitly asks +for a new disposable fork. -The task file must expose these commands: +## Validation Workflow -| Task | Purpose | -|---|---| -| `windows:check:x64` | `cargo check --workspace --target x86_64-pc-windows-msvc` | -| `windows:check:arm64` | `cargo check --workspace --target aarch64-pc-windows-msvc` | -| `windows:build:x64` | Release-build `openshell-gateway` and `openshell` for x64 | -| `windows:build:arm64` | Release-build `openshell-gateway` and `openshell` for ARM64 | -| `windows:test:x64` | Native x64 `cargo test --workspace --no-fail-fast`, excluding unsupported driver packages as top-level workspace targets | -| `windows:test:arm64` | Native ARM64 `cargo test --workspace --no-fail-fast` with the same exclusions | -| `windows:test:unsupported:x64` | Focused server/runtime tests that assert unsupported Windows driver contracts without building standalone driver binaries | -| `windows:test:unsupported:arm64` | The same focused contracts on a native ARM64 host | -| `windows:ci` | Ordered Windows check/build/test/unsupported-contract/artifact lane | - -The PowerShell wrapper must: - -1. Discover `VsDevCmd.bat` through `$env:OPENSHELL_VSDEVCMD`, `vswhere`, or standard Visual Studio install paths. -2. Add the requested rustup target before invoking Cargo. -3. Set `CARGO_INCREMENTAL=0` and keep `CARGO_TARGET_DIR` inside the fork unless the caller overrides it. -4. Clear inherited `RUSTC_WRAPPER` for this lane, because `mise run --skip-tools` intentionally does not provision `sccache`. -5. Exclude Docker, Kubernetes, Podman, and VM driver packages as top-level Windows workspace targets for check/test while still allowing their library stubs to compile as dependencies. -6. Emit logs for x64/ARM64 checks, builds, tests, and unsupported-contract tests. -7. Fail fast if a Windows-only task is invoked on a non-Windows host. - -Use `mise run --skip-tools windows:*` in GitHub Actions and local Windows automation. `--skip-tools` is intentional: this repo should not ask mise to install Rust on Windows because the MSVC flow relies on rustup plus Visual Studio Build Tools. - -For bundled Z3, the wrapper fetches the revision pinned by `z3-sys` through -Git and sets `Z3_SYS_BUNDLED_DIR_OVERRIDE`. It caches under an explicitly -configured `CARGO_TARGET_DIR`, or under the current user's local application -data directory when Cargo uses its default target tree. Concurrent commands -publish the validated source through an atomic directory rename so x64 and -ARM64 validation can share the cache safely. This bypasses the unauthenticated -GitHub Contents API lookup that can fail with HTTP 403 on shared networks. - -The repository-wide `mise run pre-commit` task is supported on Windows. -`tasks/rust.toml` and `tasks/test.toml` route compiler-bearing checks through -the wrapper for the native host target, while `tasks/markdown.toml` provides a -Windows-safe dependency setup command. Keep existing Unix `run` bodies -unchanged when adding `run_windows` behavior. Linux installer, -build-environment shell-helper, and packaging asset tests skip explicitly; -cross-platform checks continue to run. The wrapper serializes its Cargo -commands and limits Cargo compilation to four jobs by default so -concurrent pre-commit tasks do not exhaust Windows process resources. Set -`OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override the limit. -It deliberately leaves `CL` and `_CL_` unset because `clang-cl` consumes those -variables too. Injecting MSVC-only options such as `/MP` can make ARM64 crypto -dependency builds treat the option as an input file. - -### Step 6: mise check on x86_64-pc-windows-msvc +Run the smallest useful slice first, then broaden: ```powershell -$env:CARGO_TARGET_DIR = "$fork\target" mise run --skip-tools windows:check:x64 -``` - -This wraps `cargo check --workspace --target x86_64-pc-windows-msvc`. If it fails, the error log is the audit list. Iterate Step 4 through Step 6 until x64 succeeds. Common error patterns and their fixes are in [reference.md](reference.md#common-errors). - -### Step 7: mise check on aarch64-pc-windows-msvc - -```powershell -if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { - mise run --skip-tools windows:check:arm64 -} -``` - -ARM64 typically surfaces the same dependency issues as x64. If x64 passes but ARM64 fails, the fault is almost always a native dependency (a `*-sys` crate without ARM64 prebuilds) or an inline-asm block lacking aarch64 paths. Either find a pure-Rust replacement or add ARM64 to the existing cfg gate. - -On an x64 host this is a cross-build. The wrapper validates the ARM64 compiler -and Spectre libraries, adds host-native LLVM to `PATH`, lets ARM64 crypto crates -select `clang-cl`, and keeps bundled Z3 on native MSVC `cl.exe` through the -Visual Studio generator. -Use a short absolute `CARGO_TARGET_DIR` if Windows path-length limits are hit. - -### Step 8: mise build --release (both targets) - -```powershell +mise run --skip-tools windows:check:arm64 mise run --skip-tools windows:build:x64 -if ($env:OPENSHELL_MXC_SKIP_ARM64 -ne "1") { - mise run --skip-tools windows:build:arm64 -} +mise run --skip-tools windows:build:arm64 +mise run --skip-tools windows:test:x64 +mise run --skip-tools windows:test:unsupported:x64 ``` -Build only the binaries needed for build validation: `openshell-gateway` and `openshell` CLI. Skip `openshell-sandbox` — the supervisor Windows port is a follow-on skill. - -Verify the binaries exist and report their architecture: - -```powershell -Get-Item "$fork\target\x86_64-pc-windows-msvc\release\openshell-gateway.exe" -Get-Item "$fork\target\aarch64-pc-windows-msvc\release\openshell-gateway.exe" -ErrorAction SilentlyContinue -dumpbin /HEADERS "$fork\target\x86_64-pc-windows-msvc\release\openshell-gateway.exe" | Select-String "machine" -``` - -Expected machine values: `x64` for `x86_64-pc-windows-msvc`, `ARM64` for `aarch64-pc-windows-msvc`. - -### Step 9: mise test on the native Windows architecture +For full validation, detect the Windows host architecture first and choose the +native lane dynamically: ```powershell $arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture -if ($arch -eq [System.Runtime.InteropServices.Architecture]::Arm64) { - mise run --skip-tools windows:test:arm64 - mise run --skip-tools windows:test:unsupported:arm64 -} else { - mise run --skip-tools windows:test:x64 - mise run --skip-tools windows:test:unsupported:x64 -} -``` - -The wrapper rejects test targets that do not match the host architecture. This -keeps ARM64 results native and avoids reporting x64 emulation as ARM64 coverage. - -Failures fall into three buckets: - -1. **Linux-only test** — gate with `#[cfg(not(target_os = "windows"))]` and add a short comment describing the Windows-equivalent test that will eventually replace it. -2. **Path or environment assumption** — fix with conditional `%APPDATA%` paths. -3. **Genuine bug** — fix or open a follow-on issue and gate. - -ARM64 tests are not run locally — they require a native ARM64 runner and are scaffolded in Step 11. - -### Step 10: Validate `$env:OPENSHELL_WXC_EXEC_PATH` (informational) - -This skill does **not** invoke `wxc-exec.exe`. The validation here is a forward-compatibility check so the follow-on MXC driver skill can rely on a known-good path. - -```powershell -if ($env:OPENSHELL_WXC_EXEC_PATH) { - if (-not (Test-Path $env:OPENSHELL_WXC_EXEC_PATH)) { - Write-Warning "OPENSHELL_WXC_EXEC_PATH set but file missing: $env:OPENSHELL_WXC_EXEC_PATH" - } else { - $arch = (dumpbin /HEADERS $env:OPENSHELL_WXC_EXEC_PATH | Select-String "machine").Line - Write-Host "wxc-exec.exe found: $env:OPENSHELL_WXC_EXEC_PATH ($arch)" +switch ($arch.ToString()) { + "X64" { + mise run --skip-tools windows:ci + } + "Arm64" { + mise run --skip-tools windows:check:arm64 + mise run --skip-tools windows:build:arm64 + mise run --skip-tools windows:test:arm64 + mise run --skip-tools windows:test:unsupported:arm64 + mise run --skip-tools windows:artifacts + } + default { + throw "Unsupported Windows host architecture for OpenShell MSVC validation: $arch" } -} else { - Write-Host "OPENSHELL_WXC_EXEC_PATH unset — MXC driver wiring is out of scope for this skill." } ``` -### Step 11: Scaffold ARM64 CI workflow - -Add `.github/workflows/windows-msvc.yml` (or its GitLab equivalent for the fork) with two jobs: `x64` and `arm64`. The x64 job runs on `windows-2025`; the arm64 job runs on a self-hosted runner labelled `windows-arm64`. Do not provision the runner from the skill — emit a TODO comment in the workflow file noting that runner setup is operator work. - -Template (skill writes this verbatim to `.github/workflows/windows-msvc.yml` in the fork): - -```yaml -name: Windows MSVC (build-only) -on: - push: - branches: [windows-mxc-build] - pull_request: -jobs: - x64: - runs-on: windows-2025 - steps: - - uses: actions/checkout@v4 - - uses: jdx/mise-action@v3 - with: - install: false - experimental: true - - uses: dtolnay/rust-toolchain@stable - with: - targets: x86_64-pc-windows-msvc - - run: mise run --skip-tools windows:check:x64 - - run: mise run --skip-tools windows:build:x64 - - run: mise run --skip-tools windows:test:x64 - - run: mise run --skip-tools windows:test:unsupported:x64 - arm64: - # TODO: provision a windows-arm64 self-hosted runner - runs-on: [self-hosted, windows-arm64] - if: false # flip to true once the runner is online - steps: - - uses: actions/checkout@v4 - - uses: jdx/mise-action@v3 - with: - install: false - experimental: true - - uses: dtolnay/rust-toolchain@stable - with: - targets: aarch64-pc-windows-msvc - - run: mise run --skip-tools windows:check:arm64 - - run: mise run --skip-tools windows:build:arm64 -``` +On x64 hosts, `windows:ci` is the full current CI contract and runs in this +order: + +1. x64 check. +2. ARM64 check, unless `OPENSHELL_MXC_SKIP_ARM64=1`. +3. x64 release build. +4. ARM64 release build, unless skipped. +5. Native x64 workspace tests. +6. Focused unsupported-driver contract tests. +7. Artifact reporting. + +The ARM64 check/build steps in this x64-host contract are cross-builds. The +wrapper adds host-native LLVM to `PATH`, requires the ARM64 compiler and +Spectre-mitigated libraries, lets ARM64 crypto crates select `clang-cl`, and +keeps bundled Z3 on native MSVC `cl.exe` with CMake's Visual Studio ARM64 +generator. Do not substitute Ninja: `z3-sys 0.10.9` passes the MSBuild-only +`-m` argument. + +On ARM64 hosts, validate the native ARM64 check, build, and test path. The +wrapper rejects test targets that do not match the host architecture, so x64 +compatibility under emulation is not part of these tasks. The aggregate +`windows:ci` task remains the x64-host CI contract; run the explicit ARM64 +commands above on an ARM64 host. + +The repository-wide `mise run pre-commit` task is also supported on Windows. +Its Rust check, Clippy, and test dependencies enter the same MSVC environment +for the native host target and clear inherited `RUSTC_WRAPPER`. Linux glibc +installer tests and Linux service/RPM packaging-asset tests skip explicitly; +the Linux build-environment shell-helper test also skips; cross-platform checks +continue to run. The blocking Windows Clippy pass excludes unsupported +Windows runtime packages as top-level targets. It allows only unused imports, +dead code, and unused async functions that result from cfg-gated Windows stubs; +other warnings remain errors. + +The wrapper limits Cargo to four jobs by default and serializes wrapper-owned +Cargo commands with a host-local mutex. It deliberately does not set `CL` or +`_CL_`: those variables are also consumed by `clang-cl`, where a global MSVC +option such as `/MP4` can be interpreted as an input file and break ARM64 +crypto dependency builds. + +## Expected Task Behavior + +| Task | Expected behavior | +|---|---| +| `windows:check:x64` | `cargo check --workspace` for `x86_64-pc-windows-msvc`, excluding unsupported Windows packages as top-level workspace targets. | +| `windows:check:arm64` | `cargo check --workspace` for `aarch64-pc-windows-msvc`, with the same top-level exclusions. | +| `windows:build:x64` | Release-builds `openshell-gateway.exe` and `openshell.exe` for x64. | +| `windows:build:arm64` | Release-builds `openshell-gateway.exe` and `openshell.exe` for ARM64. | +| `windows:test:x64` | Runs native x64 workspace tests with `--no-fail-fast`, excluding unsupported Windows packages as top-level workspace targets. | +| `windows:test:arm64` | Runs native ARM64 workspace tests with `--no-fail-fast` and the same package exclusions. Rejects non-ARM64 hosts. | +| `windows:test:unsupported:x64` | Re-runs focused `openshell-server` tests for unsupported Windows driver behavior. | +| `windows:test:unsupported:arm64` | Re-runs the same focused contracts natively on ARM64. Rejects non-ARM64 hosts. | +| `windows:artifacts` | Reports size and SHA256 for release artifacts that exist. | +| `windows:ci` | Runs the full ordered x64-host Windows CI lane, plus ARM64 check/build when not skipped. | + +The unsupported driver package excludes are intentional. They prevent standalone +driver crates from being top-level Windows check/test targets while still +allowing Windows stubs to compile through gateway dependencies. + +## Unsupported Driver Contract + +Windows must continue to reject unsupported compute drivers clearly. + +| Driver | Windows build behavior | Runtime behavior | +|---|---|---| +| Docker | Config/library stub may compile as a gateway dependency. | Gateway construction returns unsupported. | +| Kubernetes | Config/library stub may compile as a gateway dependency. | Gateway construction returns unsupported. | +| Podman | Config/library stub may compile as a gateway dependency. | Gateway construction returns unsupported. | +| VM | Server-side VM path compiles. | VM spawn returns unsupported. | -### Step 12: Commit and report +The focused contract tasks for either native architecture run: -Commit each logical change as a Conventional Commit (per AGENTS.md). Suggested commit sequence: +```text +windows_compute_driver_stubs_report_unsupported +windows_spawn_reports_unsupported +``` -```powershell -git add Cargo.toml Cargo.lock -git commit -m "chore(windows): platform-gate Linux-only dependencies" +These tests are also included in the full x64 workspace test run; the focused +task intentionally re-runs them so unsupported Windows behavior is visible in +the CI report. -git add crates/ -git commit -m "feat(windows): cfg-gate Linux-only modules for MSVC target" +## Test Accounting Guidance -git add .github/workflows/windows-msvc.yml -git commit -m "ci(windows): scaffold x64 + arm64 MSVC workflow" +When reporting `windows:ci`, distinguish these categories: -git add tasks/windows.toml tasks/scripts/windows-msvc.ps1 -git commit -m "chore(windows): add mise MSVC task lane" -``` +- Passed tests from the full x64 workspace test log. +- Passed tests from the full ARM64 workspace test log when run on a native + ARM64 host. +- The two focused unsupported-contract re-runs. +- Explicit Cargo ignored tests, usually ignored doc examples. +- Tests hidden by `#[cfg(not(target_os = "windows"))]`; these often appear as + `running 0 tests`, not as ignored tests. +- Test-name `filtered out` counts from focused `cargo test` invocations. +- Package-level exclusions for unsupported Windows crates; Cargo does not report + those as ignored tests. -Final report should include: +Useful log files: -| Item | Value | +| Log | Meaning | +|---|---| +| `build-x86_64-pc-windows-msvc-check.log` | x64 check output. | +| `build-aarch64-pc-windows-msvc-check.log` | ARM64 check output. | +| `build-x86_64-pc-windows-msvc-release.log` | x64 release build output. | +| `build-aarch64-pc-windows-msvc-release.log` | ARM64 release build output. | +| `test-x86_64-pc-windows-msvc.log` | Full native x64 workspace test output. | +| `test-aarch64-pc-windows-msvc.log` | Full native ARM64 workspace test output. | +| `test-x86_64-pc-windows-msvc-unsupported-*.log` | Focused unsupported-driver contract output. | +| `test-aarch64-pc-windows-msvc-unsupported-*.log` | Focused native ARM64 contract output. | + +The first bundled-Z3 check or test can spend several minutes in CMake/MSBuild +without much console output because Cargo output is redirected to the log. Look +for native `MSBuild.exe` workers before treating the process as stalled. The +artifact report computes SHA256 through .NET directly and does not rely on the +`Get-FileHash` module being available inside the mise-launched Windows +PowerShell process. + +## Common Fix Patterns + +When Windows validation fails: + +1. Identify whether the error is from a top-level Windows deliverable, a + gateway dependency stub, or a Unix-only module leaking into the Windows build. +2. Prefer existing local patterns in the same crate. +3. Gate Unix imports and modules with `#[cfg(unix)]` or + `#[cfg(target_os = "linux")]`. +4. Add or preserve Windows stubs that return unsupported errors. +5. Keep Linux behavior unchanged. +6. Run `cargo fmt --all`, `git diff --check`, and the relevant `windows:*` + tasks after changes. + +Do not add broad abstractions or new Windows runtime support to satisfy a build +error. If a missing runtime feature is required, stop and propose a follow-on +skill or design doc. + +## Final Report Checklist + +Every substantial Windows build run should report: + +| Item | Required detail | |---|---| -| Host preconditions | "OK" or the warning list captured in Step 1 | -| Fork directory | `$env:OPENSHELL_MXC_FORK_DIR` | -| Branch | `$env:OPENSHELL_MXC_FORK_BRANCH` | -| x64 binary | `target\x86_64-pc-windows-msvc\release\openshell-gateway.exe` (size, sha256) | -| ARM64 binary | `target\aarch64-pc-windows-msvc\release\openshell-gateway.exe` (size, sha256) or "skipped" | -| Test summary | passed / failed / gated-out from `test-x64.log` | -| Windows mise lane | status of `windows:check:*`, `windows:build:*`, `windows:test:x64`, and `windows:test:unsupported:x64` | -| Gated modules | list of crates and modules with new `#[cfg(...)]` guards | -| `wxc-exec.exe` | path and architecture, or "unset" | -| Next skills to run | follow-on driver and policy-translation skills | - -## Additional resources - -- [reference.md](reference.md) — Unix dependency audit, common cargo errors and fixes, cfg gating patterns +| Git state | Branch, latest GitLab commit, and whether local changes existed. | +| Host preconditions | OS, Rust, MSVC discovery, and notable warnings. | +| Commands run | Exact `mise run --skip-tools windows:*` commands. | +| x64 check/build | Pass/fail and log path. | +| ARM64 check/build | Pass/fail/skipped and log path. | +| Native tests | Passed/failed/ignored/filtered counts and log path for the host architecture. | +| Unsupported contracts | Which focused tests ran and their result. | +| Artifacts | Binary paths, size, and SHA256 when available. | +| Skips | Explicitly explain tests not run for a non-native architecture, unsupported driver package exclusions, and Windows cfg-gated tests. | +| Follow-ups | Only concrete follow-ups tied to failures or requested scope. | diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index b0c221a0e6..e0d905d8ca 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -1,220 +1,225 @@ -# Reference: Unix dependency audit and cfg gating patterns +# Reference: Windows MSVC maintenance lane -Companion to [SKILL.md](SKILL.md). Use these tables and patterns when Step 4 (apply minimum Windows compatibility shims) hits a specific error during `cargo check --target x86_64-pc-windows-msvc`. +Companion to [SKILL.md](SKILL.md). Use this file for quick lookup while +maintaining the existing build-only Windows MSVC lane. -## Linux-only workspace dependencies +## Lane Files -These crates from the OpenShell workspace `Cargo.toml` do not build on `x86_64-pc-windows-msvc`. Gate them out of the Windows build graph in each consuming crate's `Cargo.toml`. - -| Crate | Used by | Windows fix | -|---|---|---| -| `nix` (features: signal, process, user, fs, term) | `openshell-sandbox`, `openshell-driver-vm`, `openshell-cli` | Move to `[target.'cfg(unix)'.dependencies]` in each consumer | -| `rustix` (features: process) | `openshell-server`, `openshell-sandbox` | Has Windows support — usually compiles; only gate the call sites that use Unix-only functions | -| `landlock` (if present in any crate) | `openshell-sandbox` | `[target.'cfg(target_os = "linux")'.dependencies]` | -| `libseccomp` / `seccompiler` | `openshell-sandbox` | `[target.'cfg(target_os = "linux")'.dependencies]` | -| `caps` | `openshell-sandbox` | `[target.'cfg(unix)'.dependencies]` | -| `procfs` | `openshell-sandbox`, `openshell-driver-vm` | `[target.'cfg(target_os = "linux")'.dependencies]` | -| `libkrun-sys` (transitive via VM driver) | `openshell-driver-vm` | Move Linux implementation behind non-Windows cfg and expose a Windows stub that returns unsupported | - -Compute drivers that must remain unsupported on Windows: - -- `openshell-driver-docker` - preserve config parsing, but Windows runtime construction returns unsupported -- `openshell-driver-podman` - preserve config parsing, but Windows runtime construction returns unsupported -- `openshell-driver-vm` - preserve config parsing, but Windows VM spawn returns unsupported -- `openshell-driver-kubernetes` - preserve config parsing, but Windows runtime construction returns unsupported - -The build-only slice does not need Docker, Kubernetes, Podman, or VM runtime support. Keep these crates in the Windows build graph as library stubs so config files still deserialize and the gateway can return clear unsupported errors. Do not build, package, ship, or smoke-test standalone driver binaries as Windows deliverables, and do not enable Docker Desktop, WSL, Hyper-V, Kubernetes, Podman machine, Podman Desktop, or any VM-backed runtime. - -## Common errors and fixes - -### `error[E0432]: unresolved import 'tokio::net::UnixListener'` - -Wrap the import and all use sites: - -```rust -#[cfg(unix)] -use tokio::net::{UnixListener, UnixStream}; +| File | Purpose | +|---|---| +| `tasks/windows.toml` | Mise task definitions for `windows:*`. | +| `tasks/scripts/windows-msvc.ps1` | Visual Studio environment discovery, rustup target setup, Cargo invocation, logs, artifact report. | +| `.github/workflows/windows-msvc.yml` | GitHub Actions x64 job and disabled ARM64 scaffold. | +| `architecture/windows-msvc-build.md` | Human-readable design contract. | + +## Commands + +Use `--skip-tools` for all Windows mise tasks: + +```powershell +mise run --skip-tools windows:check:x64 +mise run --skip-tools windows:check:arm64 +mise run --skip-tools windows:build:x64 +mise run --skip-tools windows:build:arm64 +mise run --skip-tools windows:test:x64 +mise run --skip-tools windows:test:arm64 +mise run --skip-tools windows:test:unsupported:x64 +mise run --skip-tools windows:test:unsupported:arm64 +mise run --skip-tools windows:ci ``` -For the Windows side, leave a compile-time stub: - -```rust -#[cfg(target_os = "windows")] -pub fn bind_driver_socket(_path: &str) -> anyhow::Result<()> { - anyhow::bail!("named-pipe driver IPC not implemented"); +For host-native full validation, detect architecture first: + +```powershell +$arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture +if ($arch -eq [System.Runtime.InteropServices.Architecture]::Arm64) { + mise run --skip-tools windows:check:arm64 + mise run --skip-tools windows:build:arm64 + mise run --skip-tools windows:test:arm64 + mise run --skip-tools windows:test:unsupported:arm64 + mise run --skip-tools windows:artifacts +} else { + mise run --skip-tools windows:ci } ``` -### `error: failed to run custom build command for 'libseccomp-sys'` +The native test tasks reject a target that does not match the host architecture. +Do not report x64 compatibility-under-emulation coverage from an ARM64 run. -The crate has no Windows backend. Gate it out: +The wrapper adds missing rustup targets and clears inherited +`RUSTC_WRAPPER`. It does not install Visual Studio, Rust, Docker, Kubernetes, +Podman, WSL, Hyper-V, or VM tooling. -```toml -# in crates/openshell-sandbox/Cargo.toml -[target.'cfg(target_os = "linux")'.dependencies] -libseccomp = "..." -``` - -Then in `src/lib.rs`: - -```rust -#[cfg(target_os = "linux")] -mod seccomp; +On Windows, `mise run pre-commit` routes `rust:check`, `rust:lint`, and +`test:rust` through this wrapper for the host-native target. The shared task +definitions retain their existing Unix commands. Only tests for Linux glibc +installer behavior, Linux build-environment shell helpers, and Linux +service/RPM packaging assets skip on Windows. The Windows Clippy command +excludes unsupported runtime packages as top-level targets and allows only +unused imports, dead code, and unused async functions caused by cfg-gated +Windows stubs; other warnings remain errors. + +The wrapper limits Cargo to four jobs by default and serializes wrapper-owned +Cargo commands with a host-local mutex. It does not set `CL` or `_CL_` because +`clang-cl` also consumes them and can parse a global `/MP4` option as an input +file. + +For ARM64, verify the Visual Studio instance contains the ARM64 MSVC tools, +ARM64 Spectre-mitigated libraries, Clang tools, CMake tools, and a Windows SDK. +Clang supplies host-native `libclang.dll` for `bindgen` and `clang-cl.exe` for +ARM64 crypto dependencies such as `ring` and `aws-lc-sys`. Native ARM64 uses +the normal bundled-Z3 CMake path. An x64-to-ARM64 check/build uses CMake's +Visual Studio ARM64 generator with native MSVC `cl.exe` for bundled Z3 while +the crypto crates select `clang-cl`. Do not substitute Ninja: `z3-sys 0.10.9` +passes the MSBuild-only `-m` argument. Use a short `CARGO_TARGET_DIR` if +Windows path-length limits are reached. + +## Unsupported Driver Rules + +Windows is a build target only. These runtimes remain unsupported: + +- Docker +- Kubernetes +- Podman +- VM + +Rules: + +- Keep config/library stubs where the gateway needs them. +- Return clear unsupported errors at runtime. +- Do not build standalone Windows driver binaries. +- Do not add Docker Desktop, WSL, Hyper-V, Podman machine, Podman Desktop, or + VM-backed execution as part of this skill. + +Current focused unsupported-contract tests: + +```text +windows_compute_driver_stubs_report_unsupported +windows_spawn_reports_unsupported ``` -### `error: linking with 'link.exe' failed: exit code: 1120` referencing `nix_*` symbols +Run them with the architecture-specific focused task on the native host. -A `use nix::...` is reachable from the Windows build path. Either gate the `use` with `#[cfg(unix)]` or move the function into a Unix-only module. +## Cargo Excludes -### `error: the trait bound 'PathBuf: From<&str>' is not satisfied` on Windows-only paths +The Windows wrapper intentionally excludes unsupported driver packages as +top-level workspace targets for check/test: -Usually caused by hardcoded forward-slash paths. Replace with `PathBuf::from()` and rely on `std::path::MAIN_SEPARATOR` or use `dirs::data_local_dir()` for `%APPDATA%`. - -### `error[E0599]: no method 'set_nonblocking' found` on `std::os::unix::net::UnixStream` - -Same pattern — gate with `#[cfg(unix)]`. +```text +--exclude openshell-driver-docker +--exclude openshell-driver-kubernetes +--exclude openshell-driver-podman +--exclude openshell-driver-vm +--exclude openshell-supervisor-process +``` -### `cargo:warning=libsecret-sys ... not found` +This does not mean all driver code disappears from the build graph. Docker, +Kubernetes, and Podman stubs can still compile as gateway dependencies. The +process supervisor is also excluded because its runtime is not supported on +Windows. -`libsecret` only works on Linux. If `openshell-providers` pulls it in, gate it. The build-only skill does not need credential storage on Windows; that is a follow-on skill. +## Common Errors -### Bundled Z3 fails with HTTP 403 +### Unix imports leak into Windows builds -The wrapper should fetch the pinned Z3 revision through Git before Cargo starts. -If that prefetch fails, inspect the reported partial checkout and verify that -Git can reach `https://github.com/Z3Prover/z3.git`. +Symptoms: -## Cfg gating patterns +```text +unresolved import std::os::unix +unresolved import tokio::net::UnixListener +unresolved import nix::... +``` -### Module-level +Fix pattern: ```rust -// crates/openshell-sandbox/src/sandbox/mod.rs -#[cfg(target_os = "linux")] -mod linux; +#[cfg(unix)] +use tokio::net::{UnixListener, UnixStream}; +``` + +Move Unix-only functions into Unix-only modules, or add a Windows stub that +returns an unsupported error. -#[cfg(target_os = "windows")] -mod windows; +### Linux-only dependency reaches Windows -#[cfg(target_os = "linux")] -pub use linux::*; +Symptoms: -#[cfg(target_os = "windows")] -pub use windows::*; +```text +failed to run custom build command for libseccomp-sys +pkg-config could not find libsecret ``` -### Cargo dependency-level +Fix pattern: ```toml [target.'cfg(target_os = "linux")'.dependencies] -landlock = "0.4" -libseccomp = "0.3" - -[target.'cfg(target_os = "windows")'.dependencies] -windows-sys = { version = "0.59", features = ["Win32_Storage_FileSystem", "Win32_System_Pipes"] } +libseccomp = "..." ``` -Do **not** add the `windows-sys` dependency in this build-only skill unless a specific compile error demands it. The follow-on MXC driver skill will introduce the Windows API surface. - -### Path defaults +Only gate the dependency if no Windows path should use it. -```rust -pub fn default_state_dir() -> PathBuf { - #[cfg(target_os = "linux")] - { - dirs::data_local_dir() - .unwrap_or_else(|| PathBuf::from("/var/lib")) - .join("openshell") - } - #[cfg(target_os = "windows")] - { - // %APPDATA%\OpenShell - dirs::config_dir() - .unwrap_or_else(|| PathBuf::from(r"C:\ProgramData")) - .join("OpenShell") - } - #[cfg(not(any(target_os = "linux", target_os = "windows")))] - { - PathBuf::from(".openshell") - } -} -``` +### ARM64 check fails but x64 passes -### Test gating +Likely causes: -```rust -#[cfg(not(target_os = "windows"))] -#[test] -fn linux_landlock_smoke() { - // ... -} +- Native dependency does not support `aarch64-pc-windows-msvc`. +- ARM64 MSVC or Spectre-mitigated libraries are missing. +- Host-native `clang-cl` or CMake is missing during an x64-to-ARM64 build. +- `CL` or `_CL_` injects a global MSVC option such as `/MP4` into `clang-cl`. +- Build script assumes x64 tools. +- Inline assembly or prebuilt artifact lacks ARM64 handling. -#[cfg(target_os = "windows")] -#[test] -#[ignore = "supervisor Windows port is a follow-on skill"] -fn windows_supervisor_smoke() { - unimplemented!() -} -``` +Do not skip ARM64 silently. Either fix the target handling or report the exact +blocked dependency. -## Linux build must stay green +### Focused tests report many filtered-out tests -After every cfg gate change, run `cargo check --workspace` on the Linux baseline before committing. The skill's commits must not regress Linux. Use a separate clone or WSL session if running on a Windows-only host, or rely on the existing Linux CI to fail the PR. +This is expected for `windows:test:unsupported:x64`. Cargo runs one named test +and filters the other `openshell-server` tests. Report these as filtered, not +ignored. -## Windows mise lane expectations +## Reporting Counts -The Windows build path is additive. Keep the repository's Linux `mise run ci`, default Cargo tasks, and Linux documentation unchanged. Windows automation lives in `tasks/windows.toml` and delegates to `tasks/scripts/windows-msvc.ps1`. +Use the log summaries from: -On Windows, `mise run pre-commit` routes `rust:check`, `rust:lint`, and -`test:rust` through the same wrapper for the host-native target. Shared task -definitions retain their Unix commands. Only Linux installer, -build-environment shell-helper, and service/RPM-packaging tests skip. The -Windows Clippy command allows unused imports, dead code, and unused async -functions caused by cfg-gated stubs; other warnings remain errors. -Wrapper-owned Cargo commands are serialized across processes and default to -four Cargo jobs to avoid multiplying bundled-Z3 compilation across -parallel pre-commit tasks. Override the limit with a positive integer in -`OPENSHELL_WINDOWS_BUILD_JOBS`. -The wrapper leaves `CL` and `_CL_` unset because `clang-cl` consumes those -variables too. Injecting MSVC-only options such as `/MP` can make ARM64 crypto -dependency builds treat the option as an input file. - -| Task | Expected behavior | +| Log | Count source | |---|---| -| `mise run --skip-tools windows:check:x64` | Runs x64 MSVC `cargo check --workspace` | -| `mise run --skip-tools windows:check:arm64` | Runs ARM64 MSVC `cargo check --workspace` | -| `mise run --skip-tools windows:build:x64` | Builds release `openshell-gateway.exe` and `openshell.exe` for x64 | -| `mise run --skip-tools windows:build:arm64` | Builds release `openshell-gateway.exe` and `openshell.exe` for ARM64 | -| `mise run --skip-tools windows:test:x64` | Runs native x64 workspace tests | -| `mise run --skip-tools windows:test:arm64` | Runs workspace tests on a native ARM64 Windows host | -| `mise run --skip-tools windows:test:unsupported:x64` | Verifies unsupported driver contracts through server/runtime tests without building standalone driver binaries | -| `mise run --skip-tools windows:test:unsupported:arm64` | Verifies the same contracts on a native ARM64 Windows host | -| `mise run --skip-tools windows:ci` | Runs the full Windows lane in order | - -Use `--skip-tools` for Windows CI and automation. Rust must come from rustup with MSVC targets, and Visual Studio Build Tools must provide the linker and SDK. Because `--skip-tools` does not provision mise-managed tools, the Windows wrapper clears inherited `RUSTC_WRAPPER=sccache` before invoking Cargo. The wrapper excludes unsupported driver packages as top-level workspace targets for Windows check/test, but those library stubs still compile when the gateway depends on them. The wrapper may discover `VsDevCmd.bat`, but it must not install Visual Studio, Rust, Docker, Kubernetes, Podman, VM tooling, WSL, or Hyper-V. - -ARM64 validation requires the Visual Studio ARM64 C++ tools, ARM64 -Spectre-mitigated libraries, host-native `libclang.dll` and `clang-cl.exe`, -CMake tools, and an ARM64-capable Windows SDK. During x64-to-ARM64 check/build, -ARM64 crypto crates use `clang-cl` while bundled Z3 stays on native MSVC -`cl.exe` through CMake's Visual Studio generator. The generator must accept the -MSBuild `-m` argument emitted by `z3-sys`. Use a short `CARGO_TARGET_DIR` if -Windows path-length limits are reached. Test tasks reject a target that does -not match the Windows host architecture. - -Bundled Z3 source is pinned by revision, fetched through Git, and cached under -`CARGO_TARGET_DIR`. The wrapper sets `Z3_SYS_BUNDLED_DIR_OVERRIDE` so `z3-sys` -does not make an unauthenticated GitHub Contents API call. An explicit override -must point to a source tree containing `src/api/z3.h`. - -## What NOT to do in this skill - -- Do not implement named-pipe IPC. Stubs only. (Belongs to the follow-on MXC driver skill.) -- Do not add Windows Credential Manager integration. (Follow-on skill.) -- Do not implement DPAPI encryption. (Follow-on skill.) -- Do not create a new MXC driver crate. (Separate skill.) -- Do not write OpenShell → MXC JSON translation. (Separate skill.) -- Do not build MSI installers. (Follow-on skill.) -- Do not modify the `openshell-sandbox` supervisor for Windows beyond cfg-gating to compile. The full port is a follow-on skill. - -The success criterion is **the workspace compiles and basic tests pass on Windows MSVC for both architectures, with the Linux build unchanged**. Nothing more. +| `test-x86_64-pc-windows-msvc.log` | Full x64 workspace test pass. | +| `test-aarch64-pc-windows-msvc.log` | Full native ARM64 workspace test pass. | +| `test-x86_64-pc-windows-msvc-unsupported-*.log` | Focused unsupported-contract re-runs and filtered counts. | +| `test-aarch64-pc-windows-msvc-unsupported-*.log` | Focused native ARM64 re-runs and filtered counts. | + +Separate: + +- passed +- failed +- ignored +- filtered out +- cfg-gated zero-test targets +- package-level excludes + +Package-level excludes are not printed as ignored tests by Cargo. + +## Final Sanity Checks + +Before committing Windows-lane changes, choose checks based on the host +architecture: + +```powershell +cargo fmt --all +git diff --check +$arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture +if ($arch -eq [System.Runtime.InteropServices.Architecture]::Arm64) { + mise run --skip-tools windows:check:arm64 + mise run --skip-tools windows:build:arm64 + mise run --skip-tools windows:test:arm64 + mise run --skip-tools windows:test:unsupported:arm64 +} else { + mise run --skip-tools windows:check:x64 + mise run --skip-tools windows:check:arm64 + mise run --skip-tools windows:test:unsupported:x64 +} +``` + +Run the full x64-host `windows:ci` lane when build or test behavior changed and +the host can run that lane natively. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f23a761441..874e9610b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,7 @@ Skills live in `.agents/skills/`. Your agent's harness can discover and load the | Platform | `generate-sandbox-policy` | Generate YAML sandbox policies from requirements or API docs | | Platform | `helm-dev-environment` | Start and manage the local Kubernetes development environment | | Platform | `tui-development` | Development guide for the ratatui-based terminal UI | +| Platform | `build-openshell-mxc-windows` | Maintain and validate the build-only x64 and ARM64 Windows MSVC lane | | Documentation | `update-docs` | Scan recent commits and draft doc updates for user-facing changes | | Maintenance | `sync-agent-infra` | Detect and fix drift across agent-first infrastructure files | | Reference | `sbom` | Generate SBOMs and resolve dependency licenses | diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 7f5de5a848..b7c5f38183 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -106,9 +106,9 @@ The wrapper defaults Cargo compilation to four jobs. Set `OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override that limit. A host-local mutex serializes wrapper-owned Cargo commands so concurrent pre-commit tasks do not multiply the process count while bundled Z3 compiles. -The wrapper deliberately leaves `CL` and `_CL_` unset because `clang-cl` -consumes those variables too. Injecting MSVC-only options such as `/MP` can -make ARM64 crypto dependency builds treat the option as an input file. +The wrapper does not set `CL` or `_CL_`: those variables are also consumed by +`clang-cl`, where MSVC's `/MP` option can be interpreted as an input file and +break ARM64 crypto dependency builds. ## CI Shape From 6d0f1485ea43f5f7a81041aa15944792a37dfe4d Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Fri, 24 Jul 2026 22:11:18 -0500 Subject: [PATCH 28/30] fix(windows): restore ARM64 Ninja discovery Signed-off-by: Akber Raza --- .../build-openshell-mxc-windows/SKILL.md | 23 ++++++++----- .../build-openshell-mxc-windows/reference.md | 12 +++---- architecture/windows-msvc-build.md | 13 ++++---- tasks/scripts/windows-msvc.ps1 | 33 +++++++++++++++++++ 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index a36f75745b..a2db0a635c 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -117,7 +117,7 @@ The lane targets a Windows host with Visual Studio Build Tools and rustup. | Visual C++ ARM64 tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.ARM64 -property installationPath` | Required for native ARM64 check, build, and tests and for x64-to-ARM64 check/build. Tests always require a native runner. | | Visual C++ ARM64 Spectre-mitigated libraries | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre -property installationPath` | Required by `regorus` through `msvc_spectre_libs`; the build fails when the selected MSVC toolset lacks `lib\spectre\arm64`. | | Visual C++ Clang tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Llvm.Clang -property installationPath` | Provides host-native `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies such as `ring` and `aws-lc-sys`. On ARM64, the wrapper uses `VC\Tools\Llvm\Arm64\bin`. | -| Visual C++ CMake tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.CMake.Project -property installationPath` | Required for bundled Z3. The x64-to-ARM64 path uses CMake's Visual Studio ARM64 generator with native MSVC `cl.exe` so Z3 does not inherit the crypto crates' `clang-cl` requirement. | +| Visual C++ CMake tools | `vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.CMake.Project -property installationPath` | Provides CMake and Ninja. The x64-to-ARM64 path adds Ninja to `PATH` for native dependencies but keeps bundled Z3 on CMake's Visual Studio ARM64 generator with native MSVC `cl.exe`. | | Windows SDK | `where.exe rc.exe` from a Developer PowerShell | Install an SDK containing target libraries and ARM64 tools. | | Rust via rustup | `rustc --version` | Add each target being validated: `x86_64-pc-windows-msvc` and/or `aarch64-pc-windows-msvc`. The wrapper also adds the selected target. | | mise | `mise --version` | Used as a task runner only. | @@ -137,7 +137,7 @@ from this skill. | `CARGO_TARGET_DIR` | `target` under repo root | Override Cargo output location. Use a short absolute path when x64-to-ARM64 builds approach Windows path-length limits. | | `Z3_LIBRARY_PATH_OVERRIDE` | unset | Directory containing an x64 system `libz3.lib`; not valid for ARM64. | | `Z3_SYS_Z3_HEADER` | unset | Full `z3.h` path required with a system Z3 library. | -| `Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned cached source | Existing Z3 source tree containing `src/api/z3.h`. | +| `Z3_SYS_BUNDLED_DIR_OVERRIDE` | pinned source cached under `CARGO_TARGET_DIR` when explicit, otherwise `%LOCALAPPDATA%\OpenShell\cache\z3` | Use an existing Z3 source tree containing `src/api/z3.h`; otherwise the wrapper fetches the pinned revision through Git and sets this automatically. | | `RUSTC_WRAPPER` | cleared by wrapper | The wrapper clears inherited values because `--skip-tools` does not provision `sccache`. | Legacy fork variables such as `OPENSHELL_UPSTREAM`, @@ -192,11 +192,11 @@ order: 7. Artifact reporting. The ARM64 check/build steps in this x64-host contract are cross-builds. The -wrapper adds host-native LLVM to `PATH`, requires the ARM64 compiler and -Spectre-mitigated libraries, lets ARM64 crypto crates select `clang-cl`, and -keeps bundled Z3 on native MSVC `cl.exe` with CMake's Visual Studio ARM64 -generator. Do not substitute Ninja: `z3-sys 0.10.9` passes the MSBuild-only -`-m` argument. +wrapper discovers and adds host-native LLVM and Ninja to `PATH`, requires the +ARM64 compiler and Spectre-mitigated libraries, lets ARM64 crypto crates select +`clang-cl`, and keeps bundled Z3 on native MSVC `cl.exe` with CMake's Visual +Studio ARM64 generator. Z3 does not use Ninja because `z3-sys 0.10.9` passes +the MSBuild-only `-m` argument. On ARM64 hosts, validate the native ARM64 check, build, and test path. The wrapper rejects test targets that do not match the host architecture, so x64 @@ -292,7 +292,14 @@ Useful log files: The first bundled-Z3 check or test can spend several minutes in CMake/MSBuild without much console output because Cargo output is redirected to the log. Look for native `MSBuild.exe` workers before treating the process as stalled. The -artifact report computes SHA256 through .NET directly and does not rely on the +wrapper fetches the pinned Z3 source through Git before Cargo starts. It caches +under an explicitly configured `CARGO_TARGET_DIR`, or under the current user's +local application data directory when Cargo uses its default target tree. +Concurrent commands publish the validated source through an atomic directory +rename, so x64 and ARM64 validation can share the cache safely. The wrapper does +not rely on the rate-limited GitHub Contents API used by `z3-sys`. A failed +fetch reports the partial checkout path for diagnosis. The artifact report +computes SHA256 through .NET directly and does not rely on the `Get-FileHash` module being available inside the mise-launched Windows PowerShell process. diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index e0d905d8ca..38121609c2 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -68,11 +68,11 @@ For ARM64, verify the Visual Studio instance contains the ARM64 MSVC tools, ARM64 Spectre-mitigated libraries, Clang tools, CMake tools, and a Windows SDK. Clang supplies host-native `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies such as `ring` and `aws-lc-sys`. Native ARM64 uses -the normal bundled-Z3 CMake path. An x64-to-ARM64 check/build uses CMake's -Visual Studio ARM64 generator with native MSVC `cl.exe` for bundled Z3 while -the crypto crates select `clang-cl`. Do not substitute Ninja: `z3-sys 0.10.9` -passes the MSBuild-only `-m` argument. Use a short `CARGO_TARGET_DIR` if -Windows path-length limits are reached. +the normal bundled-Z3 CMake path. An x64-to-ARM64 check/build discovers and +adds host-native Ninja to `PATH`, while the crypto crates select `clang-cl`. +Bundled Z3 uses CMake's Visual Studio ARM64 generator with native MSVC `cl.exe` +because `z3-sys 0.10.9` passes the MSBuild-only `-m` argument. Use a short +`CARGO_TARGET_DIR` if Windows path-length limits are reached. ## Unsupported Driver Rules @@ -164,7 +164,7 @@ Likely causes: - Native dependency does not support `aarch64-pc-windows-msvc`. - ARM64 MSVC or Spectre-mitigated libraries are missing. -- Host-native `clang-cl` or CMake is missing during an x64-to-ARM64 build. +- Host-native `clang-cl`, Ninja, or CMake is missing during an x64-to-ARM64 build. - `CL` or `_CL_` injects a global MSVC option such as `/MP4` into `clang-cl`. - Build script assumes x64 tools. - Inline assembly or prebuilt artifact lacks ARM64 handling. diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index b7c5f38183..9c619788ec 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -95,12 +95,13 @@ ARM64 validation requires the Visual Studio ARM64 MSVC tools, ARM64 Spectre-mitigated libraries, host-native Clang tools, CMake tools, and an ARM64-capable Windows SDK. Clang provides `libclang.dll` for `bindgen` and `clang-cl.exe` for ARM64 crypto dependencies. During x64-to-ARM64 check/build, -the wrapper lets `cmake-rs` select the Visual Studio ARM64 generator with native -MSVC `cl.exe` for bundled Z3 so the Z3 build does not inherit the crypto crates' -compiler requirement. The Visual Studio generator is also compatible with the -MSBuild `-m` argument emitted by `z3-sys`; Ninja is not. Artifact hashing uses -.NET SHA256 directly because module autoloading in the mise-launched Windows -PowerShell process is not guaranteed. +the wrapper discovers and adds the Visual Studio-bundled Ninja to `PATH` for +native dependencies. It lets `cmake-rs` select the Visual Studio ARM64 +generator with native MSVC `cl.exe` for bundled Z3 so the Z3 build does not +inherit the crypto crates' compiler requirement. Z3 stays on the Visual Studio +generator because `z3-sys` emits an MSBuild-only `-m` argument that Ninja +rejects. Artifact hashing uses .NET SHA256 directly because module autoloading +in the mise-launched Windows PowerShell process is not guaranteed. The wrapper defaults Cargo compilation to four jobs. Set `OPENSHELL_WINDOWS_BUILD_JOBS` to a positive integer to override that limit. diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index ba7d5a7558..aa02d66e77 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -257,6 +257,35 @@ function Resolve-LibclangPath { throw "Could not find libclang.dll. Install Visual Studio C++ Clang tools, or set LIBCLANG_PATH to the directory containing libclang.dll." } +function Resolve-NinjaPath { + $fromPath = Get-Command ninja.exe -ErrorAction SilentlyContinue + if ($fromPath) { + return $fromPath.Source + } + + $programFilesX86 = [Environment]::GetEnvironmentVariable("ProgramFiles(x86)") + if ($programFilesX86) { + $vswhere = Join-Path $programFilesX86 "Microsoft Visual Studio\Installer\vswhere.exe" + } else { + $vswhere = $null + } + if ($vswhere -and (Test-Path $vswhere)) { + $found = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.CMake.Project -find "Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" | Select-Object -First 1 + if ($found -and (Test-Path $found -PathType Leaf)) { + return (Resolve-Path $found).Path + } + } + + foreach ($installRoot in Get-VsInstallRoots) { + $candidate = Join-Path $installRoot "Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + if (Test-Path $candidate -PathType Leaf) { + return (Resolve-Path $candidate).Path + } + } + + throw "Could not find ninja.exe. Install Microsoft.VisualStudio.Component.VC.CMake.Project." +} + function Add-PathEntry([string] $Directory) { if (($env:PATH -split ";") -notcontains $Directory) { $env:PATH = "$Directory;$env:PATH" @@ -274,8 +303,12 @@ function Configure-Arm64CrossBuild([string[]] $RustTargets) { } Add-PathEntry $env:LIBCLANG_PATH + $ninja = Resolve-NinjaPath + Add-PathEntry (Split-Path -Parent $ninja) + Write-Host "==> ARM64 cross-build toolchain" Write-Host " clang-cl: $clangCl" + Write-Host " ninja: $ninja" Write-Host " Z3: MSVC cl.exe with the Visual Studio generator" } From 15c622a4d4587428b61e6722a5bc2f1916f831fe Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Wed, 29 Jul 2026 10:24:22 -0500 Subject: [PATCH 29/30] refactor(windows): separate platform crate roots Signed-off-by: Akber Raza --- crates/openshell-driver-docker/Cargo.toml | 3 + crates/openshell-driver-docker/src/lib.rs | 3523 ++++++++++++++- .../openshell-driver-docker/src/lib_entry.rs | 8 + .../openshell-driver-docker/src/lib_unix.rs | 3522 --------------- crates/openshell-driver-docker/src/lib_win.rs | 6 + crates/openshell-driver-kubernetes/Cargo.toml | 3 +- .../openshell-driver-kubernetes/src/main.rs | 182 +- .../src/main_entry.rs | 8 + .../src/main_unix.rs | 181 - .../src/main_win.rs | 7 + crates/openshell-driver-podman/Cargo.toml | 6 +- crates/openshell-driver-podman/src/lib.rs | 18 +- .../openshell-driver-podman/src/lib_entry.rs | 8 + .../openshell-driver-podman/src/lib_unix.rs | 15 - crates/openshell-driver-podman/src/lib_win.rs | 6 + crates/openshell-driver-podman/src/main.rs | 186 +- .../openshell-driver-podman/src/main_entry.rs | 8 + .../openshell-driver-podman/src/main_unix.rs | 185 - .../openshell-driver-podman/src/main_win.rs | 7 + crates/openshell-driver-vm/Cargo.toml | 5 +- crates/openshell-driver-vm/src/lib.rs | 25 +- crates/openshell-driver-vm/src/lib_entry.rs | 8 + crates/openshell-driver-vm/src/lib_unix.rs | 23 - crates/openshell-driver-vm/src/lib_win.rs | 6 + crates/openshell-driver-vm/src/main.rs | 729 +++- crates/openshell-driver-vm/src/main_entry.rs | 8 + crates/openshell-driver-vm/src/main_unix.rs | 728 ---- crates/openshell-driver-vm/src/main_win.rs | 7 + crates/openshell-prover/src/lib_unix.rs | 293 -- crates/openshell-sandbox/Cargo.toml | 6 +- crates/openshell-sandbox/src/lib.rs | 3773 ++++++++++++++++- crates/openshell-sandbox/src/lib_entry.rs | 8 + crates/openshell-sandbox/src/lib_unix.rs | 3773 ----------------- crates/openshell-sandbox/src/lib_win.rs | 6 + crates/openshell-sandbox/src/main.rs | 750 +++- crates/openshell-sandbox/src/main_entry.rs | 8 + crates/openshell-sandbox/src/main_unix.rs | 749 ---- crates/openshell-sandbox/src/main_win.rs | 7 + 38 files changed, 9272 insertions(+), 9522 deletions(-) create mode 100644 crates/openshell-driver-docker/src/lib_entry.rs delete mode 100644 crates/openshell-driver-docker/src/lib_unix.rs create mode 100644 crates/openshell-driver-docker/src/lib_win.rs create mode 100644 crates/openshell-driver-kubernetes/src/main_entry.rs delete mode 100644 crates/openshell-driver-kubernetes/src/main_unix.rs create mode 100644 crates/openshell-driver-kubernetes/src/main_win.rs create mode 100644 crates/openshell-driver-podman/src/lib_entry.rs delete mode 100644 crates/openshell-driver-podman/src/lib_unix.rs create mode 100644 crates/openshell-driver-podman/src/lib_win.rs create mode 100644 crates/openshell-driver-podman/src/main_entry.rs delete mode 100644 crates/openshell-driver-podman/src/main_unix.rs create mode 100644 crates/openshell-driver-podman/src/main_win.rs create mode 100644 crates/openshell-driver-vm/src/lib_entry.rs delete mode 100644 crates/openshell-driver-vm/src/lib_unix.rs create mode 100644 crates/openshell-driver-vm/src/lib_win.rs create mode 100644 crates/openshell-driver-vm/src/main_entry.rs delete mode 100644 crates/openshell-driver-vm/src/main_unix.rs create mode 100644 crates/openshell-driver-vm/src/main_win.rs delete mode 100644 crates/openshell-prover/src/lib_unix.rs create mode 100644 crates/openshell-sandbox/src/lib_entry.rs delete mode 100644 crates/openshell-sandbox/src/lib_unix.rs create mode 100644 crates/openshell-sandbox/src/lib_win.rs create mode 100644 crates/openshell-sandbox/src/main_entry.rs delete mode 100644 crates/openshell-sandbox/src/main_unix.rs create mode 100644 crates/openshell-sandbox/src/main_win.rs diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 296274bf87..9ee0a3a388 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -10,6 +10,9 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +path = "src/lib_entry.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } serde = { workspace = true } diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1db6e9f00f..2f89c2229e 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1,11 +1,3522 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -mod windows; +//! Docker compute driver. -#[cfg(target_os = "windows")] -pub use windows::{DockerComputeConfig, default_docker_supervisor_image}; +#![allow(clippy::result_large_err)] -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +use bollard::Docker; +use bollard::errors::Error as BollardError; +use bollard::models::{ + ContainerCreateBody, ContainerSummary, ContainerSummaryStateEnum, CreateImageInfo, + DeviceRequest, EndpointSettings, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, + MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, ProgressDetail, RestartPolicy, + RestartPolicyNameEnum, SystemInfo, +}; +use bollard::query_parameters::{ + CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, + ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, +}; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use openshell_core::config::{ + DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, +}; +use openshell_core::driver_mounts; +use openshell_core::driver_utils::{ + LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, + LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, + supervisor_image_should_refresh, +}; +use openshell_core::gpu::{ + CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, + effective_driver_gpu_count, validate_specific_gpu_device_request, +}; +use openshell_core::progress::{ + PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX, + format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail, +}; +use openshell_core::proto::compute::v1::{ + CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, + DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, + DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, + GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, + compute_driver_server::ComputeDriver, watch_sandboxes_event, +}; +use openshell_core::proto_struct::{ + deserialize_optional_non_empty_string_list, struct_to_json_value, +}; +use openshell_core::{Config, Error, Result as CoreResult}; +use std::collections::{HashMap, HashSet}; +use std::io::Read; +use std::net::{IpAddr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio::task::JoinHandle; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use tracing::{info, warn}; +use url::Url; + +const WATCH_BUFFER: usize = 128; +const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); +const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); + +const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; +const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; +const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; +const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +const SANDBOX_COMMAND: &str = "sleep infinity"; +const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; +const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; +const DOCKER_NETWORK_DRIVER: &str = "bridge"; + +/// Queried by the Docker driver to decide when a sandbox's supervisor +/// relay is live. Implementations return `true` once a sandbox has an +/// active `ConnectSupervisor` session registered. +/// +/// The driver cannot observe the supervisor's SSH socket directly (it +/// lives inside the container), so it leans on this signal to flip the +/// Ready condition from `DependenciesNotReady` to `True`. +pub trait SupervisorReadiness: Send + Sync + 'static { + fn is_supervisor_connected(&self, sandbox_id: &str) -> bool; +} + +/// Gateway-local configuration for the Docker compute driver. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DockerComputeConfig { + /// Docker API Unix socket. When unset, use the socket selected by gateway + /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly + /// configured Docker driver. + pub socket_path: Option, + + /// Default OCI image for sandboxes. + pub default_image: String, + + /// Image pull policy for sandbox images. + pub image_pull_policy: String, + + /// Namespace label applied to Docker sandboxes. + pub sandbox_namespace: String, + + /// Gateway gRPC endpoint the sandbox connects back to. + pub grpc_endpoint: String, + + /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. + pub supervisor_bin: Option, + + /// Optional image used to extract the Linux `openshell-sandbox` binary. + /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for + /// the full resolution order. + pub supervisor_image: Option, + + /// Host-side CA certificate for Docker sandbox mTLS. + pub guest_tls_ca: Option, + + /// Host-side client certificate for Docker sandbox mTLS. + pub guest_tls_cert: Option, + + /// Host-side private key for Docker sandbox mTLS. + pub guest_tls_key: Option, + + /// Docker bridge network that sandbox containers join. + pub network_name: String, + + /// Host gateway IP used for sandbox host aliases. + pub host_gateway_ip: String, + + /// Unix socket path the in-container supervisor bridges relay traffic to. + pub ssh_socket_path: String, + + /// Container cgroup PID limit for Docker-managed sandboxes. + /// + /// Set to `0` to leave Docker's runtime/default PID limit unchanged. + pub sandbox_pids_limit: i64, + + /// Allow sandbox requests to attach host bind mounts through + /// `template.driver_config`. + #[serde(default)] + pub enable_bind_mounts: bool, +} + +impl Default for DockerComputeConfig { + fn default() -> Self { + Self { + socket_path: None, + default_image: openshell_core::image::default_sandbox_image(), + image_pull_policy: String::new(), + sandbox_namespace: "default".to_string(), + grpc_endpoint: String::new(), + supervisor_bin: None, + supervisor_image: None, + guest_tls_ca: None, + guest_tls_cert: None, + guest_tls_key: None, + network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), + host_gateway_ip: String::new(), + ssh_socket_path: "/run/openshell/ssh.sock".to_string(), + sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + enable_bind_mounts: false, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DockerGuestTlsPaths { + pub(crate) ca: PathBuf, + pub(crate) cert: PathBuf, + pub(crate) key: PathBuf, +} + +#[derive(Debug, Clone)] +struct DockerDriverRuntimeConfig { + default_image: String, + image_pull_policy: String, + sandbox_namespace: String, + grpc_endpoint: String, + network_name: String, + gateway_route: DockerGatewayRoute, + ssh_socket_path: String, + stop_timeout_secs: u32, + log_level: String, + supervisor_bin: PathBuf, + guest_tls: Option, + daemon_version: String, + supports_gpu: bool, + allow_all_default_gpu: bool, + sandbox_pids_limit: i64, + enable_bind_mounts: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum DockerGatewayRoute { + Bridge { + bind_address: SocketAddr, + host_alias_ip: IpAddr, + }, + HostGateway, +} + +#[derive(Clone)] +pub struct DockerComputeDriver { + docker: Arc, + config: DockerDriverRuntimeConfig, + events: broadcast::Sender, + pending: Arc>>, + supervisor_readiness: Arc, + gpu_selector: Arc, +} + +struct PendingSandboxRecord { + sandbox: DriverSandbox, + task: Option>, +} + +#[derive(Debug, Clone)] +struct DockerProvisioningFailure { + reason: &'static str, + message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +struct DockerResourceLimits { + nano_cpus: Option, + memory_bytes: Option, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(default, deny_unknown_fields)] +struct DockerSandboxDriverConfig { + #[serde( + default, + deserialize_with = "deserialize_optional_non_empty_string_list" + )] + cdi_devices: Option>, + mounts: Vec, +} + +struct ValidatedDockerSandbox<'a> { + template: &'a DriverSandboxTemplate, + driver_config: DockerSandboxDriverConfig, + gpu_requirements: Option<&'a GpuResourceRequirements>, +} + +impl DockerSandboxDriverConfig { + fn from_template(template: &DriverSandboxTemplate) -> Result { + let Some(config) = template.driver_config.as_ref() else { + return Ok(Self::default()); + }; + + serde_json::from_value(struct_to_json_value(config)) + .map_err(|err| format!("invalid docker driver_config: {err}")) + } +} + +use openshell_core::driver_mounts::SelinuxLabel; + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +enum DockerDriverMountConfig { + Bind { + source: String, + target: String, + #[serde(default = "default_true")] + read_only: bool, + #[serde(default)] + selinux_label: Option, + }, + Volume { + source: String, + target: String, + #[serde(default = "default_true")] + read_only: bool, + #[serde(default)] + subpath: Option, + }, + Tmpfs { + target: String, + #[serde(default)] + options: Vec, + #[serde(default)] + size_bytes: Option, + #[serde(default)] + mode: Option, + }, + Image { + source: String, + target: String, + #[serde(default = "default_true")] + read_only: bool, + #[serde(default)] + subpath: Option, + }, +} + +fn default_true() -> bool { + true +} + +type WatchStream = + Pin> + Send + 'static>>; + +impl DockerComputeDriver { + pub async fn new( + config: &Config, + docker_config: &DockerComputeConfig, + supervisor_readiness: Arc, + ) -> CoreResult { + let socket_path = docker_config + .socket_path + .clone() + .or_else(openshell_core::config::detect_docker_socket) + .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); + let socket_path_str = socket_path.to_str().ok_or_else(|| { + Error::config(format!( + "Docker socket path is not valid UTF-8: {}", + socket_path.display() + )) + })?; + let docker = + Docker::connect_with_socket(socket_path_str, 120, bollard::API_DEFAULT_VERSION) + .map_err(|err| { + Error::execution(format!("failed to create Docker client: {err}")) + })?; + let version = docker.version().await.map_err(|err| { + Error::execution(format!("failed to query Docker daemon version: {err}")) + })?; + let info = docker.info().await.map_err(|err| { + Error::execution(format!("failed to query Docker daemon info: {err}")) + })?; + let supports_gpu = info + .cdi_spec_dirs + .as_ref() + .is_some_and(|dirs| !dirs.is_empty()); + let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); + let allow_all_default_gpu = docker_info_reports_wsl2(&info); + validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + let gateway_port = config.bind_address.port(); + if gateway_port == 0 { + return Err(Error::config( + "docker compute driver requires a fixed non-zero gateway bind port", + )); + } + let network_name = docker_network_name(docker_config); + let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; + let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; + let gateway_route = + docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); + let mut docker_config = docker_config.clone(); + if docker_config.grpc_endpoint.trim().is_empty() { + let scheme = if docker_guest_tls_configured(&docker_config) { + "https" + } else { + "http" + }; + docker_config.grpc_endpoint = + format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); + } + let grpc_endpoint = docker_container_openshell_endpoint( + &docker_config.grpc_endpoint, + HOST_OPENSHELL_INTERNAL, + gateway_port, + ); + let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); + let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; + let guest_tls = docker_guest_tls_paths(&docker_config)?; + + let driver = Self { + docker: Arc::new(docker), + config: DockerDriverRuntimeConfig { + default_image: docker_config.default_image.clone(), + image_pull_policy: docker_config.image_pull_policy.clone(), + sandbox_namespace: docker_config.sandbox_namespace.clone(), + grpc_endpoint, + network_name, + gateway_route, + ssh_socket_path: docker_config.ssh_socket_path.clone(), + stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, + log_level: config.log_level.clone(), + supervisor_bin, + guest_tls, + daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), + supports_gpu, + allow_all_default_gpu, + sandbox_pids_limit: docker_config.sandbox_pids_limit, + enable_bind_mounts: docker_config.enable_bind_mounts, + }, + events: broadcast::channel(WATCH_BUFFER).0, + pending: Arc::new(Mutex::new(HashMap::new())), + supervisor_readiness, + gpu_selector: Arc::new(CdiGpuDefaultSelector::new( + cdi_gpu_inventory, + allow_all_default_gpu, + )), + }; + + let poll_driver = driver.clone(); + tokio::spawn(async move { + poll_driver.poll_loop().await; + }); + + Ok(driver) + } + + #[must_use] + pub fn gateway_bind_addresses(&self) -> Vec { + match self.config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], + DockerGatewayRoute::HostGateway => Vec::new(), + } + } + + fn capabilities(&self) -> GetCapabilitiesResponse { + openshell_core::driver_utils::build_capabilities_response( + "docker", + &self.config.daemon_version, + &self.config.default_image, + ) + } + + #[cfg(test)] + fn validate_sandbox( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + ) -> Result<(), Status> { + let _ = Self::validated_sandbox(sandbox, config)?; + Ok(()) + } + + fn validated_sandbox<'a>( + sandbox: &'a DriverSandbox, + config: &DockerDriverRuntimeConfig, + ) -> Result, Status> { + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; + let template = spec + .template + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + + Self::validate_sandbox_template_base(template)?; + let _ = docker_resource_limits(template)?; + let driver_config = + DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; + validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; + let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); + Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?; + Ok(ValidatedDockerSandbox { + template, + driver_config, + gpu_requirements, + }) + } + + fn validate_sandbox_template_base(template: &DriverSandboxTemplate) -> Result<(), Status> { + if template.image.trim().is_empty() { + return Err(Status::failed_precondition( + "docker sandboxes require a template image", + )); + } + if !template.agent_socket_path.trim().is_empty() { + return Err(Status::failed_precondition( + "docker compute driver does not support template.agent_socket_path", + )); + } + if template + .platform_config + .as_ref() + .is_some_and(|config| !config.fields.is_empty()) + { + return Err(Status::failed_precondition( + "docker compute driver does not support template.platform_config", + )); + } + + Ok(()) + } + + fn validate_sandbox_auth(sandbox: &DriverSandbox) -> Result<(), Status> { + let token_present = sandbox + .spec + .as_ref() + .is_some_and(|spec| !spec.sandbox_token.trim().is_empty()); + if token_present { + return Ok(()); + } + + Err(Status::failed_precondition( + "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]", + )) + } + + fn validate_gpu_request( + gpu_requirements: Option<&GpuResourceRequirements>, + supports_gpu: bool, + driver_config: &DockerSandboxDriverConfig, + ) -> Result<(), Status> { + let requested_count = + effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)?; + if requested_count.is_some() && !supports_gpu { + return Err(Status::failed_precondition( + "docker GPU sandboxes require Docker CDI support. Enable CDI on the Docker daemon, then restart the OpenShell gateway/server so GPU capability is detected.", + )); + } + + if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { + validate_specific_gpu_device_request( + gpu_requirements, + cdi_devices, + "driver_config.cdi_devices", + ) + .map_err(Status::invalid_argument)?; + } + + Ok(()) + } + + async fn validate_user_volume_mounts_available( + &self, + driver_config: &DockerSandboxDriverConfig, + ) -> Result<(), Status> { + for mount in &driver_config.mounts { + if let DockerDriverMountConfig::Volume { source, .. } = mount { + match self.docker.inspect_volume(source).await { + Ok(volume) => { + if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) + { + return Err(Status::failed_precondition(format!( + "docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]" + ))); + } + } + Err(err) if is_not_found_error(&err) => { + return Err(Status::failed_precondition(format!( + "docker volume '{source}' does not exist" + ))); + } + Err(err) => { + return Err(internal_status("inspect docker volume", err)); + } + } + } + } + Ok(()) + } + + async fn refresh_gpu_inventory(&self) -> Result<(), Status> { + let info = self + .docker + .info() + .await + .map_err(|err| internal_status("query Docker daemon info", err))?; + self.gpu_selector.refresh( + docker_cdi_gpu_inventory(&info), + self.config.allow_all_default_gpu, + ); + Ok(()) + } + + async fn resolve_gpu_cdi_devices( + &self, + gpu_requirements: Option<&GpuResourceRequirements>, + driver_config: &DockerSandboxDriverConfig, + select_default_devices: fn( + &CdiGpuDefaultSelector, + u32, + ) -> Result, CdiGpuSelectionError>, + ) -> Result>, Status> { + if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { + validate_specific_gpu_device_request( + gpu_requirements, + cdi_devices, + "driver_config.cdi_devices", + ) + .map_err(Status::invalid_argument)?; + return Ok(Some(cdi_devices.to_vec())); + } + + let Some(count) = + effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)? + else { + return Ok(None); + }; + + self.refresh_gpu_inventory().await?; + select_default_devices(&self.gpu_selector, count) + .map(Some) + .map_err(docker_gpu_selection_status) + } + + async fn get_sandbox_snapshot( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result, Status> { + let container = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await?; + if let Some(sandbox) = container.and_then(|summary| { + sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) + }) { + return Ok(Some(sandbox)); + } + + Ok(self.pending_snapshot(sandbox_id, sandbox_name).await) + } + + async fn current_snapshots(&self) -> Result, Status> { + let containers = self.list_managed_container_summaries().await?; + let container_sandboxes = containers + .iter() + .filter_map(|summary| { + sandbox_from_container_summary(summary, self.supervisor_readiness.as_ref()) + }) + .collect::>(); + let mut by_id = self.pending_snapshot_map().await; + for sandbox in container_sandboxes { + by_id.insert(sandbox.id.clone(), sandbox); + } + let mut sandboxes = by_id.into_values().collect::>(); + sandboxes.sort_by(|left, right| left.id.cmp(&right.id)); + Ok(sandboxes) + } + + async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { + let validated = Self::validated_sandbox(sandbox, &self.config)?; + Self::validate_sandbox_auth(sandbox)?; + self.validate_user_volume_mounts_available(&validated.driver_config) + .await?; + let gpu_devices = self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await?; + let _ = build_container_create_body_with_gpu_devices( + sandbox, + &self.config, + &validated.driver_config, + gpu_devices.as_deref(), + )?; + + if self + .find_managed_container_summary(&sandbox.id, &sandbox.name) + .await? + .is_some() + { + return Err(Status::already_exists("sandbox already exists")); + } + + self.reserve_pending_sandbox(sandbox).await?; + let image = sandbox_image(sandbox).unwrap_or_default(); + self.publish_docker_progress( + &sandbox.id, + "Scheduled", + format!("Docker sandbox accepted for image \"{image}\""), + HashMap::from([("image_ref".to_string(), image)]), + ); + self.publish_sandbox_snapshot(pending_sandbox_snapshot( + sandbox, + &self.config.sandbox_namespace, + provisioning_condition(), + false, + )); + + let driver = self.clone(); + let sandbox_for_task = sandbox.clone(); + let sandbox_id = sandbox.id.clone(); + let task = tokio::spawn(async move { + driver.provision_sandbox(sandbox_for_task).await; + }); + + let mut pending = self.pending.lock().await; + if let Some(record) = pending.get_mut(&sandbox_id) { + record.task = Some(task); + } else { + task.abort(); + } + + Ok(()) + } + + async fn provision_sandbox(&self, sandbox: DriverSandbox) { + match self.provision_sandbox_inner(&sandbox).await { + Ok(()) => { + self.clear_pending_sandbox(&sandbox.id).await; + } + Err(failure) => { + self.fail_pending_sandbox(&sandbox, &failure).await; + } + } + } + + async fn provision_sandbox_inner( + &self, + sandbox: &DriverSandbox, + ) -> Result<(), DockerProvisioningFailure> { + let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { + DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) + })?; + let template = validated.template; + self.ensure_image_available(&sandbox.id, &template.image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("ImagePullFailed", status.message()) + })?; + let token_file_created = write_sandbox_token_file(sandbox, &self.config) + .await + .map_err(|status| { + DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) + })?; + + let container_name = container_name_for_sandbox(sandbox); + let gpu_devices = self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::next_device_ids, + ) + .await + .map_err(|status| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) + })?; + let create_body = build_container_create_body_with_gpu_devices( + sandbox, + &self.config, + &validated.driver_config, + gpu_devices.as_deref(), + ) + .map_err(|status| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) + })?; + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + create_body, + ) + .await + .map_err(|err| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", err), + ) + })?; + self.publish_docker_progress( + &sandbox.id, + "Created", + format!("Created Docker container \"{container_name}\""), + HashMap::from([("container_name".to_string(), container_name.clone())]), + ); + + if let Err(err) = self.docker.start_container(&container_name, None).await { + let cleanup = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + if let Err(cleanup_err) = cleanup { + warn!( + sandbox_id = %sandbox.id, + container_name, + error = %cleanup_err, + "Failed to clean up Docker container after start failure" + ); + } + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + return Err(DockerProvisioningFailure::from_status( + "ContainerStartFailed", + create_status_from_docker_error("start docker sandbox container", err), + )); + } + self.publish_docker_progress( + &sandbox.id, + "Started", + format!("Started Docker container \"{container_name}\""), + HashMap::from([("container_name".to_string(), container_name)]), + ); + if let Err(err) = self + .publish_container_snapshot(&sandbox.id, &sandbox.name) + .await + { + warn!( + sandbox_id = %sandbox.id, + error = %err, + "Failed to publish Docker sandbox snapshot after start" + ); + } + + Ok(()) + } + + async fn delete_sandbox_inner( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; + if let Some(record) = pending.as_ref() + && let Some(task) = record.task.as_ref() + { + task.abort(); + } + + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + if let Some(record) = pending { + let container_name = container_name_for_sandbox(&record.sandbox); + match self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + Ok(()) => { + cleanup_sandbox_token_file(&record.sandbox, &self.config); + return Ok(true); + } + Err(err) if is_not_found_error(&err) => { + cleanup_sandbox_token_file(&record.sandbox, &self.config); + return Ok(true); + } + Err(err) => { + return Err(internal_status("delete docker sandbox container", err)); + } + } + } + return Ok(false); + }; + let Some(target) = summary_container_target(&container) else { + return Ok(pending.is_some()); + }; + + match self + .docker + .remove_container( + &target, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + Ok(()) => { + cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + Ok(true) + } + Err(err) if is_not_found_error(&err) => { + cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + Ok(pending.is_some()) + } + Err(err) => Err(internal_status("delete docker sandbox container", err)), + } + } + + async fn stop_sandbox_inner(&self, sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + if let Some(task) = record.task { + task.abort(); + } + cleanup_sandbox_token_file(&record.sandbox, &self.config); + self.publish_deleted(record.sandbox.id); + return Ok(()); + } + return Err(Status::not_found("sandbox not found")); + }; + let Some(target) = summary_container_target(&container) else { + return Err(Status::not_found("sandbox container has no id or name")); + }; + + match self + .docker + .stop_container( + &target, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) + .build(), + ), + ) + .await + { + Ok(()) => Ok(()), + Err(err) if is_not_modified_error(&err) => Ok(()), + Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), + Err(err) => Err(internal_status("stop docker sandbox container", err)), + } + } + + /// Start a managed sandbox container that was previously stopped. Used + /// by the gateway to resume sandboxes after a restart so that running + /// state in the gateway store is matched by an actually-running + /// container. + /// + /// Returns `Ok(true)` when a container existed and was started (or was + /// already running), `Ok(false)` when no managed container is found for + /// the sandbox, and `Err(...)` for any Docker failure. + pub async fn resume_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + return Ok(false); + }; + let Some(target) = summary_container_target(&container) else { + return Ok(false); + }; + let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); + if !container_state_needs_resume(state) { + return Ok(true); + } + + match self.docker.start_container(&target, None).await { + Ok(()) => Ok(true), + // Already running — race with another resume path or the + // restart policy. Treat as success. + Err(err) if is_not_modified_error(&err) => Ok(true), + Err(err) if is_not_found_error(&err) => Ok(false), + Err(err) => Err(internal_status("start docker sandbox container", err)), + } + } + + pub async fn stop_managed_containers_on_shutdown(&self) -> Result { + let containers = self.list_managed_container_summaries().await?; + let targets = containers + .into_iter() + .filter_map(|container| { + let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); + if container_state_needs_shutdown_stop(state) { + summary_container_target(&container) + } else { + None + } + }) + .collect::>(); + let target_count = targets.len(); + let mut stopped = 0usize; + let mut failures = Vec::new(); + let stop_timeout_secs = self.config.stop_timeout_secs; + + let mut stop_results = futures::stream::iter(targets.into_iter().map(|target| { + let docker = self.docker.clone(); + async move { + let result = docker + .stop_container( + &target, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(stop_timeout_secs)) + .build(), + ), + ) + .await; + (target, result) + } + })) + .buffer_unordered(16); + + while let Some((target, result)) = stop_results.next().await { + match result { + Ok(()) => { + stopped += 1; + } + Err(err) if is_not_found_error(&err) || is_not_modified_error(&err) => {} + Err(err) => { + warn!( + container = %target, + error = %err, + "Failed to stop Docker sandbox container during shutdown" + ); + failures.push(target); + } + } + } + + if !failures.is_empty() { + return Err(Status::internal(format!( + "failed to stop {} of {target_count} Docker sandbox containers during shutdown", + failures.len() + ))); + } + + Ok(stopped) + } + + async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { + let mut pending = self.pending.lock().await; + if pending + .values() + .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) + { + return Err(Status::already_exists("sandbox already exists")); + } + + pending.insert( + sandbox.id.clone(), + PendingSandboxRecord { + sandbox: pending_sandbox_snapshot( + sandbox, + &self.config.sandbox_namespace, + provisioning_condition(), + false, + ), + task: None, + }, + ); + Ok(()) + } + + async fn pending_snapshot( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Option { + let pending = self.pending.lock().await; + pending + .values() + .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) + .map(|record| record.sandbox.clone()) + } + + async fn pending_snapshot_map(&self) -> HashMap { + let pending = self.pending.lock().await; + pending + .iter() + .map(|(sandbox_id, record)| (sandbox_id.clone(), record.sandbox.clone())) + .collect() + } + + async fn clear_pending_sandbox(&self, sandbox_id: &str) { + let mut pending = self.pending.lock().await; + pending.remove(sandbox_id); + } + + async fn remove_pending_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Option { + let mut pending = self.pending.lock().await; + let id = pending.iter().find_map(|(id, record)| { + pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) + })?; + pending.remove(&id) + } + + async fn fail_pending_sandbox( + &self, + sandbox: &DriverSandbox, + failure: &DockerProvisioningFailure, + ) { + cleanup_sandbox_token_file(sandbox, &self.config); + let snapshot = pending_sandbox_snapshot( + sandbox, + &self.config.sandbox_namespace, + error_condition(failure.reason, &failure.message), + false, + ); + { + let mut pending = self.pending.lock().await; + if let Some(record) = pending.get_mut(&sandbox.id) { + record.sandbox = snapshot.clone(); + record.task = None; + } else { + return; + } + } + + self.publish_platform_event( + sandbox.id.clone(), + platform_event( + "docker", + "Warning", + failure.reason, + format!("Docker sandbox provisioning failed: {}", failure.message), + ), + ); + self.publish_sandbox_snapshot(snapshot); + } + + async fn publish_container_snapshot( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result<(), Status> { + if let Some(summary) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + && let Some(sandbox) = + sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) + { + self.publish_sandbox_snapshot(sandbox); + } + Ok(()) + } + + fn publish_sandbox_snapshot(&self, sandbox: DriverSandbox) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + }); + } + + fn publish_deleted(&self, sandbox_id: String) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id }, + )), + }); + } + + fn publish_platform_event(&self, sandbox_id: String, event: DriverPlatformEvent) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id, + event: Some(event), + }, + )), + }); + } + + fn publish_docker_progress( + &self, + sandbox_id: &str, + reason: &str, + message: String, + mut metadata: HashMap, + ) { + attach_docker_progress_metadata(&mut metadata, reason, &message); + self.publish_platform_event( + sandbox_id.to_string(), + DriverPlatformEvent { + timestamp_ms: openshell_core::time::now_ms(), + source: "docker".to_string(), + r#type: "Normal".to_string(), + reason: reason.to_string(), + message, + metadata, + }, + ); + } + + async fn poll_loop(self) { + let mut previous = match self.current_snapshot_map().await { + Ok(snapshots) => snapshots, + Err(err) => { + warn!(error = %err, "Failed to seed Docker sandbox watch state"); + HashMap::new() + } + }; + + // Exponential backoff on consecutive Docker failures to avoid a 2s + // warn-log flood when the daemon is unreachable for an extended + // period (e.g. restart, socket removed). + let mut backoff = WATCH_POLL_INTERVAL; + loop { + tokio::time::sleep(backoff).await; + match self.current_snapshot_map().await { + Ok(current) => { + emit_snapshot_diff(&self.events, &previous, ¤t); + previous = current; + backoff = WATCH_POLL_INTERVAL; + } + Err(err) => { + warn!( + error = %err, + backoff_secs = backoff.as_secs(), + "Failed to poll Docker sandboxes" + ); + backoff = (backoff * 2).min(WATCH_POLL_MAX_BACKOFF); + } + } + } + } + + async fn current_snapshot_map(&self) -> Result, Status> { + self.current_snapshots().await.map(|snapshots| { + snapshots + .into_iter() + .map(|sandbox| (sandbox.id.clone(), sandbox)) + .collect() + }) + } + + async fn list_managed_container_summaries(&self) -> Result, Status> { + let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); + self.docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|err| internal_status("list Docker sandbox containers", err)) + } + + async fn find_managed_container_summary( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result, Status> { + let mut label_filter_values = Vec::new(); + if !sandbox_id.is_empty() { + label_filter_values.push(format!("{LABEL_SANDBOX_ID}={sandbox_id}")); + } else if !sandbox_name.is_empty() { + label_filter_values.push(format!("{LABEL_SANDBOX_NAME}={sandbox_name}")); + } + + let filters = + managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); + let containers = self + .docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|err| internal_status("find Docker sandbox container", err))?; + + Ok(containers.into_iter().find(|summary| { + let Some(labels) = summary.labels.as_ref() else { + return false; + }; + let namespace_matches = labels + .get(LABEL_SANDBOX_NAMESPACE) + .is_some_and(|value| value == &self.config.sandbox_namespace); + let id_matches = sandbox_id.is_empty() + || labels + .get(LABEL_SANDBOX_ID) + .is_some_and(|value| value == sandbox_id); + let name_matches = sandbox_name.is_empty() + || labels + .get(LABEL_SANDBOX_NAME) + .is_some_and(|value| value == sandbox_name); + namespace_matches && id_matches && name_matches + })) + } + + async fn ensure_image_available(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { + let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); + match policy.as_str() { + "" | "ifnotpresent" => { + if self.docker.inspect_image(image).await.is_ok() { + self.publish_docker_progress( + sandbox_id, + "ImagePresent", + format!("Docker image \"{image}\" is already present"), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + return Ok(()); + } + self.pull_image(sandbox_id, image).await + } + "always" => self.pull_image(sandbox_id, image).await, + "never" => match self.docker.inspect_image(image).await { + Ok(_) => { + self.publish_docker_progress( + sandbox_id, + "ImagePresent", + format!("Docker image \"{image}\" is already present"), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + Ok(()) + } + Err(err) if is_not_found_error(&err) => Err(Status::failed_precondition(format!( + "docker image '{image}' is not present locally and image_pull_policy=Never" + ))), + Err(err) => Err(internal_status("inspect Docker image", err)), + }, + other => Err(Status::failed_precondition(format!( + "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", + ))), + } + } + + async fn pull_image(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { + self.publish_docker_progress( + sandbox_id, + "Pulling", + format!("Pulling Docker image \"{image}\""), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + let mut stream = self.docker.create_image( + Some(CreateImageOptions { + from_image: Some(image.to_string()), + ..Default::default() + }), + None, + None, + ); + while let Some(result) = stream.next().await { + let info = result.map_err(|err| internal_status("pull Docker image", err))?; + if let Some(message) = info + .error_detail + .as_ref() + .and_then(|detail| detail.message.as_ref()) + { + return Err(Status::failed_precondition(format!( + "pull Docker image '{image}' failed: {message}" + ))); + } + if let Some(event) = docker_pull_progress_event(image, &info) { + self.publish_platform_event(sandbox_id.to_string(), event); + } + } + self.publish_docker_progress( + sandbox_id, + "Pulled", + format!("Pulled Docker image \"{image}\""), + HashMap::from([("image_ref".to_string(), image.to_string())]), + ); + Ok(()) + } +} + +#[tonic::async_trait] +impl ComputeDriver for DockerComputeDriver { + type WatchSandboxesStream = WatchStream; + + async fn get_capabilities( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(self.capabilities())) + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + let validated = Self::validated_sandbox(&sandbox, &self.config)?; + self.validate_user_volume_mounts_available(&validated.driver_config) + .await?; + let _ = self + .resolve_gpu_cdi_devices( + validated.gpu_requirements, + &validated.driver_config, + CdiGpuDefaultSelector::peek_device_ids, + ) + .await?; + Ok(Response::new(ValidateSandboxCreateResponse {})) + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; + + let sandbox = self + .get_sandbox_snapshot(&request.sandbox_id, &request.sandbox_name) + .await? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + + if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { + return Err(Status::failed_precondition( + "sandbox_id did not match the fetched sandbox", + )); + } + + Ok(Response::new(GetSandboxResponse { + sandbox: Some(sandbox), + })) + } + + async fn list_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(ListSandboxesResponse { + sandboxes: self.current_snapshots().await?, + })) + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.create_sandbox_inner(&sandbox).await?; + Ok(Response::new(CreateSandboxResponse {})) + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; + + self.stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) + .await?; + Ok(Response::new(StopSandboxResponse {})) + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; + + let event_sandbox_id = request.sandbox_id.clone(); + let deleted = self + .delete_sandbox_inner(&request.sandbox_id, &request.sandbox_name) + .await?; + if deleted && !event_sandbox_id.is_empty() { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { + sandbox_id: event_sandbox_id, + }, + )), + }); + } + + Ok(Response::new(DeleteSandboxResponse { deleted })) + } + + async fn watch_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + // Subscribe before taking the initial snapshot so any event emitted + // between the snapshot and this subscriber becoming active is still + // delivered. Downstream consumers treat sandbox events as + // idempotent (keyed by sandbox id), so a duplicate event is benign + // while a missed one leaks state. + let mut rx = self.events.subscribe(); + let initial = self.current_snapshots().await?; + let (tx, out_rx) = mpsc::channel(WATCH_BUFFER); + tokio::spawn(async move { + for sandbox in initial { + if tx + .send(Ok(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + })) + .await + .is_err() + { + return; + } + } + + loop { + match rx.recv().await { + Ok(event) => { + if tx.send(Ok(event)).await.is_err() { + return; + } + } + Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => return, + } + } + }); + + Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) + } +} + +impl DockerProvisioningFailure { + fn new(reason: &'static str, message: impl Into) -> Self { + Self { + reason, + message: message.into(), + } + } + + fn from_status(reason: &'static str, status: Status) -> Self { + Self::new(reason, status.message()) + } +} + +fn sandbox_image(sandbox: &DriverSandbox) -> Option { + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map(|template| template.image.clone()) + .filter(|image| !image.trim().is_empty()) +} + +fn pending_sandbox_snapshot( + sandbox: &DriverSandbox, + namespace: &str, + condition: DriverCondition, + deleting: bool, +) -> DriverSandbox { + DriverSandbox { + id: sandbox.id.clone(), + name: sandbox.name.clone(), + namespace: namespace.to_string(), + spec: None, + status: Some(DriverSandboxStatus { + sandbox_name: sandbox.name.clone(), + instance_id: String::new(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![condition], + deleting, + }), + workspace: sandbox.workspace.clone(), + } +} + +fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { + (!sandbox_id.is_empty() && sandbox.id == sandbox_id) + || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) +} + +fn provisioning_condition() -> DriverCondition { + DriverCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "Starting".to_string(), + message: "Docker container is starting".to_string(), + last_transition_time: String::new(), + } +} + +fn error_condition(reason: &str, message: &str) -> DriverCondition { + DriverCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + message: message.to_string(), + last_transition_time: String::new(), + } +} + +fn platform_event( + source: &str, + event_type: &str, + reason: &str, + message: String, +) -> DriverPlatformEvent { + DriverPlatformEvent { + timestamp_ms: openshell_core::time::now_ms(), + source: source.to_string(), + r#type: event_type.to_string(), + reason: reason.to_string(), + message, + metadata: HashMap::new(), + } +} + +fn docker_pull_progress_event(image: &str, info: &CreateImageInfo) -> Option { + let status = info.status.as_deref().map(str::trim)?; + if status.is_empty() { + return None; + } + + let mut metadata = HashMap::from([ + ("image_ref".to_string(), image.to_string()), + ("docker_status".to_string(), status.to_string()), + ]); + if let Some(layer_id) = info.id.as_deref().filter(|id| !id.is_empty()) { + metadata.insert("layer_id".to_string(), layer_id.to_string()); + } + if let Some(detail) = docker_pull_progress_detail(info) { + metadata.insert("detail".to_string(), detail); + } + attach_docker_progress_metadata(&mut metadata, "PullingLayer", status); + + Some(DriverPlatformEvent { + timestamp_ms: openshell_core::time::now_ms(), + source: "docker".to_string(), + r#type: "Normal".to_string(), + reason: "PullingLayer".to_string(), + message: docker_pull_message(info, status), + metadata, + }) +} + +fn docker_pull_message(info: &CreateImageInfo, status: &str) -> String { + info.id.as_deref().filter(|id| !id.is_empty()).map_or_else( + || format!("Docker image pull: {status}"), + |layer_id| format!("Docker image pull {layer_id}: {status}"), + ) +} + +fn docker_pull_progress_detail(info: &CreateImageInfo) -> Option { + let status = info.status.as_deref().unwrap_or("Pulling"); + let layer_id = info.id.as_deref().filter(|id| !id.is_empty()); + let progress = info + .progress_detail + .as_ref() + .and_then(format_progress_detail); + + match (layer_id, progress) { + (Some(layer_id), Some(progress)) => Some(format!("{status} {layer_id} ({progress})")), + (Some(layer_id), None) => Some(format!("{status} {layer_id}")), + (None, Some(progress)) => Some(format!("{status} ({progress})")), + (None, None) => (!status.is_empty()).then(|| status.to_string()), + } +} + +fn format_progress_detail(progress: &ProgressDetail) -> Option { + let current = progress.current.and_then(|value| u64::try_from(value).ok()); + let total = progress + .total + .and_then(|value| u64::try_from(value).ok()) + .filter(|value| *value > 0); + + match (current, total) { + (Some(current), Some(total)) => { + Some(format!("{}/{}", format_bytes(current), format_bytes(total))) + } + (Some(current), _) if current > 0 => Some(format_bytes(current)), + _ => None, + } +} + +fn attach_docker_progress_metadata( + metadata: &mut HashMap, + reason: &str, + message: &str, +) { + match reason { + "Scheduled" => { + mark_progress_complete( + metadata, + PROGRESS_STEP_REQUESTING_SANDBOX, + "Sandbox allocated", + ); + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(image) = metadata.get("image_ref").cloned() { + mark_progress_detail(metadata, image); + } + } + "Pulling" => { + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(image) = metadata.get("image_ref").cloned() { + mark_progress_detail(metadata, image); + } + } + "PullingLayer" => { + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(detail) = metadata + .get("detail") + .cloned() + .filter(|detail| !detail.is_empty()) + { + mark_progress_detail(metadata, detail); + } else if !message.is_empty() { + mark_progress_detail(metadata, message); + } + } + "ImagePresent" => { + mark_progress_complete( + metadata, + PROGRESS_STEP_PULLING_IMAGE, + "Image already present", + ); + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + } + "Pulled" => { + mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, "Image pulled"); + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + } + "Created" => { + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + mark_progress_detail(metadata, "Container created"); + } + "Started" => { + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + mark_progress_detail(metadata, "Waiting for supervisor relay"); + } + _ => {} + } +} + +#[cfg(test)] +fn docker_driver_config( + template: &DriverSandboxTemplate, + enable_bind_mounts: bool, +) -> Result { + let config = + DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; + validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?; + Ok(config) +} + +/// Collect user-supplied bind mounts as string-format binds. +/// +/// Bind mounts use the legacy `Binds` field (`-v` syntax) rather than the +/// structured `Mount` API because the Docker Engine Mount object does not +/// support `SELinux` relabelling (`:z` / `:Z`). The string format does. +fn docker_driver_bind_strings(config: &DockerSandboxDriverConfig) -> Result, Status> { + config + .mounts + .iter() + .filter_map(|m| match m { + DockerDriverMountConfig::Bind { + source, + target, + read_only, + selinux_label, + } => Some(docker_bind_string( + source, + target, + *read_only, + *selinux_label, + )), + _ => None, + }) + .collect() +} + +fn docker_bind_string( + source: &str, + target: &str, + read_only: bool, + selinux_label: Option, +) -> Result { + driver_mounts::validate_absolute_mount_source(source, "bind source") + .map_err(Status::failed_precondition)?; + // Legacy `-v` binds silently create missing source directories as empty, + // root-owned paths. The structured `--mount` API that was used before this + // change rejected missing sources at container-create time. Preserve that + // fail-fast behaviour with an explicit existence check. + if !Path::new(source).exists() { + return Err(Status::failed_precondition(format!( + "bind source path does not exist: {source}" + ))); + } + driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; + let normalized_target = driver_mounts::normalize_mount_target(target); + + let mut opts = Vec::new(); + if read_only { + opts.push("ro"); + } + match selinux_label { + Some(SelinuxLabel::Shared) => opts.push("z"), + Some(SelinuxLabel::Private) => opts.push("Z"), + None => {} + } + + if opts.is_empty() { + Ok(format!("{source}:{normalized_target}")) + } else { + Ok(format!("{source}:{normalized_target}:{}", opts.join(","))) + } +} + +/// Collect user-supplied non-bind mounts as structured `Mount` objects. +fn docker_driver_mounts(config: &DockerSandboxDriverConfig) -> Result, Status> { + config + .mounts + .iter() + .filter_map(|m| docker_mount_from_config(m).transpose()) + .collect() +} + +fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result, Status> { + match config { + DockerDriverMountConfig::Bind { .. } => { + // Bind mounts are handled via docker_driver_bind_strings. + Ok(None) + } + DockerDriverMountConfig::Volume { + source, + target, + read_only, + subpath, + } => Ok(Some(Mount { + typ: Some(MountTypeEnum::VOLUME), + source: Some(source.clone()), + target: Some(target.clone()), + read_only: Some(*read_only), + volume_options: subpath.as_ref().map(|subpath| MountVolumeOptions { + subpath: Some(subpath.clone()), + ..Default::default() + }), + ..Default::default() + })), + DockerDriverMountConfig::Tmpfs { + target, + options, + size_bytes, + mode, + } => Ok(Some(Mount { + typ: Some(MountTypeEnum::TMPFS), + target: Some(target.clone()), + tmpfs_options: Some(MountTmpfsOptions { + size_bytes: validate_optional_positive_integral_i64( + *size_bytes, + "tmpfs size_bytes", + )?, + mode: validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?, + options: (!options.is_empty()) + .then(|| { + options + .iter() + .map(|option| docker_tmpfs_option(option)) + .collect::, _>>() + }) + .transpose()?, + }), + ..Default::default() + })), + DockerDriverMountConfig::Image { .. } => Err(Status::failed_precondition( + "invalid docker driver_config: docker image mounts are not supported", + )), + } +} + +fn validate_docker_driver_mounts( + mounts: &[DockerDriverMountConfig], + enable_bind_mounts: bool, +) -> Result<(), Status> { + let mut targets = HashSet::new(); + for mount in mounts { + let target = match mount { + DockerDriverMountConfig::Bind { source, target, .. } => { + if !enable_bind_mounts { + return Err(Status::failed_precondition( + "docker bind mounts require enable_bind_mounts = true in [openshell.drivers.docker]", + )); + } + driver_mounts::validate_absolute_mount_source(source, "bind source") + .map_err(Status::failed_precondition)?; + target + } + DockerDriverMountConfig::Volume { + source, + target, + subpath, + .. + } => { + driver_mounts::validate_mount_source(source, "volume source") + .map_err(Status::failed_precondition)?; + if let Some(subpath) = subpath { + driver_mounts::validate_mount_subpath(subpath) + .map_err(Status::failed_precondition)?; + } + target + } + DockerDriverMountConfig::Tmpfs { + target, + options, + size_bytes, + mode, + } => { + validate_optional_positive_integral_i64(*size_bytes, "tmpfs size_bytes")?; + validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?; + for option in options { + docker_tmpfs_option(option)?; + } + target + } + DockerDriverMountConfig::Image { + source, + target, + read_only, + subpath, + } => { + let _ = (source, target, read_only, subpath); + return Err(Status::failed_precondition( + "invalid docker driver_config: docker image mounts are not supported", + )); + } + }; + driver_mounts::validate_container_mount_target(target) + .map_err(Status::failed_precondition)?; + let normalized_target = driver_mounts::normalize_mount_target(target); + if !targets.insert(normalized_target.clone()) { + return Err(Status::failed_precondition(format!( + "duplicate docker driver_config mount target '{normalized_target}'" + ))); + } + } + Ok(()) +} + +fn validate_optional_positive_integral_i64( + value: Option, + field: &str, +) -> Result, Status> { + let Some(value) = validate_optional_integral_i64(value, field)? else { + return Ok(None); + }; + if value <= 0 { + return Err(Status::failed_precondition(format!( + "{field} must be positive" + ))); + } + Ok(Some(value)) +} + +fn validate_optional_nonnegative_integral_i64( + value: Option, + field: &str, +) -> Result, Status> { + let Some(value) = validate_optional_integral_i64(value, field)? else { + return Ok(None); + }; + if value < 0 { + return Err(Status::failed_precondition(format!( + "{field} must be zero or greater" + ))); + } + Ok(Some(value)) +} + +fn validate_optional_integral_i64(value: Option, field: &str) -> Result, Status> { + let Some(value) = value else { + return Ok(None); + }; + if !value.is_finite() || value.fract() != 0.0 { + return Err(Status::failed_precondition(format!( + "{field} must be an integer" + ))); + } + value.to_string().parse::().map(Some).map_err(|_| { + Status::failed_precondition(format!("{field} must be representable as an i64")) + }) +} + +fn docker_tmpfs_option(option: &str) -> Result, Status> { + let option = option.trim(); + if option.is_empty() { + return Err(Status::failed_precondition( + "tmpfs options must not contain empty values", + )); + } + if let Some((key, value)) = option.split_once('=') { + let key = key.trim(); + let value = value.trim(); + if key.is_empty() || value.is_empty() { + return Err(Status::failed_precondition( + "tmpfs key=value options must include both key and value", + )); + } + Ok(vec![key.to_string(), value.to_string()]) + } else { + Ok(vec![option.to_string()]) + } +} + +fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { + volume.driver == "local" + && volume.options.get("o").is_some_and(|options| { + options.split(',').any(|option| { + let option = option.trim(); + option.eq_ignore_ascii_case("bind") || option.eq_ignore_ascii_case("rbind") + }) + }) +} + +fn build_binds( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let mut binds = vec![format!( + "{}:{}:ro,z", + config.supervisor_bin.display(), + SUPERVISOR_MOUNT_PATH + )]; + if let Some(tls) = &config.guest_tls { + binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); + binds.push(format!( + "{}:{}:ro,z", + tls.cert.display(), + TLS_CERT_MOUNT_PATH + )); + binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); + } + if sandbox + .spec + .as_ref() + .is_some_and(|spec| !spec.sandbox_token.is_empty()) + { + binds.push(format!( + "{}:{}:ro,z", + sandbox_token_host_path(sandbox, config)?.display(), + SANDBOX_TOKEN_MOUNT_PATH + )); + } + Ok(binds) +} + +fn sandbox_token_host_path( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + sandbox_token_host_path_by_id(&sandbox.id, config) +} + +fn sandbox_token_host_path_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result { + openshell_core::driver_utils::sandbox_token_path( + "docker-sandbox-tokens", + Some(&config.sandbox_namespace), + sandbox_id, + ) + .map_err(|err| { + Status::internal(format!( + "resolve sandbox token state directory failed: {err}" + )) + }) +} + +async fn write_sandbox_token_file( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let Some(spec) = sandbox.spec.as_ref() else { + return Ok(false); + }; + if spec.sandbox_token.is_empty() { + return Ok(false); + } + let path = sandbox_token_host_path(sandbox, config)?; + if let Some(parent) = path.parent() { + openshell_core::paths::create_dir_restricted(parent).map_err(|err| { + Status::internal(format!( + "create sandbox token directory {} failed: {err}", + parent.display() + )) + })?; + } + tokio::fs::write(&path, format!("{}\n", spec.sandbox_token)) + .await + .map_err(|err| { + Status::internal(format!( + "write sandbox token file {} failed: {err}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(&path).map_err(|err| { + Status::internal(format!( + "restrict sandbox token file {} failed: {err}", + path.display() + )) + })?; + Ok(true) +} + +fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { + cleanup_sandbox_token_file_by_id(&sandbox.id, config); +} + +fn cleanup_sandbox_token_file_for_delete( + sandbox_id: &str, + pending: Option<&PendingSandboxRecord>, + config: &DockerDriverRuntimeConfig, +) { + if !sandbox_id.is_empty() { + cleanup_sandbox_token_file_by_id(sandbox_id, config); + } else if let Some(record) = pending { + cleanup_sandbox_token_file(&record.sandbox, config); + } +} + +fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { + return; + }; + if let Err(err) = std::fs::remove_file(&path) + && err.kind() != std::io::ErrorKind::NotFound + { + warn!( + sandbox_id = %sandbox_id, + path = %path.display(), + error = %err, + "Failed to remove Docker sandbox token file" + ); + } + if let Some(dir) = path.parent() { + let _ = std::fs::remove_dir(dir); + } +} + +fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { + let mut environment = HashMap::from([ + ("HOME".to_string(), "/root".to_string()), + ("PATH".to_string(), SUPERVISOR_PATH.to_string()), + ("TERM".to_string(), "xterm".to_string()), + ( + "OPENSHELL_LOG_LEVEL".to_string(), + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), + ), + ]); + + if let Some(spec) = sandbox.spec.as_ref() { + let mut user_env = HashMap::new(); + if let Some(template) = spec.template.as_ref() { + user_env.extend(template.environment.clone()); + } + user_env.extend(spec.environment.clone()); + environment.extend(user_env.clone()); + if !user_env.is_empty() + && let Ok(json) = serde_json::to_string(&user_env) + { + environment.insert( + openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), + json, + ); + } + } + + environment.insert( + openshell_core::sandbox_env::ENDPOINT.to_string(), + config.grpc_endpoint.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SANDBOX_ID.to_string(), + sandbox.id.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SANDBOX.to_string(), + sandbox.name.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), + config.ssh_socket_path.clone(), + ); + environment.insert( + openshell_core::sandbox_env::SANDBOX_COMMAND.to_string(), + SANDBOX_COMMAND.to_string(), + ); + environment.insert( + openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), + openshell_core::telemetry::enabled_env_value().to_string(), + ); + // The root supervisor executes namespace helpers during bootstrap; keep + // their search path driver-owned even when the template/spec set PATH. + environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); + if config.guest_tls.is_some() { + environment.insert( + openshell_core::sandbox_env::TLS_CA.to_string(), + TLS_CA_MOUNT_PATH.to_string(), + ); + environment.insert( + openshell_core::sandbox_env::TLS_CERT.to_string(), + TLS_CERT_MOUNT_PATH.to_string(), + ); + environment.insert( + openshell_core::sandbox_env::TLS_KEY.to_string(), + TLS_KEY_MOUNT_PATH.to_string(), + ); + } + + environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); + environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + + // Gateway-minted sandbox JWT. Keep the raw bearer out of container + // metadata; the supervisor reads it from this driver-owned bind mount. + if let Some(spec) = sandbox.spec.as_ref() + && !spec.sandbox_token.is_empty() + { + environment.insert( + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), + SANDBOX_TOKEN_MOUNT_PATH.to_string(), + ); + } + + let mut pairs = environment.into_iter().collect::>(); + pairs.sort_by(|left, right| left.0.cmp(&right.0)); + pairs + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect() +} + +fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { + CdiGpuInventory::new( + info.discovered_devices + .as_deref() + .unwrap_or_default() + .iter() + .filter(|device| device.source.as_deref() == Some("cdi")) + .filter_map(|device| device.id.as_deref()), + ) +} + +fn docker_info_reports_wsl2(info: &SystemInfo) -> bool { + [ + info.kernel_version.as_deref(), + info.operating_system.as_deref(), + ] + .into_iter() + .flatten() + .any(os_or_kernel_reports_wsl2) +} + +fn os_or_kernel_reports_wsl2(value: &str) -> bool { + let value = value.to_ascii_lowercase(); + value.contains("wsl2") || value.contains("microsoft-standard") +} + +fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status { + Status::failed_precondition(err.to_string()) +} + +#[cfg(test)] +fn build_container_create_body( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let template = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let driver_config = docker_driver_config(template, config.enable_bind_mounts)?; + let gpu_requirements = sandbox + .spec + .as_ref() + .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); + let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() { + validate_specific_gpu_device_request( + gpu_requirements, + cdi_devices, + "driver_config.cdi_devices", + ) + .map_err(Status::invalid_argument)?; + Some(cdi_devices.as_slice()) + } else { + None + }; + build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) +} + +fn build_container_create_body_with_gpu_devices( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + driver_config: &DockerSandboxDriverConfig, + gpu_device_ids: Option<&[String]>, +) -> Result { + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; + let template = spec + .template + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let resource_limits = docker_resource_limits(template)?; + let user_mounts = docker_driver_mounts(driver_config)?; + let user_bind_strings = docker_driver_bind_strings(driver_config)?; + let device_requests = gpu_device_ids.map(|device_ids| { + vec![DeviceRequest { + driver: Some("cdi".to_string()), + device_ids: Some(device_ids.to_vec()), + ..Default::default() + }] + }); + let mut labels = template.labels.clone(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); + labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); + labels.insert( + LABEL_SANDBOX_WORKSPACE.to_string(), + sandbox.workspace.clone(), + ); + // The list/get/find paths filter by `config.sandbox_namespace`, so use + // the same value here. `DriverSandbox.namespace` is unset on the request + // path (the gateway elides it), and using it would produce containers + // that the driver itself cannot find afterwards. + labels.insert( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ); + + Ok(ContainerCreateBody { + image: Some(template.image.clone()), + user: Some("0".to_string()), + env: Some(build_environment(sandbox, config)), + entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), + // Clear the image CMD so Docker does not append inherited args to the + // supervisor entrypoint. + cmd: Some(Vec::new()), + labels: Some(labels), + host_config: Some(HostConfig { + nano_cpus: resource_limits.nano_cpus, + memory: resource_limits.memory_bytes, + pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, + device_requests, + binds: { + let mut binds = build_binds(sandbox, config)?; + binds.extend(user_bind_strings); + Some(binds) + }, + mounts: Some(user_mounts), + restart_policy: Some(RestartPolicy { + name: Some(RestartPolicyNameEnum::UNLESS_STOPPED), + maximum_retry_count: None, + }), + cap_add: Some(vec![ + "SYS_ADMIN".to_string(), + "NET_ADMIN".to_string(), + "SYS_PTRACE".to_string(), + "SYSLOG".to_string(), + ]), + // The sandbox supervisor needs to bind-mount `/run/netns`, + // mark it shared, and create per-process network namespaces. + // Docker's default AppArmor profile (`docker-default`) denies + // these mount operations even with CAP_SYS_ADMIN, so we opt + // out of AppArmor confinement for sandbox containers. The + // sandbox enforces its own security boundary via Landlock, + // seccomp, OPA policy evaluation, and the dedicated network + // namespace it sets up for the agent — AppArmor at the + // container layer is redundant relative to those controls + // and conflicts with them in this case. + security_opt: Some(vec!["apparmor=unconfined".to_string()]), + network_mode: Some(config.network_name.clone()), + extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), + ..Default::default() + }), + networking_config: Some(NetworkingConfig { + endpoints_config: Some(HashMap::from([( + config.network_name.clone(), + EndpointSettings::default(), + )])), + }), + ..Default::default() + }) +} + +/// Reject driver requests that arrive with neither a sandbox id nor a +/// sandbox name. Without this guard, downstream label filters degenerate +/// to "match every managed container in the namespace", which would let +/// `delete_sandbox`/`stop_sandbox`/`get_sandbox` pick an arbitrary +/// sandbox out of the set the driver manages. +fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { + if sandbox_id.is_empty() && sandbox_name.is_empty() { + return Err(Status::invalid_argument( + "sandbox_id or sandbox_name is required", + )); + } + Ok(()) +} + +fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { + let Ok(mut url) = Url::parse(endpoint) else { + return endpoint.to_string(); + }; + + if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { + return url.to_string(); + } + + endpoint.to_string() +} + +fn docker_network_name(config: &DockerComputeConfig) -> String { + let name = config.network_name.trim(); + if name.is_empty() { + return DEFAULT_DOCKER_NETWORK_NAME.to_string(); + } + name.to_string() +} + +fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Ok(None); + } + + trimmed + .parse() + .map(Some) + .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) +} + +fn docker_gateway_route( + info: &SystemInfo, + bridge_gateway_ip: IpAddr, + port: u16, + host_gateway_ip: Option, +) -> DockerGatewayRoute { + docker_gateway_route_for_host( + info, + bridge_gateway_ip, + port, + host_gateway_ip, + host_runtime_requires_host_gateway_alias(), + ) +} + +fn docker_gateway_route_for_host( + info: &SystemInfo, + bridge_gateway_ip: IpAddr, + port: u16, + host_gateway_ip: Option, + host_requires_host_gateway_alias: bool, +) -> DockerGatewayRoute { + if let Some(host_alias_ip) = host_gateway_ip { + return DockerGatewayRoute::Bridge { + bind_address: SocketAddr::new(host_alias_ip, port), + host_alias_ip, + }; + } + + if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { + DockerGatewayRoute::HostGateway + } else { + DockerGatewayRoute::Bridge { + bind_address: SocketAddr::new(bridge_gateway_ip, port), + host_alias_ip: bridge_gateway_ip, + } + } +} + +fn host_runtime_requires_host_gateway_alias() -> bool { + cfg!(target_os = "macos") +} + +/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, +/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing +/// constraint: the bridge gateway IP is reachable from inside containers but +/// not from the `OpenShell` server process running on the host, so callbacks +/// must traverse `host-gateway`. +/// +/// Each runtime is detected via the daemon's reported OS string or hostname, +/// supplemented by labels where the runtime publishes them. +fn uses_host_gateway_alias(info: &SystemInfo) -> bool { + let operating_system = info + .operating_system + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase(); + if operating_system.contains("docker desktop") { + return true; + } + + let name = info + .name + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase(); + if name.starts_with("colima") + || name.starts_with("lima-") + || name.starts_with("rancher-desktop") + || name.starts_with("orbstack") + { + return true; + } + + info.labels.as_ref().is_some_and(|labels| { + labels.iter().any(|label| { + label.starts_with("com.docker.desktop.") + || label.starts_with("dev.rancherdesktop.") + || label.starts_with("dev.orbstack.") + }) + }) +} + +fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { + match route { + DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ + format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), + format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), + ], + DockerGatewayRoute::HostGateway => vec![ + format!("{HOST_DOCKER_INTERNAL}:host-gateway"), + format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), + ], + } +} + +async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { + match docker.inspect_network(network_name, None).await { + Ok(network) => return validate_bridge_network(network_name, &network), + Err(err) if !is_not_found_error(&err) => { + return Err(Error::execution(format!( + "failed to inspect Docker network '{network_name}': {err}" + ))); + } + Err(_) => {} + } + + docker + .create_network(NetworkCreateRequest { + name: network_name.to_string(), + driver: Some(DOCKER_NETWORK_DRIVER.to_string()), + attachable: Some(true), + labels: Some(HashMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }) + .await + .map(|_| ()) + .or_else(|err| { + if is_conflict_error(&err) { + Ok(()) + } else { + Err(Error::execution(format!( + "failed to create Docker network '{network_name}': {err}" + ))) + } + })?; + + let network = docker + .inspect_network(network_name, None) + .await + .map_err(|err| { + Error::execution(format!( + "failed to inspect Docker network '{network_name}' after create: {err}" + )) + })?; + validate_bridge_network(network_name, &network) +} + +fn validate_bridge_network( + network_name: &str, + network: &bollard::models::NetworkInspect, +) -> CoreResult { + if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { + return Err(Error::config(format!( + "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", + network.driver.as_deref().unwrap_or("unknown") + ))); + } + + docker_bridge_gateway_ip(network_name, network) +} + +fn docker_bridge_gateway_ip( + network_name: &str, + network: &bollard::models::NetworkInspect, +) -> CoreResult { + let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { + return Err(Error::config(format!( + "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" + ))); + }; + + for config in configs { + let Some(gateway) = config.gateway.as_deref() else { + continue; + }; + let ip = gateway.parse::().map_err(|err| { + Error::config(format!( + "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" + )) + })?; + if matches!(ip, IpAddr::V4(_)) { + return Ok(ip); + } + } + + Err(Error::config(format!( + "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" + ))) +} + +fn docker_resource_limits( + template: &DriverSandboxTemplate, +) -> Result { + let Some(resources) = template.resources.as_ref() else { + return Ok(DockerResourceLimits::default()); + }; + + if !resources.cpu_request.trim().is_empty() { + return Err(Status::failed_precondition( + "docker compute driver does not support resources.requests.cpu", + )); + } + if !resources.memory_request.trim().is_empty() { + return Err(Status::failed_precondition( + "docker compute driver does not support resources.requests.memory", + )); + } + + Ok(DockerResourceLimits { + nano_cpus: parse_cpu_limit(&resources.cpu_limit)?, + memory_bytes: parse_memory_limit(&resources.memory_limit)?, + }) +} + +fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { + if value < 0 { + return Err(Error::config( + "docker sandbox_pids_limit must be zero or greater", + )); + } + Ok(()) +} + +fn docker_pids_limit(value: i64) -> Result, Status> { + if value < 0 { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be zero or greater", + )); + } + if value == 0 { + Ok(None) + } else { + Ok(Some(value)) + } +} + +#[allow(clippy::cast_possible_truncation)] +fn parse_cpu_limit(value: &str) -> Result, Status> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + if let Some(millicores) = value.strip_suffix('m') { + let millicores = millicores.parse::().map_err(|_| { + Status::failed_precondition(format!( + "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", + )) + })?; + if millicores <= 0 { + return Err(Status::failed_precondition( + "docker cpu_limit must be greater than zero", + )); + } + return Ok(Some(millicores.saturating_mul(1_000_000))); + } + + let cores = value.parse::().map_err(|_| { + Status::failed_precondition(format!( + "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", + )) + })?; + if !cores.is_finite() || cores <= 0.0 { + return Err(Status::failed_precondition( + "docker cpu_limit must be greater than zero", + )); + } + + Ok(Some((cores * 1_000_000_000.0).round() as i64)) +} + +#[allow(clippy::cast_possible_truncation)] +fn parse_memory_limit(value: &str) -> Result, Status> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + + let number_end = value + .find(|ch: char| !(ch.is_ascii_digit() || ch == '.')) + .unwrap_or(value.len()); + let (number, suffix) = value.split_at(number_end); + let amount = number.parse::().map_err(|_| { + Status::failed_precondition(format!( + "invalid docker memory_limit '{value}'; expected a Kubernetes-style quantity", + )) + })?; + if !amount.is_finite() || amount <= 0.0 { + return Err(Status::failed_precondition( + "docker memory_limit must be greater than zero", + )); + } + + let multiplier = match suffix { + "" => 1_f64, + "Ki" => 1024_f64, + "Mi" => 1024_f64.powi(2), + "Gi" => 1024_f64.powi(3), + "Ti" => 1024_f64.powi(4), + "Pi" => 1024_f64.powi(5), + "Ei" => 1024_f64.powi(6), + "K" => 1000_f64, + "M" => 1000_f64.powi(2), + "G" => 1000_f64.powi(3), + "T" => 1000_f64.powi(4), + "P" => 1000_f64.powi(5), + "E" => 1000_f64.powi(6), + _ => { + return Err(Status::failed_precondition(format!( + "invalid docker memory_limit suffix '{suffix}'", + ))); + } + }; + + Ok(Some((amount * multiplier).round() as i64)) +} + +fn sandbox_from_container_summary( + summary: &ContainerSummary, + readiness: &dyn SupervisorReadiness, +) -> Option { + let labels = summary.labels.as_ref()?; + let id = labels.get(LABEL_SANDBOX_ID)?.clone(); + let name = labels.get(LABEL_SANDBOX_NAME)?.clone(); + let namespace = labels + .get(LABEL_SANDBOX_NAMESPACE) + .cloned() + .unwrap_or_default(); + let workspace = labels + .get(LABEL_SANDBOX_WORKSPACE) + .cloned() + .unwrap_or_default(); + + let supervisor_connected = readiness.is_supervisor_connected(&id); + Some(DriverSandbox { + id, + name: name.clone(), + namespace, + spec: None, + status: Some(driver_status_from_summary( + summary, + &name, + supervisor_connected, + )), + workspace, + }) +} + +fn driver_status_from_summary( + summary: &ContainerSummary, + sandbox_name: &str, + supervisor_connected: bool, +) -> DriverSandboxStatus { + let state = summary.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); + let (ready, reason, message, deleting) = container_ready_condition(state, supervisor_connected); + + DriverSandboxStatus { + sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), + instance_id: summary.id.clone().unwrap_or_default(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![DriverCondition { + r#type: "Ready".to_string(), + status: ready.to_string(), + reason: reason.to_string(), + message: message.to_string(), + last_transition_time: String::new(), + }], + deleting, + } +} + +fn container_ready_condition( + state: ContainerSummaryStateEnum, + supervisor_connected: bool, +) -> (&'static str, &'static str, &'static str, bool) { + match state { + ContainerSummaryStateEnum::RUNNING => { + if supervisor_connected { + ( + "True", + "SupervisorConnected", + "Supervisor relay is live", + false, + ) + } else { + ( + "False", + "DependenciesNotReady", + "Container is running; waiting for supervisor relay", + false, + ) + } + } + ContainerSummaryStateEnum::CREATED => ("False", "Starting", "Container created", false), + ContainerSummaryStateEnum::RESTARTING => ( + "False", + "ContainerRestarting", + "Container is restarting after a failure", + false, + ), + ContainerSummaryStateEnum::EMPTY => { + ("False", "Starting", "Container state is unknown", false) + } + ContainerSummaryStateEnum::REMOVING => { + ("False", "Deleting", "Container is being removed", true) + } + ContainerSummaryStateEnum::PAUSED => { + ("False", "ContainerPaused", "Container is paused", false) + } + ContainerSummaryStateEnum::EXITED => { + ("False", "ContainerExited", "Container exited", false) + } + ContainerSummaryStateEnum::DEAD => ("False", "ContainerDead", "Container is dead", false), + } +} + +fn summary_container_name(summary: &ContainerSummary) -> Option { + summary + .names + .as_ref() + .and_then(|names| names.first()) + .map(|name| name.trim_start_matches('/').to_string()) + .filter(|name| !name.is_empty()) +} + +fn summary_container_target(summary: &ContainerSummary) -> Option { + // Prefer the container ID: it's stable while the container exists and is + // accepted by Docker APIs just like a name. Fall back to the parsed name + // for transient summaries that do not include an ID. + summary + .id + .as_deref() + .filter(|id| !id.is_empty()) + .map(str::to_string) + .or_else(|| summary_container_name(summary)) +} + +fn container_state_needs_shutdown_stop(state: ContainerSummaryStateEnum) -> bool { + matches!( + state, + ContainerSummaryStateEnum::RUNNING + | ContainerSummaryStateEnum::RESTARTING + | ContainerSummaryStateEnum::PAUSED + ) +} + +/// States from which a managed container can be brought back to running by +/// `start_container`. Skip `Restarting` (already coming up), `Removing`, +/// `Dead` (terminal), `Paused` (needs `unpause`, not `start`), and +/// `Running` (nothing to do). +fn container_state_needs_resume(state: ContainerSummaryStateEnum) -> bool { + matches!( + state, + ContainerSummaryStateEnum::EXITED | ContainerSummaryStateEnum::CREATED + ) +} + +fn docker_stop_timeout_secs(timeout_secs: u32) -> i32 { + i32::try_from(timeout_secs).unwrap_or(i32::MAX) +} + +fn emit_snapshot_diff( + events: &broadcast::Sender, + previous: &HashMap, + current: &HashMap, +) { + for (sandbox_id, sandbox) in current { + if previous.get(sandbox_id) == Some(sandbox) { + continue; + } + let _ = events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox.clone()), + }, + )), + }); + } + + for sandbox_id in previous.keys() { + if current.contains_key(sandbox_id) { + continue; + } + let _ = events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { + sandbox_id: sandbox_id.clone(), + }, + )), + }); + } +} + +fn label_filters(values: impl IntoIterator) -> HashMap> { + HashMap::from([("label".to_string(), values.into_iter().collect())]) +} + +fn managed_container_label_filters( + sandbox_namespace: &str, + extra_values: impl IntoIterator, +) -> HashMap> { + let mut values = vec![ + format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), + ]; + values.extend(extra_values); + label_filters(values) +} + +/// Maximum Docker container name length. Docker's own limit is 253 bytes, but +/// we cap at a conservative 200 to leave headroom for tooling that truncates +/// names further. +const MAX_CONTAINER_NAME_LEN: usize = 200; +const CONTAINER_NAME_PREFIX: &str = "openshell-"; + +fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String { + let id_suffix = sanitize_docker_name(&sandbox.id); + let workspace = sanitize_docker_name(&sandbox.workspace); + let name = sanitize_docker_name(&sandbox.name); + + // Format: openshell-{workspace}--{name}-{id} + // The workspace and id are never truncated — they ensure uniqueness. + // Only the sandbox name portion is truncated when the total exceeds + // MAX_CONTAINER_NAME_LEN. + + if name.is_empty() { + let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); + if base.len() > MAX_CONTAINER_NAME_LEN { + base.truncate(MAX_CONTAINER_NAME_LEN); + } + return trim_container_name_tail(base); + } + + // Reserve space for fixed parts: prefix + workspace + "--" + "-" + id + let reserved = CONTAINER_NAME_PREFIX.len() + workspace.len() + 2 + 1 + id_suffix.len(); + if reserved >= MAX_CONTAINER_NAME_LEN { + let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); + base.truncate(MAX_CONTAINER_NAME_LEN); + return trim_container_name_tail(base); + } + + let name_budget = MAX_CONTAINER_NAME_LEN - reserved; + let truncated_name = if name.len() > name_budget { + trim_container_name_tail(name[..name_budget].to_string()) + } else { + name + }; + format!("{CONTAINER_NAME_PREFIX}{workspace}--{truncated_name}-{id_suffix}") +} + +/// Docker container names may not end with `-`, `.`, or `_`. Truncation can +/// leave one of those trailing, so strip them before returning. +fn trim_container_name_tail(mut value: String) -> String { + while value + .chars() + .last() + .is_some_and(|ch| matches!(ch, '-' | '.' | '_')) + { + value.pop(); + } + value +} + +fn sanitize_docker_name(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '-') { + ch.to_ascii_lowercase() + } else { + '-' + } + }) + .collect::() + .trim_matches('-') + .to_string() +} + +fn normalize_docker_arch(arch: &str) -> String { + match arch { + "x86_64" => "amd64".to_string(), + "aarch64" => "arm64".to_string(), + other => other.to_ascii_lowercase(), + } +} + +#[derive(Debug, Eq, PartialEq)] +enum SupervisorBinSource { + Binary(PathBuf), + Image(String), +} + +fn resolve_supervisor_bin_source( + docker_config: &DockerComputeConfig, + current_exe: Option<&Path>, + target_candidates: &[PathBuf], +) -> CoreResult { + // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. + if let Some(path) = docker_config.supervisor_bin.clone() { + let path = canonicalize_existing_file(&path, "docker supervisor binary")?; + validate_linux_elf_binary(&path)?; + return Ok(SupervisorBinSource::Binary(path)); + } + + // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. + // A configured image should be the source of truth even when a local + // developer build is present under target/. + if let Some(image) = docker_config.supervisor_image.clone() { + return Ok(SupervisorBinSource::Image(image)); + } + + // Tier 3: sibling `openshell-sandbox` next to the running gateway + // (release artifact layout). Linux-only because the sibling must be a + // Linux ELF to bind-mount into a Linux container. + if cfg!(target_os = "linux") + && let Some(current_exe) = current_exe + && let Some(parent) = current_exe.parent() + { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; + if validate_linux_elf_binary(&path).is_ok() { + return Ok(SupervisorBinSource::Binary(path)); + } + } + } + + // Tier 4: local cargo target build (developer workflow). Preferred + // over the default registry image when available because it matches + // whatever the developer just built. + for candidate in target_candidates { + if candidate.is_file() { + let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; + if validate_linux_elf_binary(&path).is_ok() { + return Ok(SupervisorBinSource::Binary(path)); + } + } + } + + // Tier 5: pull the release-matched default supervisor image and extract + // the binary to a host-side cache keyed by image content digest. + Ok(SupervisorBinSource::Image( + openshell_core::config::default_supervisor_image(), + )) +} + +pub(crate) async fn resolve_supervisor_bin( + docker: &Docker, + docker_config: &DockerComputeConfig, + daemon_arch: &str, +) -> CoreResult { + let current_exe = + if cfg!(target_os = "linux") + && docker_config.supervisor_bin.is_none() + && docker_config.supervisor_image.is_none() + { + Some(std::env::current_exe().map_err(|err| { + Error::config(format!("failed to resolve current executable: {err}")) + })?) + } else { + None + }; + let target_candidates = linux_supervisor_candidates(daemon_arch); + + match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? + { + SupervisorBinSource::Binary(path) => Ok(path), + SupervisorBinSource::Image(image) => { + extract_supervisor_bin_from_image(docker, &image).await + } + } +} + +fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { + match daemon_arch { + "arm64" => vec![PathBuf::from( + "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", + )], + "amd64" => vec![PathBuf::from( + "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", + )], + _ => Vec::new(), + } +} + +/// Pull the supervisor image (if not already local), extract +/// `/openshell-sandbox` to a host cache keyed by the image's content +/// digest, and return the cache path. +/// +/// The extraction is atomic: the binary is written to a sibling temp file +/// inside the digest-keyed directory and renamed into place, so concurrent +/// gateway starts don't observe a partial file. +async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { + let refresh_attempted = if supervisor_image_should_refresh(image) { + info!(image = image, "Refreshing mutable docker supervisor image"); + match pull_supervisor_image(docker, image).await { + Ok(()) => true, + Err(err) => { + warn!( + image = image, + error = %err, + "failed to refresh mutable docker supervisor image; falling back to local image if present", + ); + true + } + } + } else { + false + }; + + // Inspect first to see if the image is already present; only pull on miss. + let inspect = match docker.inspect_image(image).await { + Ok(inspect) => inspect, + Err(err) if is_not_found_error(&err) && !refresh_attempted => { + info!(image = image, "Pulling docker supervisor image"); + pull_supervisor_image(docker, image).await?; + docker.inspect_image(image).await.map_err(|err| { + Error::config(format!( + "failed to inspect docker supervisor image '{image}' after pull: {err}", + )) + })? + } + Err(err) if is_not_found_error(&err) => { + return Err(Error::config(format!( + "docker supervisor image '{image}' is not present locally after refresh attempt", + ))); + } + Err(err) => { + return Err(Error::config(format!( + "failed to inspect docker supervisor image '{image}': {err}", + ))); + } + }; + + let digest = inspect.id.clone().ok_or_else(|| { + Error::config(format!( + "docker supervisor image '{image}' inspect response has no Id", + )) + })?; + + let cache_path = supervisor_cache_path(&digest)?; + if cache_path.is_file() { + validate_linux_elf_binary(&cache_path)?; + return Ok(cache_path); + } + + let cache_dir = cache_path.parent().ok_or_else(|| { + Error::config(format!( + "docker supervisor cache path '{}' has no parent directory", + cache_path.display(), + )) + })?; + std::fs::create_dir_all(cache_dir).map_err(|err| { + Error::config(format!( + "failed to create docker supervisor cache dir '{}': {err}", + cache_dir.display(), + )) + })?; + + info!( + image = image, + digest = digest, + cache_path = %cache_path.display(), + "Extracting supervisor binary from image to host cache", + ); + + let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; + write_cache_binary_atomic(&cache_path, &binary_bytes)?; + validate_linux_elf_binary(&cache_path)?; + Ok(cache_path) +} + +async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { + let mut stream = docker.create_image( + Some(CreateImageOptions { + from_image: Some(image.to_string()), + ..Default::default() + }), + None, + None, + ); + while let Some(result) = stream.next().await { + result.map_err(|err| { + Error::config(format!( + "failed to pull docker supervisor image '{image}': {err}", + )) + })?; + } + Ok(()) +} + +/// Create a short-lived container from `image`, stream out the supervisor +/// binary as a tar archive, and return the untarred file bytes. The +/// container is always removed, even on error paths. +async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + let container_name = temp_extract_container_name(); + docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + ContainerCreateBody { + image: Some(image.to_string()), + entrypoint: Some(vec![SUPERVISOR_IMAGE_BINARY_PATH.to_string()]), + cmd: Some(Vec::new()), + ..Default::default() + }, + ) + .await + .map_err(|err| { + Error::config(format!( + "failed to create extractor container from '{image}': {err}", + )) + })?; + + // Always tear down the extractor container, even if extraction fails. + let result = download_binary_from_container(docker, &container_name).await; + if let Err(remove_err) = docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + warn!( + container = container_name, + error = %remove_err, + "Failed to remove supervisor extractor container", + ); + } + result +} + +async fn download_binary_from_container( + docker: &Docker, + container_name: &str, +) -> CoreResult> { + let options = DownloadFromContainerOptionsBuilder::default() + .path(SUPERVISOR_IMAGE_BINARY_PATH) + .build(); + let mut stream = docker.download_from_container(container_name, Some(options)); + + let mut tar_bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk: Bytes = chunk.map_err(|err| { + Error::config(format!( + "failed to read supervisor binary stream from '{container_name}': {err}", + )) + })?; + tar_bytes.extend_from_slice(&chunk); + } + + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) +} + +/// Extract the payload of the first regular-file entry in a tar archive. +/// Docker's `/containers//archive` endpoint returns a single-file tar +/// when `path` points to a file, so we only need the first entry. +fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { + let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); + let mut entries = archive + .entries() + .map_err(|err| format!("open tar archive: {err}"))?; + let mut entry = entries + .next() + .ok_or_else(|| "tar archive was empty".to_string())? + .map_err(|err| format!("read tar entry: {err}"))?; + let mut bytes = Vec::new(); + entry + .read_to_end(&mut bytes) + .map_err(|err| format!("read tar entry payload: {err}"))?; + Ok(bytes) +} + +fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { + let dir = final_path.parent().ok_or_else(|| { + Error::config(format!( + "docker supervisor cache path '{}' has no parent directory", + final_path.display(), + )) + })?; + let mut temp = tempfile::Builder::new() + .prefix(".openshell-sandbox-") + .tempfile_in(dir) + .map_err(|err| { + Error::config(format!( + "failed to create temp file for supervisor binary in '{}': {err}", + dir.display(), + )) + })?; + std::io::Write::write_all(&mut temp, bytes).map_err(|err| { + Error::config(format!( + "failed to write supervisor binary to temp file: {err}", + )) + })?; + temp.as_file().sync_all().map_err(|err| { + Error::config(format!("failed to sync supervisor binary temp file: {err}")) + })?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( + |err| { + Error::config(format!( + "failed to chmod supervisor binary temp file: {err}", + )) + }, + )?; + } + + temp.persist(final_path).map_err(|err| { + Error::config(format!( + "failed to rename supervisor binary into '{}': {}", + final_path.display(), + err.error, + )) + })?; + Ok(()) +} + +/// Cache path for an extracted supervisor binary, keyed by the image's +/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed +/// directory keeps stale extractions from earlier releases isolated so they +/// can be GC'd without affecting the active binary. +fn supervisor_cache_path(digest: &str) -> CoreResult { + let base = openshell_core::paths::xdg_data_dir() + .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; + Ok(supervisor_cache_path_with_base(&base, digest)) +} + +fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { + let sanitized = digest.replace(':', "-"); + base.join("openshell") + .join("docker-supervisor") + .join(sanitized) + .join("openshell-sandbox") +} + +fn temp_extract_container_name() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let pid = std::process::id(); + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + format!("openshell-supervisor-extract-{pid}-{seq}") +} + +fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { + if !path.is_file() { + return Err(Error::config(format!( + "{description} '{}' does not exist or is not a file", + path.display() + ))); + } + std::fs::canonicalize(path).map_err(|err| { + Error::config(format!( + "failed to resolve {description} '{}': {err}", + path.display() + )) + }) +} + +pub(crate) fn validate_linux_elf_binary(path: &Path) -> CoreResult<()> { + let mut file = std::fs::File::open(path).map_err(|err| { + Error::config(format!( + "failed to open docker supervisor binary '{}': {err}", + path.display() + )) + })?; + let mut magic = [0_u8; 4]; + file.read_exact(&mut magic).map_err(|err| { + Error::config(format!( + "failed to read docker supervisor binary '{}': {err}", + path.display() + )) + })?; + if magic != [0x7f, b'E', b'L', b'F'] { + return Err(Error::config(format!( + "docker supervisor binary '{}' must be a Linux ELF executable", + path.display() + ))); + } + Ok(()) +} + +fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { + docker_config.guest_tls_ca.is_some() + && docker_config.guest_tls_cert.is_some() + && docker_config.guest_tls_key.is_some() +} + +pub(crate) fn docker_guest_tls_paths( + docker_config: &DockerComputeConfig, +) -> CoreResult> { + let tls_flags_provided = docker_config.guest_tls_ca.is_some() + || docker_config.guest_tls_cert.is_some() + || docker_config.guest_tls_key.is_some(); + + if !docker_config.grpc_endpoint.starts_with("https://") { + if tls_flags_provided { + return Err(Error::config(format!( + "guest_tls_ca/guest_tls_cert/guest_tls_key were provided but grpc_endpoint is '{}'; TLS materials require an https:// endpoint", + docker_config.grpc_endpoint, + ))); + } + return Ok(None); + } + + let provided = [ + docker_config.guest_tls_ca.as_ref(), + docker_config.guest_tls_cert.as_ref(), + docker_config.guest_tls_key.as_ref(), + ]; + if provided.iter().all(Option::is_none) { + return Err(Error::config( + "docker compute driver requires guest_tls_ca, guest_tls_cert, and guest_tls_key when grpc_endpoint uses https://", + )); + } + + let Some(ca) = docker_config.guest_tls_ca.clone() else { + return Err(Error::config( + "guest_tls_ca is required when Docker sandbox TLS materials are configured", + )); + }; + let Some(cert) = docker_config.guest_tls_cert.clone() else { + return Err(Error::config( + "guest_tls_cert is required when Docker sandbox TLS materials are configured", + )); + }; + let Some(key) = docker_config.guest_tls_key.clone() else { + return Err(Error::config( + "guest_tls_key is required when Docker sandbox TLS materials are configured", + )); + }; + + Ok(Some(DockerGuestTlsPaths { + ca: canonicalize_existing_file(&ca, "docker TLS CA certificate")?, + cert: canonicalize_existing_file(&cert, "docker TLS client certificate")?, + key: canonicalize_existing_file(&key, "docker TLS client private key")?, + })) +} + +fn is_not_found_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 404, + .. + } + ) +} + +fn is_conflict_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 409, + .. + } + ) +} + +fn is_not_modified_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 304, + .. + } + ) +} + +fn create_status_from_docker_error(operation: &str, err: BollardError) -> Status { + if matches!( + err, + BollardError::DockerResponseServerError { + status_code: 409, + .. + } + ) { + Status::already_exists("sandbox already exists") + } else { + internal_status(operation, err) + } +} + +fn internal_status(operation: &str, err: BollardError) -> Status { + Status::internal(format!("{operation} failed: {err}")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-driver-docker/src/lib_entry.rs b/crates/openshell-driver-docker/src/lib_entry.rs new file mode 100644 index 0000000000..4272e7c537 --- /dev/null +++ b/crates/openshell-driver-docker/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-driver-docker/src/lib_unix.rs b/crates/openshell-driver-docker/src/lib_unix.rs deleted file mode 100644 index 2f89c2229e..0000000000 --- a/crates/openshell-driver-docker/src/lib_unix.rs +++ /dev/null @@ -1,3522 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Docker compute driver. - -#![allow(clippy::result_large_err)] - -use bollard::Docker; -use bollard::errors::Error as BollardError; -use bollard::models::{ - ContainerCreateBody, ContainerSummary, ContainerSummaryStateEnum, CreateImageInfo, - DeviceRequest, EndpointSettings, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, - MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, ProgressDetail, RestartPolicy, - RestartPolicyNameEnum, SystemInfo, -}; -use bollard::query_parameters::{ - CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, - ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, -}; -use bytes::Bytes; -use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; -use openshell_core::driver_mounts; -use openshell_core::driver_utils::{ - LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - supervisor_image_should_refresh, -}; -use openshell_core::gpu::{ - CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, - effective_driver_gpu_count, validate_specific_gpu_device_request, -}; -use openshell_core::progress::{ - PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX, - format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail, -}; -use openshell_core::proto::compute::v1::{ - CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, watch_sandboxes_event, -}; -use openshell_core::proto_struct::{ - deserialize_optional_non_empty_string_list, struct_to_json_value, -}; -use openshell_core::{Config, Error, Result as CoreResult}; -use std::collections::{HashMap, HashSet}; -use std::io::Read; -use std::net::{IpAddr, SocketAddr}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; -use tokio::task::JoinHandle; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status}; -use tracing::{info, warn}; -use url::Url; - -const WATCH_BUFFER: usize = 128; -const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); -const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); - -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const SANDBOX_COMMAND: &str = "sleep infinity"; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; -const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; -const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; -const DOCKER_NETWORK_DRIVER: &str = "bridge"; - -/// Queried by the Docker driver to decide when a sandbox's supervisor -/// relay is live. Implementations return `true` once a sandbox has an -/// active `ConnectSupervisor` session registered. -/// -/// The driver cannot observe the supervisor's SSH socket directly (it -/// lives inside the container), so it leans on this signal to flip the -/// Ready condition from `DependenciesNotReady` to `True`. -pub trait SupervisorReadiness: Send + Sync + 'static { - fn is_supervisor_connected(&self, sandbox_id: &str) -> bool; -} - -/// Gateway-local configuration for the Docker compute driver. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct DockerComputeConfig { - /// Docker API Unix socket. When unset, use the socket selected by gateway - /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly - /// configured Docker driver. - pub socket_path: Option, - - /// Default OCI image for sandboxes. - pub default_image: String, - - /// Image pull policy for sandbox images. - pub image_pull_policy: String, - - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, - - /// Gateway gRPC endpoint the sandbox connects back to. - pub grpc_endpoint: String, - - /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. - pub supervisor_bin: Option, - - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. - pub supervisor_image: Option, - - /// Host-side CA certificate for Docker sandbox mTLS. - pub guest_tls_ca: Option, - - /// Host-side client certificate for Docker sandbox mTLS. - pub guest_tls_cert: Option, - - /// Host-side private key for Docker sandbox mTLS. - pub guest_tls_key: Option, - - /// Docker bridge network that sandbox containers join. - pub network_name: String, - - /// Host gateway IP used for sandbox host aliases. - pub host_gateway_ip: String, - - /// Unix socket path the in-container supervisor bridges relay traffic to. - pub ssh_socket_path: String, - - /// Container cgroup PID limit for Docker-managed sandboxes. - /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, - - /// Allow sandbox requests to attach host bind mounts through - /// `template.driver_config`. - #[serde(default)] - pub enable_bind_mounts: bool, -} - -impl Default for DockerComputeConfig { - fn default() -> Self { - Self { - socket_path: None, - default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), - grpc_endpoint: String::new(), - supervisor_bin: None, - supervisor_image: None, - guest_tls_ca: None, - guest_tls_cert: None, - guest_tls_key: None, - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - host_gateway_ip: String::new(), - ssh_socket_path: "/run/openshell/ssh.sock".to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, - enable_bind_mounts: false, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct DockerGuestTlsPaths { - pub(crate) ca: PathBuf, - pub(crate) cert: PathBuf, - pub(crate) key: PathBuf, -} - -#[derive(Debug, Clone)] -struct DockerDriverRuntimeConfig { - default_image: String, - image_pull_policy: String, - sandbox_namespace: String, - grpc_endpoint: String, - network_name: String, - gateway_route: DockerGatewayRoute, - ssh_socket_path: String, - stop_timeout_secs: u32, - log_level: String, - supervisor_bin: PathBuf, - guest_tls: Option, - daemon_version: String, - supports_gpu: bool, - allow_all_default_gpu: bool, - sandbox_pids_limit: i64, - enable_bind_mounts: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, - HostGateway, -} - -#[derive(Clone)] -pub struct DockerComputeDriver { - docker: Arc, - config: DockerDriverRuntimeConfig, - events: broadcast::Sender, - pending: Arc>>, - supervisor_readiness: Arc, - gpu_selector: Arc, -} - -struct PendingSandboxRecord { - sandbox: DriverSandbox, - task: Option>, -} - -#[derive(Debug, Clone)] -struct DockerProvisioningFailure { - reason: &'static str, - message: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -struct DockerResourceLimits { - nano_cpus: Option, - memory_bytes: Option, -} - -#[derive(Debug, Clone, Default, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -struct DockerSandboxDriverConfig { - #[serde( - default, - deserialize_with = "deserialize_optional_non_empty_string_list" - )] - cdi_devices: Option>, - mounts: Vec, -} - -struct ValidatedDockerSandbox<'a> { - template: &'a DriverSandboxTemplate, - driver_config: DockerSandboxDriverConfig, - gpu_requirements: Option<&'a GpuResourceRequirements>, -} - -impl DockerSandboxDriverConfig { - fn from_template(template: &DriverSandboxTemplate) -> Result { - let Some(config) = template.driver_config.as_ref() else { - return Ok(Self::default()); - }; - - serde_json::from_value(struct_to_json_value(config)) - .map_err(|err| format!("invalid docker driver_config: {err}")) - } -} - -use openshell_core::driver_mounts::SelinuxLabel; - -#[derive(Debug, Clone, serde::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum DockerDriverMountConfig { - Bind { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - selinux_label: Option, - }, - Volume { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, - Tmpfs { - target: String, - #[serde(default)] - options: Vec, - #[serde(default)] - size_bytes: Option, - #[serde(default)] - mode: Option, - }, - Image { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, -} - -fn default_true() -> bool { - true -} - -type WatchStream = - Pin> + Send + 'static>>; - -impl DockerComputeDriver { - pub async fn new( - config: &Config, - docker_config: &DockerComputeConfig, - supervisor_readiness: Arc, - ) -> CoreResult { - let socket_path = docker_config - .socket_path - .clone() - .or_else(openshell_core::config::detect_docker_socket) - .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); - let socket_path_str = socket_path.to_str().ok_or_else(|| { - Error::config(format!( - "Docker socket path is not valid UTF-8: {}", - socket_path.display() - )) - })?; - let docker = - Docker::connect_with_socket(socket_path_str, 120, bollard::API_DEFAULT_VERSION) - .map_err(|err| { - Error::execution(format!("failed to create Docker client: {err}")) - })?; - let version = docker.version().await.map_err(|err| { - Error::execution(format!("failed to query Docker daemon version: {err}")) - })?; - let info = docker.info().await.map_err(|err| { - Error::execution(format!("failed to query Docker daemon info: {err}")) - })?; - let supports_gpu = info - .cdi_spec_dirs - .as_ref() - .is_some_and(|dirs| !dirs.is_empty()); - let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); - let allow_all_default_gpu = docker_info_reports_wsl2(&info); - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - let gateway_port = config.bind_address.port(); - if gateway_port == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } - let network_name = docker_network_name(docker_config); - let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; - let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; - let gateway_route = - docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); - let mut docker_config = docker_config.clone(); - if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); - } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, - ); - let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); - let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; - let guest_tls = docker_guest_tls_paths(&docker_config)?; - - let driver = Self { - docker: Arc::new(docker), - config: DockerDriverRuntimeConfig { - default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), - grpc_endpoint, - network_name, - gateway_route, - ssh_socket_path: docker_config.ssh_socket_path.clone(), - stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, - log_level: config.log_level.clone(), - supervisor_bin, - guest_tls, - daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), - supports_gpu, - allow_all_default_gpu, - sandbox_pids_limit: docker_config.sandbox_pids_limit, - enable_bind_mounts: docker_config.enable_bind_mounts, - }, - events: broadcast::channel(WATCH_BUFFER).0, - pending: Arc::new(Mutex::new(HashMap::new())), - supervisor_readiness, - gpu_selector: Arc::new(CdiGpuDefaultSelector::new( - cdi_gpu_inventory, - allow_all_default_gpu, - )), - }; - - let poll_driver = driver.clone(); - tokio::spawn(async move { - poll_driver.poll_loop().await; - }); - - Ok(driver) - } - - #[must_use] - pub fn gateway_bind_addresses(&self) -> Vec { - match self.config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], - DockerGatewayRoute::HostGateway => Vec::new(), - } - } - - fn capabilities(&self) -> GetCapabilitiesResponse { - openshell_core::driver_utils::build_capabilities_response( - "docker", - &self.config.daemon_version, - &self.config.default_image, - ) - } - - #[cfg(test)] - fn validate_sandbox( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - ) -> Result<(), Status> { - let _ = Self::validated_sandbox(sandbox, config)?; - Ok(()) - } - - fn validated_sandbox<'a>( - sandbox: &'a DriverSandbox, - config: &DockerDriverRuntimeConfig, - ) -> Result, Status> { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - - Self::validate_sandbox_template_base(template)?; - let _ = docker_resource_limits(template)?; - let driver_config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; - let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); - Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?; - Ok(ValidatedDockerSandbox { - template, - driver_config, - gpu_requirements, - }) - } - - fn validate_sandbox_template_base(template: &DriverSandboxTemplate) -> Result<(), Status> { - if template.image.trim().is_empty() { - return Err(Status::failed_precondition( - "docker sandboxes require a template image", - )); - } - if !template.agent_socket_path.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support template.agent_socket_path", - )); - } - if template - .platform_config - .as_ref() - .is_some_and(|config| !config.fields.is_empty()) - { - return Err(Status::failed_precondition( - "docker compute driver does not support template.platform_config", - )); - } - - Ok(()) - } - - fn validate_sandbox_auth(sandbox: &DriverSandbox) -> Result<(), Status> { - let token_present = sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.trim().is_empty()); - if token_present { - return Ok(()); - } - - Err(Status::failed_precondition( - "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]", - )) - } - - fn validate_gpu_request( - gpu_requirements: Option<&GpuResourceRequirements>, - supports_gpu: bool, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - let requested_count = - effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)?; - if requested_count.is_some() && !supports_gpu { - return Err(Status::failed_precondition( - "docker GPU sandboxes require Docker CDI support. Enable CDI on the Docker daemon, then restart the OpenShell gateway/server so GPU capability is detected.", - )); - } - - if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - } - - Ok(()) - } - - async fn validate_user_volume_mounts_available( - &self, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - for mount in &driver_config.mounts { - if let DockerDriverMountConfig::Volume { source, .. } = mount { - match self.docker.inspect_volume(source).await { - Ok(volume) => { - if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) - { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]" - ))); - } - } - Err(err) if is_not_found_error(&err) => { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' does not exist" - ))); - } - Err(err) => { - return Err(internal_status("inspect docker volume", err)); - } - } - } - } - Ok(()) - } - - async fn refresh_gpu_inventory(&self) -> Result<(), Status> { - let info = self - .docker - .info() - .await - .map_err(|err| internal_status("query Docker daemon info", err))?; - self.gpu_selector.refresh( - docker_cdi_gpu_inventory(&info), - self.config.allow_all_default_gpu, - ); - Ok(()) - } - - async fn resolve_gpu_cdi_devices( - &self, - gpu_requirements: Option<&GpuResourceRequirements>, - driver_config: &DockerSandboxDriverConfig, - select_default_devices: fn( - &CdiGpuDefaultSelector, - u32, - ) -> Result, CdiGpuSelectionError>, - ) -> Result>, Status> { - if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - return Ok(Some(cdi_devices.to_vec())); - } - - let Some(count) = - effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)? - else { - return Ok(None); - }; - - self.refresh_gpu_inventory().await?; - select_default_devices(&self.gpu_selector, count) - .map(Some) - .map_err(docker_gpu_selection_status) - } - - async fn get_sandbox_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result, Status> { - let container = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await?; - if let Some(sandbox) = container.and_then(|summary| { - sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) - }) { - return Ok(Some(sandbox)); - } - - Ok(self.pending_snapshot(sandbox_id, sandbox_name).await) - } - - async fn current_snapshots(&self) -> Result, Status> { - let containers = self.list_managed_container_summaries().await?; - let container_sandboxes = containers - .iter() - .filter_map(|summary| { - sandbox_from_container_summary(summary, self.supervisor_readiness.as_ref()) - }) - .collect::>(); - let mut by_id = self.pending_snapshot_map().await; - for sandbox in container_sandboxes { - by_id.insert(sandbox.id.clone(), sandbox); - } - let mut sandboxes = by_id.into_values().collect::>(); - sandboxes.sort_by(|left, right| left.id.cmp(&right.id)); - Ok(sandboxes) - } - - async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { - let validated = Self::validated_sandbox(sandbox, &self.config)?; - Self::validate_sandbox_auth(sandbox)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; - let gpu_devices = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::peek_device_ids, - ) - .await?; - let _ = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - )?; - - if self - .find_managed_container_summary(&sandbox.id, &sandbox.name) - .await? - .is_some() - { - return Err(Status::already_exists("sandbox already exists")); - } - - self.reserve_pending_sandbox(sandbox).await?; - let image = sandbox_image(sandbox).unwrap_or_default(); - self.publish_docker_progress( - &sandbox.id, - "Scheduled", - format!("Docker sandbox accepted for image \"{image}\""), - HashMap::from([("image_ref".to_string(), image)]), - ); - self.publish_sandbox_snapshot(pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - provisioning_condition(), - false, - )); - - let driver = self.clone(); - let sandbox_for_task = sandbox.clone(); - let sandbox_id = sandbox.id.clone(); - let task = tokio::spawn(async move { - driver.provision_sandbox(sandbox_for_task).await; - }); - - let mut pending = self.pending.lock().await; - if let Some(record) = pending.get_mut(&sandbox_id) { - record.task = Some(task); - } else { - task.abort(); - } - - Ok(()) - } - - async fn provision_sandbox(&self, sandbox: DriverSandbox) { - match self.provision_sandbox_inner(&sandbox).await { - Ok(()) => { - self.clear_pending_sandbox(&sandbox.id).await; - } - Err(failure) => { - self.fail_pending_sandbox(&sandbox, &failure).await; - } - } - } - - async fn provision_sandbox_inner( - &self, - sandbox: &DriverSandbox, - ) -> Result<(), DockerProvisioningFailure> { - let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let template = validated.template; - self.ensure_image_available(&sandbox.id, &template.image) - .await - .map_err(|status| { - DockerProvisioningFailure::new("ImagePullFailed", status.message()) - })?; - let token_file_created = write_sandbox_token_file(sandbox, &self.config) - .await - .map_err(|status| { - DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) - })?; - - let container_name = container_name_for_sandbox(sandbox); - let gpu_devices = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::next_device_ids, - ) - .await - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let create_body = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - ) - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, - ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - })?; - self.publish_docker_progress( - &sandbox.id, - "Created", - format!("Created Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name.clone())]), - ); - - if let Err(err) = self.docker.start_container(&container_name, None).await { - let cleanup = self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await; - if let Err(cleanup_err) = cleanup { - warn!( - sandbox_id = %sandbox.id, - container_name, - error = %cleanup_err, - "Failed to clean up Docker container after start failure" - ); - } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - return Err(DockerProvisioningFailure::from_status( - "ContainerStartFailed", - create_status_from_docker_error("start docker sandbox container", err), - )); - } - self.publish_docker_progress( - &sandbox.id, - "Started", - format!("Started Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name)]), - ); - if let Err(err) = self - .publish_container_snapshot(&sandbox.id, &sandbox.name) - .await - { - warn!( - sandbox_id = %sandbox.id, - error = %err, - "Failed to publish Docker sandbox snapshot after start" - ); - } - - Ok(()) - } - - async fn delete_sandbox_inner( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; - if let Some(record) = pending.as_ref() - && let Some(task) = record.task.as_ref() - { - task.abort(); - } - - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = pending { - let container_name = container_name_for_sandbox(&record.sandbox); - match self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) => { - return Err(internal_status("delete docker sandbox container", err)); - } - } - } - return Ok(false); - }; - let Some(target) = summary_container_target(&container) else { - return Ok(pending.is_some()); - }; - - match self - .docker - .remove_container( - &target, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); - Ok(true) - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); - Ok(pending.is_some()) - } - Err(err) => Err(internal_status("delete docker sandbox container", err)), - } - } - - async fn stop_sandbox_inner(&self, sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { - if let Some(task) = record.task { - task.abort(); - } - cleanup_sandbox_token_file(&record.sandbox, &self.config); - self.publish_deleted(record.sandbox.id); - return Ok(()); - } - return Err(Status::not_found("sandbox not found")); - }; - let Some(target) = summary_container_target(&container) else { - return Err(Status::not_found("sandbox container has no id or name")); - }; - - match self - .docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) - .build(), - ), - ) - .await - { - Ok(()) => Ok(()), - Err(err) if is_not_modified_error(&err) => Ok(()), - Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), - Err(err) => Err(internal_status("stop docker sandbox container", err)), - } - } - - /// Start a managed sandbox container that was previously stopped. Used - /// by the gateway to resume sandboxes after a restart so that running - /// state in the gateway store is matched by an actually-running - /// container. - /// - /// Returns `Ok(true)` when a container existed and was started (or was - /// already running), `Ok(false)` when no managed container is found for - /// the sandbox, and `Err(...)` for any Docker failure. - pub async fn resume_sandbox( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - return Ok(false); - }; - let Some(target) = summary_container_target(&container) else { - return Ok(false); - }; - let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if !container_state_needs_resume(state) { - return Ok(true); - } - - match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), - // Already running — race with another resume path or the - // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), - } - } - - pub async fn stop_managed_containers_on_shutdown(&self) -> Result { - let containers = self.list_managed_container_summaries().await?; - let targets = containers - .into_iter() - .filter_map(|container| { - let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if container_state_needs_shutdown_stop(state) { - summary_container_target(&container) - } else { - None - } - }) - .collect::>(); - let target_count = targets.len(); - let mut stopped = 0usize; - let mut failures = Vec::new(); - let stop_timeout_secs = self.config.stop_timeout_secs; - - let mut stop_results = futures::stream::iter(targets.into_iter().map(|target| { - let docker = self.docker.clone(); - async move { - let result = docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(stop_timeout_secs)) - .build(), - ), - ) - .await; - (target, result) - } - })) - .buffer_unordered(16); - - while let Some((target, result)) = stop_results.next().await { - match result { - Ok(()) => { - stopped += 1; - } - Err(err) if is_not_found_error(&err) || is_not_modified_error(&err) => {} - Err(err) => { - warn!( - container = %target, - error = %err, - "Failed to stop Docker sandbox container during shutdown" - ); - failures.push(target); - } - } - } - - if !failures.is_empty() { - return Err(Status::internal(format!( - "failed to stop {} of {target_count} Docker sandbox containers during shutdown", - failures.len() - ))); - } - - Ok(stopped) - } - - async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { - let mut pending = self.pending.lock().await; - if pending - .values() - .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) - { - return Err(Status::already_exists("sandbox already exists")); - } - - pending.insert( - sandbox.id.clone(), - PendingSandboxRecord { - sandbox: pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - provisioning_condition(), - false, - ), - task: None, - }, - ); - Ok(()) - } - - async fn pending_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Option { - let pending = self.pending.lock().await; - pending - .values() - .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) - .map(|record| record.sandbox.clone()) - } - - async fn pending_snapshot_map(&self) -> HashMap { - let pending = self.pending.lock().await; - pending - .iter() - .map(|(sandbox_id, record)| (sandbox_id.clone(), record.sandbox.clone())) - .collect() - } - - async fn clear_pending_sandbox(&self, sandbox_id: &str) { - let mut pending = self.pending.lock().await; - pending.remove(sandbox_id); - } - - async fn remove_pending_sandbox( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Option { - let mut pending = self.pending.lock().await; - let id = pending.iter().find_map(|(id, record)| { - pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) - })?; - pending.remove(&id) - } - - async fn fail_pending_sandbox( - &self, - sandbox: &DriverSandbox, - failure: &DockerProvisioningFailure, - ) { - cleanup_sandbox_token_file(sandbox, &self.config); - let snapshot = pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - error_condition(failure.reason, &failure.message), - false, - ); - { - let mut pending = self.pending.lock().await; - if let Some(record) = pending.get_mut(&sandbox.id) { - record.sandbox = snapshot.clone(); - record.task = None; - } else { - return; - } - } - - self.publish_platform_event( - sandbox.id.clone(), - platform_event( - "docker", - "Warning", - failure.reason, - format!("Docker sandbox provisioning failed: {}", failure.message), - ), - ); - self.publish_sandbox_snapshot(snapshot); - } - - async fn publish_container_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result<(), Status> { - if let Some(summary) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - && let Some(sandbox) = - sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) - { - self.publish_sandbox_snapshot(sandbox); - } - Ok(()) - } - - fn publish_sandbox_snapshot(&self, sandbox: DriverSandbox) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox), - }, - )), - }); - } - - fn publish_deleted(&self, sandbox_id: String) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { sandbox_id }, - )), - }); - } - - fn publish_platform_event(&self, sandbox_id: String, event: DriverPlatformEvent) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::PlatformEvent( - WatchSandboxesPlatformEvent { - sandbox_id, - event: Some(event), - }, - )), - }); - } - - fn publish_docker_progress( - &self, - sandbox_id: &str, - reason: &str, - message: String, - mut metadata: HashMap, - ) { - attach_docker_progress_metadata(&mut metadata, reason, &message); - self.publish_platform_event( - sandbox_id.to_string(), - DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: "docker".to_string(), - r#type: "Normal".to_string(), - reason: reason.to_string(), - message, - metadata, - }, - ); - } - - async fn poll_loop(self) { - let mut previous = match self.current_snapshot_map().await { - Ok(snapshots) => snapshots, - Err(err) => { - warn!(error = %err, "Failed to seed Docker sandbox watch state"); - HashMap::new() - } - }; - - // Exponential backoff on consecutive Docker failures to avoid a 2s - // warn-log flood when the daemon is unreachable for an extended - // period (e.g. restart, socket removed). - let mut backoff = WATCH_POLL_INTERVAL; - loop { - tokio::time::sleep(backoff).await; - match self.current_snapshot_map().await { - Ok(current) => { - emit_snapshot_diff(&self.events, &previous, ¤t); - previous = current; - backoff = WATCH_POLL_INTERVAL; - } - Err(err) => { - warn!( - error = %err, - backoff_secs = backoff.as_secs(), - "Failed to poll Docker sandboxes" - ); - backoff = (backoff * 2).min(WATCH_POLL_MAX_BACKOFF); - } - } - } - } - - async fn current_snapshot_map(&self) -> Result, Status> { - self.current_snapshots().await.map(|snapshots| { - snapshots - .into_iter() - .map(|sandbox| (sandbox.id.clone(), sandbox)) - .collect() - }) - } - - async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); - self.docker - .list_containers(Some( - ListContainersOptionsBuilder::default() - .all(true) - .filters(&filters) - .build(), - )) - .await - .map_err(|err| internal_status("list Docker sandbox containers", err)) - } - - async fn find_managed_container_summary( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result, Status> { - let mut label_filter_values = Vec::new(); - if !sandbox_id.is_empty() { - label_filter_values.push(format!("{LABEL_SANDBOX_ID}={sandbox_id}")); - } else if !sandbox_name.is_empty() { - label_filter_values.push(format!("{LABEL_SANDBOX_NAME}={sandbox_name}")); - } - - let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); - let containers = self - .docker - .list_containers(Some( - ListContainersOptionsBuilder::default() - .all(true) - .filters(&filters) - .build(), - )) - .await - .map_err(|err| internal_status("find Docker sandbox container", err))?; - - Ok(containers.into_iter().find(|summary| { - let Some(labels) = summary.labels.as_ref() else { - return false; - }; - let namespace_matches = labels - .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); - let id_matches = sandbox_id.is_empty() - || labels - .get(LABEL_SANDBOX_ID) - .is_some_and(|value| value == sandbox_id); - let name_matches = sandbox_name.is_empty() - || labels - .get(LABEL_SANDBOX_NAME) - .is_some_and(|value| value == sandbox_name); - namespace_matches && id_matches && name_matches - })) - } - - async fn ensure_image_available(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - match policy.as_str() { - "" | "ifnotpresent" => { - if self.docker.inspect_image(image).await.is_ok() { - self.publish_docker_progress( - sandbox_id, - "ImagePresent", - format!("Docker image \"{image}\" is already present"), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - return Ok(()); - } - self.pull_image(sandbox_id, image).await - } - "always" => self.pull_image(sandbox_id, image).await, - "never" => match self.docker.inspect_image(image).await { - Ok(_) => { - self.publish_docker_progress( - sandbox_id, - "ImagePresent", - format!("Docker image \"{image}\" is already present"), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - Ok(()) - } - Err(err) if is_not_found_error(&err) => Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" - ))), - Err(err) => Err(internal_status("inspect Docker image", err)), - }, - other => Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))), - } - } - - async fn pull_image(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { - self.publish_docker_progress( - sandbox_id, - "Pulling", - format!("Pulling Docker image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - let mut stream = self.docker.create_image( - Some(CreateImageOptions { - from_image: Some(image.to_string()), - ..Default::default() - }), - None, - None, - ); - while let Some(result) = stream.next().await { - let info = result.map_err(|err| internal_status("pull Docker image", err))?; - if let Some(message) = info - .error_detail - .as_ref() - .and_then(|detail| detail.message.as_ref()) - { - return Err(Status::failed_precondition(format!( - "pull Docker image '{image}' failed: {message}" - ))); - } - if let Some(event) = docker_pull_progress_event(image, &info) { - self.publish_platform_event(sandbox_id.to_string(), event); - } - } - self.publish_docker_progress( - sandbox_id, - "Pulled", - format!("Pulled Docker image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - Ok(()) - } -} - -#[tonic::async_trait] -impl ComputeDriver for DockerComputeDriver { - type WatchSandboxesStream = WatchStream; - - async fn get_capabilities( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(self.capabilities())) - } - - async fn validate_sandbox_create( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - let validated = Self::validated_sandbox(&sandbox, &self.config)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; - let _ = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::peek_device_ids, - ) - .await?; - Ok(Response::new(ValidateSandboxCreateResponse {})) - } - - async fn get_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - let sandbox = self - .get_sandbox_snapshot(&request.sandbox_id, &request.sandbox_name) - .await? - .ok_or_else(|| Status::not_found("sandbox not found"))?; - - if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { - return Err(Status::failed_precondition( - "sandbox_id did not match the fetched sandbox", - )); - } - - Ok(Response::new(GetSandboxResponse { - sandbox: Some(sandbox), - })) - } - - async fn list_sandboxes( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(ListSandboxesResponse { - sandboxes: self.current_snapshots().await?, - })) - } - - async fn create_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.create_sandbox_inner(&sandbox).await?; - Ok(Response::new(CreateSandboxResponse {})) - } - - async fn stop_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - self.stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - Ok(Response::new(StopSandboxResponse {})) - } - - async fn delete_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - let event_sandbox_id = request.sandbox_id.clone(); - let deleted = self - .delete_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - if deleted && !event_sandbox_id.is_empty() { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { - sandbox_id: event_sandbox_id, - }, - )), - }); - } - - Ok(Response::new(DeleteSandboxResponse { deleted })) - } - - async fn watch_sandboxes( - &self, - _request: Request, - ) -> Result, Status> { - // Subscribe before taking the initial snapshot so any event emitted - // between the snapshot and this subscriber becoming active is still - // delivered. Downstream consumers treat sandbox events as - // idempotent (keyed by sandbox id), so a duplicate event is benign - // while a missed one leaks state. - let mut rx = self.events.subscribe(); - let initial = self.current_snapshots().await?; - let (tx, out_rx) = mpsc::channel(WATCH_BUFFER); - tokio::spawn(async move { - for sandbox in initial { - if tx - .send(Ok(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox), - }, - )), - })) - .await - .is_err() - { - return; - } - } - - loop { - match rx.recv().await { - Ok(event) => { - if tx.send(Ok(event)).await.is_err() { - return; - } - } - Err(broadcast::error::RecvError::Lagged(_)) => {} - Err(broadcast::error::RecvError::Closed) => return, - } - } - }); - - Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) - } -} - -impl DockerProvisioningFailure { - fn new(reason: &'static str, message: impl Into) -> Self { - Self { - reason, - message: message.into(), - } - } - - fn from_status(reason: &'static str, status: Status) -> Self { - Self::new(reason, status.message()) - } -} - -fn sandbox_image(sandbox: &DriverSandbox) -> Option { - sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .map(|template| template.image.clone()) - .filter(|image| !image.trim().is_empty()) -} - -fn pending_sandbox_snapshot( - sandbox: &DriverSandbox, - namespace: &str, - condition: DriverCondition, - deleting: bool, -) -> DriverSandbox { - DriverSandbox { - id: sandbox.id.clone(), - name: sandbox.name.clone(), - namespace: namespace.to_string(), - spec: None, - status: Some(DriverSandboxStatus { - sandbox_name: sandbox.name.clone(), - instance_id: String::new(), - agent_fd: String::new(), - sandbox_fd: String::new(), - conditions: vec![condition], - deleting, - }), - workspace: sandbox.workspace.clone(), - } -} - -fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { - (!sandbox_id.is_empty() && sandbox.id == sandbox_id) - || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) -} - -fn provisioning_condition() -> DriverCondition { - DriverCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: "Starting".to_string(), - message: "Docker container is starting".to_string(), - last_transition_time: String::new(), - } -} - -fn error_condition(reason: &str, message: &str) -> DriverCondition { - DriverCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: reason.to_string(), - message: message.to_string(), - last_transition_time: String::new(), - } -} - -fn platform_event( - source: &str, - event_type: &str, - reason: &str, - message: String, -) -> DriverPlatformEvent { - DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: source.to_string(), - r#type: event_type.to_string(), - reason: reason.to_string(), - message, - metadata: HashMap::new(), - } -} - -fn docker_pull_progress_event(image: &str, info: &CreateImageInfo) -> Option { - let status = info.status.as_deref().map(str::trim)?; - if status.is_empty() { - return None; - } - - let mut metadata = HashMap::from([ - ("image_ref".to_string(), image.to_string()), - ("docker_status".to_string(), status.to_string()), - ]); - if let Some(layer_id) = info.id.as_deref().filter(|id| !id.is_empty()) { - metadata.insert("layer_id".to_string(), layer_id.to_string()); - } - if let Some(detail) = docker_pull_progress_detail(info) { - metadata.insert("detail".to_string(), detail); - } - attach_docker_progress_metadata(&mut metadata, "PullingLayer", status); - - Some(DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: "docker".to_string(), - r#type: "Normal".to_string(), - reason: "PullingLayer".to_string(), - message: docker_pull_message(info, status), - metadata, - }) -} - -fn docker_pull_message(info: &CreateImageInfo, status: &str) -> String { - info.id.as_deref().filter(|id| !id.is_empty()).map_or_else( - || format!("Docker image pull: {status}"), - |layer_id| format!("Docker image pull {layer_id}: {status}"), - ) -} - -fn docker_pull_progress_detail(info: &CreateImageInfo) -> Option { - let status = info.status.as_deref().unwrap_or("Pulling"); - let layer_id = info.id.as_deref().filter(|id| !id.is_empty()); - let progress = info - .progress_detail - .as_ref() - .and_then(format_progress_detail); - - match (layer_id, progress) { - (Some(layer_id), Some(progress)) => Some(format!("{status} {layer_id} ({progress})")), - (Some(layer_id), None) => Some(format!("{status} {layer_id}")), - (None, Some(progress)) => Some(format!("{status} ({progress})")), - (None, None) => (!status.is_empty()).then(|| status.to_string()), - } -} - -fn format_progress_detail(progress: &ProgressDetail) -> Option { - let current = progress.current.and_then(|value| u64::try_from(value).ok()); - let total = progress - .total - .and_then(|value| u64::try_from(value).ok()) - .filter(|value| *value > 0); - - match (current, total) { - (Some(current), Some(total)) => { - Some(format!("{}/{}", format_bytes(current), format_bytes(total))) - } - (Some(current), _) if current > 0 => Some(format_bytes(current)), - _ => None, - } -} - -fn attach_docker_progress_metadata( - metadata: &mut HashMap, - reason: &str, - message: &str, -) { - match reason { - "Scheduled" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_REQUESTING_SANDBOX, - "Sandbox allocated", - ); - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = metadata.get("image_ref").cloned() { - mark_progress_detail(metadata, image); - } - } - "Pulling" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = metadata.get("image_ref").cloned() { - mark_progress_detail(metadata, image); - } - } - "PullingLayer" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(detail) = metadata - .get("detail") - .cloned() - .filter(|detail| !detail.is_empty()) - { - mark_progress_detail(metadata, detail); - } else if !message.is_empty() { - mark_progress_detail(metadata, message); - } - } - "ImagePresent" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_PULLING_IMAGE, - "Image already present", - ); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - "Pulled" => { - mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, "Image pulled"); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - "Created" => { - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - mark_progress_detail(metadata, "Container created"); - } - "Started" => { - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - mark_progress_detail(metadata, "Waiting for supervisor relay"); - } - _ => {} - } -} - -#[cfg(test)] -fn docker_driver_config( - template: &DriverSandboxTemplate, - enable_bind_mounts: bool, -) -> Result { - let config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?; - Ok(config) -} - -/// Collect user-supplied bind mounts as string-format binds. -/// -/// Bind mounts use the legacy `Binds` field (`-v` syntax) rather than the -/// structured `Mount` API because the Docker Engine Mount object does not -/// support `SELinux` relabelling (`:z` / `:Z`). The string format does. -fn docker_driver_bind_strings(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| match m { - DockerDriverMountConfig::Bind { - source, - target, - read_only, - selinux_label, - } => Some(docker_bind_string( - source, - target, - *read_only, - *selinux_label, - )), - _ => None, - }) - .collect() -} - -fn docker_bind_string( - source: &str, - target: &str, - read_only: bool, - selinux_label: Option, -) -> Result { - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - // Legacy `-v` binds silently create missing source directories as empty, - // root-owned paths. The structured `--mount` API that was used before this - // change rejected missing sources at container-create time. Preserve that - // fail-fast behaviour with an explicit existence check. - if !Path::new(source).exists() { - return Err(Status::failed_precondition(format!( - "bind source path does not exist: {source}" - ))); - } - driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - - let mut opts = Vec::new(); - if read_only { - opts.push("ro"); - } - match selinux_label { - Some(SelinuxLabel::Shared) => opts.push("z"), - Some(SelinuxLabel::Private) => opts.push("Z"), - None => {} - } - - if opts.is_empty() { - Ok(format!("{source}:{normalized_target}")) - } else { - Ok(format!("{source}:{normalized_target}:{}", opts.join(","))) - } -} - -/// Collect user-supplied non-bind mounts as structured `Mount` objects. -fn docker_driver_mounts(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| docker_mount_from_config(m).transpose()) - .collect() -} - -fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result, Status> { - match config { - DockerDriverMountConfig::Bind { .. } => { - // Bind mounts are handled via docker_driver_bind_strings. - Ok(None) - } - DockerDriverMountConfig::Volume { - source, - target, - read_only, - subpath, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::VOLUME), - source: Some(source.clone()), - target: Some(target.clone()), - read_only: Some(*read_only), - volume_options: subpath.as_ref().map(|subpath| MountVolumeOptions { - subpath: Some(subpath.clone()), - ..Default::default() - }), - ..Default::default() - })), - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::TMPFS), - target: Some(target.clone()), - tmpfs_options: Some(MountTmpfsOptions { - size_bytes: validate_optional_positive_integral_i64( - *size_bytes, - "tmpfs size_bytes", - )?, - mode: validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?, - options: (!options.is_empty()) - .then(|| { - options - .iter() - .map(|option| docker_tmpfs_option(option)) - .collect::, _>>() - }) - .transpose()?, - }), - ..Default::default() - })), - DockerDriverMountConfig::Image { .. } => Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )), - } -} - -fn validate_docker_driver_mounts( - mounts: &[DockerDriverMountConfig], - enable_bind_mounts: bool, -) -> Result<(), Status> { - let mut targets = HashSet::new(); - for mount in mounts { - let target = match mount { - DockerDriverMountConfig::Bind { source, target, .. } => { - if !enable_bind_mounts { - return Err(Status::failed_precondition( - "docker bind mounts require enable_bind_mounts = true in [openshell.drivers.docker]", - )); - } - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - target - } - DockerDriverMountConfig::Volume { - source, - target, - subpath, - .. - } => { - driver_mounts::validate_mount_source(source, "volume source") - .map_err(Status::failed_precondition)?; - if let Some(subpath) = subpath { - driver_mounts::validate_mount_subpath(subpath) - .map_err(Status::failed_precondition)?; - } - target - } - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => { - validate_optional_positive_integral_i64(*size_bytes, "tmpfs size_bytes")?; - validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?; - for option in options { - docker_tmpfs_option(option)?; - } - target - } - DockerDriverMountConfig::Image { - source, - target, - read_only, - subpath, - } => { - let _ = (source, target, read_only, subpath); - return Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )); - } - }; - driver_mounts::validate_container_mount_target(target) - .map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - if !targets.insert(normalized_target.clone()) { - return Err(Status::failed_precondition(format!( - "duplicate docker driver_config mount target '{normalized_target}'" - ))); - } - } - Ok(()) -} - -fn validate_optional_positive_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value <= 0 { - return Err(Status::failed_precondition(format!( - "{field} must be positive" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_nonnegative_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value < 0 { - return Err(Status::failed_precondition(format!( - "{field} must be zero or greater" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_integral_i64(value: Option, field: &str) -> Result, Status> { - let Some(value) = value else { - return Ok(None); - }; - if !value.is_finite() || value.fract() != 0.0 { - return Err(Status::failed_precondition(format!( - "{field} must be an integer" - ))); - } - value.to_string().parse::().map(Some).map_err(|_| { - Status::failed_precondition(format!("{field} must be representable as an i64")) - }) -} - -fn docker_tmpfs_option(option: &str) -> Result, Status> { - let option = option.trim(); - if option.is_empty() { - return Err(Status::failed_precondition( - "tmpfs options must not contain empty values", - )); - } - if let Some((key, value)) = option.split_once('=') { - let key = key.trim(); - let value = value.trim(); - if key.is_empty() || value.is_empty() { - return Err(Status::failed_precondition( - "tmpfs key=value options must include both key and value", - )); - } - Ok(vec![key.to_string(), value.to_string()]) - } else { - Ok(vec![option.to_string()]) - } -} - -fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { - volume.driver == "local" - && volume.options.get("o").is_some_and(|options| { - options.split(',').any(|option| { - let option = option.trim(); - option.eq_ignore_ascii_case("bind") || option.eq_ignore_ascii_case("rbind") - }) - }) -} - -fn build_binds( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - Ok(binds) -} - -fn sandbox_token_host_path( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - sandbox_token_host_path_by_id(&sandbox.id, config) -} - -fn sandbox_token_host_path_by_id( - sandbox_id: &str, - config: &DockerDriverRuntimeConfig, -) -> Result { - openshell_core::driver_utils::sandbox_token_path( - "docker-sandbox-tokens", - Some(&config.sandbox_namespace), - sandbox_id, - ) - .map_err(|err| { - Status::internal(format!( - "resolve sandbox token state directory failed: {err}" - )) - }) -} - -async fn write_sandbox_token_file( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let Some(spec) = sandbox.spec.as_ref() else { - return Ok(false); - }; - if spec.sandbox_token.is_empty() { - return Ok(false); - } - let path = sandbox_token_host_path(sandbox, config)?; - if let Some(parent) = path.parent() { - openshell_core::paths::create_dir_restricted(parent).map_err(|err| { - Status::internal(format!( - "create sandbox token directory {} failed: {err}", - parent.display() - )) - })?; - } - tokio::fs::write(&path, format!("{}\n", spec.sandbox_token)) - .await - .map_err(|err| { - Status::internal(format!( - "write sandbox token file {} failed: {err}", - path.display() - )) - })?; - openshell_core::paths::set_file_owner_only(&path).map_err(|err| { - Status::internal(format!( - "restrict sandbox token file {} failed: {err}", - path.display() - )) - })?; - Ok(true) -} - -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); -} - -fn cleanup_sandbox_token_file_for_delete( - sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, - config: &DockerDriverRuntimeConfig, -) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); - } -} - -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { - return; - }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" - ); - } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); - } -} - -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), - ), - ]); - - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - } - - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_COMMAND.to_string(), - SANDBOX_COMMAND.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); - } - - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), - ); - } - - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() -} - -fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { - CdiGpuInventory::new( - info.discovered_devices - .as_deref() - .unwrap_or_default() - .iter() - .filter(|device| device.source.as_deref() == Some("cdi")) - .filter_map(|device| device.id.as_deref()), - ) -} - -fn docker_info_reports_wsl2(info: &SystemInfo) -> bool { - [ - info.kernel_version.as_deref(), - info.operating_system.as_deref(), - ] - .into_iter() - .flatten() - .any(os_or_kernel_reports_wsl2) -} - -fn os_or_kernel_reports_wsl2(value: &str) -> bool { - let value = value.to_ascii_lowercase(); - value.contains("wsl2") || value.contains("microsoft-standard") -} - -fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status { - Status::failed_precondition(err.to_string()) -} - -#[cfg(test)] -fn build_container_create_body( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let template = sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let driver_config = docker_driver_config(template, config.enable_bind_mounts)?; - let gpu_requirements = sandbox - .spec - .as_ref() - .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); - let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - Some(cdi_devices.as_slice()) - } else { - None - }; - build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) -} - -fn build_container_create_body_with_gpu_devices( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - driver_config: &DockerSandboxDriverConfig, - gpu_device_ids: Option<&[String]>, -) -> Result { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let resource_limits = docker_resource_limits(template)?; - let user_mounts = docker_driver_mounts(driver_config)?; - let user_bind_strings = docker_driver_bind_strings(driver_config)?; - let device_requests = gpu_device_ids.map(|device_ids| { - vec![DeviceRequest { - driver: Some("cdi".to_string()), - device_ids: Some(device_ids.to_vec()), - ..Default::default() - }] - }); - let mut labels = template.labels.clone(); - labels.insert( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ); - labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); - labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); - labels.insert( - LABEL_SANDBOX_WORKSPACE.to_string(), - sandbox.workspace.clone(), - ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use - // the same value here. `DriverSandbox.namespace` is unset on the request - // path (the gateway elides it), and using it would produce containers - // that the driver itself cannot find afterwards. - labels.insert( - LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), - ); - - Ok(ContainerCreateBody { - image: Some(template.image.clone()), - user: Some("0".to_string()), - env: Some(build_environment(sandbox, config)), - entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Clear the image CMD so Docker does not append inherited args to the - // supervisor entrypoint. - cmd: Some(Vec::new()), - labels: Some(labels), - host_config: Some(HostConfig { - nano_cpus: resource_limits.nano_cpus, - memory: resource_limits.memory_bytes, - pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, - device_requests, - binds: { - let mut binds = build_binds(sandbox, config)?; - binds.extend(user_bind_strings); - Some(binds) - }, - mounts: Some(user_mounts), - restart_policy: Some(RestartPolicy { - name: Some(RestartPolicyNameEnum::UNLESS_STOPPED), - maximum_retry_count: None, - }), - cap_add: Some(vec![ - "SYS_ADMIN".to_string(), - "NET_ADMIN".to_string(), - "SYS_PTRACE".to_string(), - "SYSLOG".to_string(), - ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), - )])), - }), - ..Default::default() - }) -} - -/// Reject driver requests that arrive with neither a sandbox id nor a -/// sandbox name. Without this guard, downstream label filters degenerate -/// to "match every managed container in the namespace", which would let -/// `delete_sandbox`/`stop_sandbox`/`get_sandbox` pick an arbitrary -/// sandbox out of the set the driver manages. -fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { - if sandbox_id.is_empty() && sandbox_name.is_empty() { - return Err(Status::invalid_argument( - "sandbox_id or sandbox_name is required", - )); - } - Ok(()) -} - -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); - } - - endpoint.to_string() -} - -fn docker_network_name(config: &DockerComputeConfig) -> String { - let name = config.network_name.trim(); - if name.is_empty() { - return DEFAULT_DOCKER_NETWORK_NAME.to_string(); - } - name.to_string() -} - -fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { - let trimmed = value.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - trimmed - .parse() - .map(Some) - .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) -} - -fn docker_gateway_route( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, -) -> DockerGatewayRoute { - docker_gateway_route_for_host( - info, - bridge_gateway_ip, - port, - host_gateway_ip, - host_runtime_requires_host_gateway_alias(), - ) -} - -fn docker_gateway_route_for_host( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, - host_requires_host_gateway_alias: bool, -) -> DockerGatewayRoute { - if let Some(host_alias_ip) = host_gateway_ip { - return DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, - }; - } - - if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { - DockerGatewayRoute::HostGateway - } else { - DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, - } - } -} - -fn host_runtime_requires_host_gateway_alias() -> bool { - cfg!(target_os = "macos") -} - -/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, -/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing -/// constraint: the bridge gateway IP is reachable from inside containers but -/// not from the `OpenShell` server process running on the host, so callbacks -/// must traverse `host-gateway`. -/// -/// Each runtime is detected via the daemon's reported OS string or hostname, -/// supplemented by labels where the runtime publishes them. -fn uses_host_gateway_alias(info: &SystemInfo) -> bool { - let operating_system = info - .operating_system - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if operating_system.contains("docker desktop") { - return true; - } - - let name = info - .name - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if name.starts_with("colima") - || name.starts_with("lima-") - || name.starts_with("rancher-desktop") - || name.starts_with("orbstack") - { - return true; - } - - info.labels.as_ref().is_some_and(|labels| { - labels.iter().any(|label| { - label.starts_with("com.docker.desktop.") - || label.starts_with("dev.rancherdesktop.") - || label.starts_with("dev.orbstack.") - }) - }) -} - -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - -async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { - match docker.inspect_network(network_name, None).await { - Ok(network) => return validate_bridge_network(network_name, &network), - Err(err) if !is_not_found_error(&err) => { - return Err(Error::execution(format!( - "failed to inspect Docker network '{network_name}': {err}" - ))); - } - Err(_) => {} - } - - docker - .create_network(NetworkCreateRequest { - name: network_name.to_string(), - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - attachable: Some(true), - labels: Some(HashMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }) - .await - .map(|_| ()) - .or_else(|err| { - if is_conflict_error(&err) { - Ok(()) - } else { - Err(Error::execution(format!( - "failed to create Docker network '{network_name}': {err}" - ))) - } - })?; - - let network = docker - .inspect_network(network_name, None) - .await - .map_err(|err| { - Error::execution(format!( - "failed to inspect Docker network '{network_name}' after create: {err}" - )) - })?; - validate_bridge_network(network_name, &network) -} - -fn validate_bridge_network( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { - return Err(Error::config(format!( - "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", - network.driver.as_deref().unwrap_or("unknown") - ))); - } - - docker_bridge_gateway_ip(network_name, network) -} - -fn docker_bridge_gateway_ip( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { - return Err(Error::config(format!( - "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" - ))); - }; - - for config in configs { - let Some(gateway) = config.gateway.as_deref() else { - continue; - }; - let ip = gateway.parse::().map_err(|err| { - Error::config(format!( - "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" - )) - })?; - if matches!(ip, IpAddr::V4(_)) { - return Ok(ip); - } - } - - Err(Error::config(format!( - "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" - ))) -} - -fn docker_resource_limits( - template: &DriverSandboxTemplate, -) -> Result { - let Some(resources) = template.resources.as_ref() else { - return Ok(DockerResourceLimits::default()); - }; - - if !resources.cpu_request.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support resources.requests.cpu", - )); - } - if !resources.memory_request.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support resources.requests.memory", - )); - } - - Ok(DockerResourceLimits { - nano_cpus: parse_cpu_limit(&resources.cpu_limit)?, - memory_bytes: parse_memory_limit(&resources.memory_limit)?, - }) -} - -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { - return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", - )); - } - Ok(()) -} - -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", - )); - } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) - } -} - -#[allow(clippy::cast_possible_truncation)] -fn parse_cpu_limit(value: &str) -> Result, Status> { - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - if let Some(millicores) = value.strip_suffix('m') { - let millicores = millicores.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", - )) - })?; - if millicores <= 0 { - return Err(Status::failed_precondition( - "docker cpu_limit must be greater than zero", - )); - } - return Ok(Some(millicores.saturating_mul(1_000_000))); - } - - let cores = value.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", - )) - })?; - if !cores.is_finite() || cores <= 0.0 { - return Err(Status::failed_precondition( - "docker cpu_limit must be greater than zero", - )); - } - - Ok(Some((cores * 1_000_000_000.0).round() as i64)) -} - -#[allow(clippy::cast_possible_truncation)] -fn parse_memory_limit(value: &str) -> Result, Status> { - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - - let number_end = value - .find(|ch: char| !(ch.is_ascii_digit() || ch == '.')) - .unwrap_or(value.len()); - let (number, suffix) = value.split_at(number_end); - let amount = number.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker memory_limit '{value}'; expected a Kubernetes-style quantity", - )) - })?; - if !amount.is_finite() || amount <= 0.0 { - return Err(Status::failed_precondition( - "docker memory_limit must be greater than zero", - )); - } - - let multiplier = match suffix { - "" => 1_f64, - "Ki" => 1024_f64, - "Mi" => 1024_f64.powi(2), - "Gi" => 1024_f64.powi(3), - "Ti" => 1024_f64.powi(4), - "Pi" => 1024_f64.powi(5), - "Ei" => 1024_f64.powi(6), - "K" => 1000_f64, - "M" => 1000_f64.powi(2), - "G" => 1000_f64.powi(3), - "T" => 1000_f64.powi(4), - "P" => 1000_f64.powi(5), - "E" => 1000_f64.powi(6), - _ => { - return Err(Status::failed_precondition(format!( - "invalid docker memory_limit suffix '{suffix}'", - ))); - } - }; - - Ok(Some((amount * multiplier).round() as i64)) -} - -fn sandbox_from_container_summary( - summary: &ContainerSummary, - readiness: &dyn SupervisorReadiness, -) -> Option { - let labels = summary.labels.as_ref()?; - let id = labels.get(LABEL_SANDBOX_ID)?.clone(); - let name = labels.get(LABEL_SANDBOX_NAME)?.clone(); - let namespace = labels - .get(LABEL_SANDBOX_NAMESPACE) - .cloned() - .unwrap_or_default(); - let workspace = labels - .get(LABEL_SANDBOX_WORKSPACE) - .cloned() - .unwrap_or_default(); - - let supervisor_connected = readiness.is_supervisor_connected(&id); - Some(DriverSandbox { - id, - name: name.clone(), - namespace, - spec: None, - status: Some(driver_status_from_summary( - summary, - &name, - supervisor_connected, - )), - workspace, - }) -} - -fn driver_status_from_summary( - summary: &ContainerSummary, - sandbox_name: &str, - supervisor_connected: bool, -) -> DriverSandboxStatus { - let state = summary.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - let (ready, reason, message, deleting) = container_ready_condition(state, supervisor_connected); - - DriverSandboxStatus { - sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), - instance_id: summary.id.clone().unwrap_or_default(), - agent_fd: String::new(), - sandbox_fd: String::new(), - conditions: vec![DriverCondition { - r#type: "Ready".to_string(), - status: ready.to_string(), - reason: reason.to_string(), - message: message.to_string(), - last_transition_time: String::new(), - }], - deleting, - } -} - -fn container_ready_condition( - state: ContainerSummaryStateEnum, - supervisor_connected: bool, -) -> (&'static str, &'static str, &'static str, bool) { - match state { - ContainerSummaryStateEnum::RUNNING => { - if supervisor_connected { - ( - "True", - "SupervisorConnected", - "Supervisor relay is live", - false, - ) - } else { - ( - "False", - "DependenciesNotReady", - "Container is running; waiting for supervisor relay", - false, - ) - } - } - ContainerSummaryStateEnum::CREATED => ("False", "Starting", "Container created", false), - ContainerSummaryStateEnum::RESTARTING => ( - "False", - "ContainerRestarting", - "Container is restarting after a failure", - false, - ), - ContainerSummaryStateEnum::EMPTY => { - ("False", "Starting", "Container state is unknown", false) - } - ContainerSummaryStateEnum::REMOVING => { - ("False", "Deleting", "Container is being removed", true) - } - ContainerSummaryStateEnum::PAUSED => { - ("False", "ContainerPaused", "Container is paused", false) - } - ContainerSummaryStateEnum::EXITED => { - ("False", "ContainerExited", "Container exited", false) - } - ContainerSummaryStateEnum::DEAD => ("False", "ContainerDead", "Container is dead", false), - } -} - -fn summary_container_name(summary: &ContainerSummary) -> Option { - summary - .names - .as_ref() - .and_then(|names| names.first()) - .map(|name| name.trim_start_matches('/').to_string()) - .filter(|name| !name.is_empty()) -} - -fn summary_container_target(summary: &ContainerSummary) -> Option { - // Prefer the container ID: it's stable while the container exists and is - // accepted by Docker APIs just like a name. Fall back to the parsed name - // for transient summaries that do not include an ID. - summary - .id - .as_deref() - .filter(|id| !id.is_empty()) - .map(str::to_string) - .or_else(|| summary_container_name(summary)) -} - -fn container_state_needs_shutdown_stop(state: ContainerSummaryStateEnum) -> bool { - matches!( - state, - ContainerSummaryStateEnum::RUNNING - | ContainerSummaryStateEnum::RESTARTING - | ContainerSummaryStateEnum::PAUSED - ) -} - -/// States from which a managed container can be brought back to running by -/// `start_container`. Skip `Restarting` (already coming up), `Removing`, -/// `Dead` (terminal), `Paused` (needs `unpause`, not `start`), and -/// `Running` (nothing to do). -fn container_state_needs_resume(state: ContainerSummaryStateEnum) -> bool { - matches!( - state, - ContainerSummaryStateEnum::EXITED | ContainerSummaryStateEnum::CREATED - ) -} - -fn docker_stop_timeout_secs(timeout_secs: u32) -> i32 { - i32::try_from(timeout_secs).unwrap_or(i32::MAX) -} - -fn emit_snapshot_diff( - events: &broadcast::Sender, - previous: &HashMap, - current: &HashMap, -) { - for (sandbox_id, sandbox) in current { - if previous.get(sandbox_id) == Some(sandbox) { - continue; - } - let _ = events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox.clone()), - }, - )), - }); - } - - for sandbox_id in previous.keys() { - if current.contains_key(sandbox_id) { - continue; - } - let _ = events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { - sandbox_id: sandbox_id.clone(), - }, - )), - }); - } -} - -fn label_filters(values: impl IntoIterator) -> HashMap> { - HashMap::from([("label".to_string(), values.into_iter().collect())]) -} - -fn managed_container_label_filters( - sandbox_namespace: &str, - extra_values: impl IntoIterator, -) -> HashMap> { - let mut values = vec![ - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), - ]; - values.extend(extra_values); - label_filters(values) -} - -/// Maximum Docker container name length. Docker's own limit is 253 bytes, but -/// we cap at a conservative 200 to leave headroom for tooling that truncates -/// names further. -const MAX_CONTAINER_NAME_LEN: usize = 200; -const CONTAINER_NAME_PREFIX: &str = "openshell-"; - -fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String { - let id_suffix = sanitize_docker_name(&sandbox.id); - let workspace = sanitize_docker_name(&sandbox.workspace); - let name = sanitize_docker_name(&sandbox.name); - - // Format: openshell-{workspace}--{name}-{id} - // The workspace and id are never truncated — they ensure uniqueness. - // Only the sandbox name portion is truncated when the total exceeds - // MAX_CONTAINER_NAME_LEN. - - if name.is_empty() { - let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); - if base.len() > MAX_CONTAINER_NAME_LEN { - base.truncate(MAX_CONTAINER_NAME_LEN); - } - return trim_container_name_tail(base); - } - - // Reserve space for fixed parts: prefix + workspace + "--" + "-" + id - let reserved = CONTAINER_NAME_PREFIX.len() + workspace.len() + 2 + 1 + id_suffix.len(); - if reserved >= MAX_CONTAINER_NAME_LEN { - let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); - base.truncate(MAX_CONTAINER_NAME_LEN); - return trim_container_name_tail(base); - } - - let name_budget = MAX_CONTAINER_NAME_LEN - reserved; - let truncated_name = if name.len() > name_budget { - trim_container_name_tail(name[..name_budget].to_string()) - } else { - name - }; - format!("{CONTAINER_NAME_PREFIX}{workspace}--{truncated_name}-{id_suffix}") -} - -/// Docker container names may not end with `-`, `.`, or `_`. Truncation can -/// leave one of those trailing, so strip them before returning. -fn trim_container_name_tail(mut value: String) -> String { - while value - .chars() - .last() - .is_some_and(|ch| matches!(ch, '-' | '.' | '_')) - { - value.pop(); - } - value -} - -fn sanitize_docker_name(value: &str) -> String { - value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '-') { - ch.to_ascii_lowercase() - } else { - '-' - } - }) - .collect::() - .trim_matches('-') - .to_string() -} - -fn normalize_docker_arch(arch: &str) -> String { - match arch { - "x86_64" => "amd64".to_string(), - "aarch64" => "arm64".to_string(), - other => other.to_ascii_lowercase(), - } -} - -#[derive(Debug, Eq, PartialEq)] -enum SupervisorBinSource { - Binary(PathBuf), - Image(String), -} - -fn resolve_supervisor_bin_source( - docker_config: &DockerComputeConfig, - current_exe: Option<&Path>, - target_candidates: &[PathBuf], -) -> CoreResult { - // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. - if let Some(path) = docker_config.supervisor_bin.clone() { - let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path)?; - return Ok(SupervisorBinSource::Binary(path)); - } - - // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. - // A configured image should be the source of truth even when a local - // developer build is present under target/. - if let Some(image) = docker_config.supervisor_image.clone() { - return Ok(SupervisorBinSource::Image(image)); - } - - // Tier 3: sibling `openshell-sandbox` next to the running gateway - // (release artifact layout). Linux-only because the sibling must be a - // Linux ELF to bind-mount into a Linux container. - if cfg!(target_os = "linux") - && let Some(current_exe) = current_exe - && let Some(parent) = current_exe.parent() - { - let sibling = parent.join("openshell-sandbox"); - if sibling.is_file() { - let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 4: local cargo target build (developer workflow). Preferred - // over the default registry image when available because it matches - // whatever the developer just built. - for candidate in target_candidates { - if candidate.is_file() { - let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 5: pull the release-matched default supervisor image and extract - // the binary to a host-side cache keyed by image content digest. - Ok(SupervisorBinSource::Image( - openshell_core::config::default_supervisor_image(), - )) -} - -pub(crate) async fn resolve_supervisor_bin( - docker: &Docker, - docker_config: &DockerComputeConfig, - daemon_arch: &str, -) -> CoreResult { - let current_exe = - if cfg!(target_os = "linux") - && docker_config.supervisor_bin.is_none() - && docker_config.supervisor_image.is_none() - { - Some(std::env::current_exe().map_err(|err| { - Error::config(format!("failed to resolve current executable: {err}")) - })?) - } else { - None - }; - let target_candidates = linux_supervisor_candidates(daemon_arch); - - match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? - { - SupervisorBinSource::Binary(path) => Ok(path), - SupervisorBinSource::Image(image) => { - extract_supervisor_bin_from_image(docker, &image).await - } - } -} - -fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { - match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], - _ => Vec::new(), - } -} - -/// Pull the supervisor image (if not already local), extract -/// `/openshell-sandbox` to a host cache keyed by the image's content -/// digest, and return the cache path. -/// -/// The extraction is atomic: the binary is written to a sibling temp file -/// inside the digest-keyed directory and renamed into place, so concurrent -/// gateway starts don't observe a partial file. -async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { - let refresh_attempted = if supervisor_image_should_refresh(image) { - info!(image = image, "Refreshing mutable docker supervisor image"); - match pull_supervisor_image(docker, image).await { - Ok(()) => true, - Err(err) => { - warn!( - image = image, - error = %err, - "failed to refresh mutable docker supervisor image; falling back to local image if present", - ); - true - } - } - } else { - false - }; - - // Inspect first to see if the image is already present; only pull on miss. - let inspect = match docker.inspect_image(image).await { - Ok(inspect) => inspect, - Err(err) if is_not_found_error(&err) && !refresh_attempted => { - info!(image = image, "Pulling docker supervisor image"); - pull_supervisor_image(docker, image).await?; - docker.inspect_image(image).await.map_err(|err| { - Error::config(format!( - "failed to inspect docker supervisor image '{image}' after pull: {err}", - )) - })? - } - Err(err) if is_not_found_error(&err) => { - return Err(Error::config(format!( - "docker supervisor image '{image}' is not present locally after refresh attempt", - ))); - } - Err(err) => { - return Err(Error::config(format!( - "failed to inspect docker supervisor image '{image}': {err}", - ))); - } - }; - - let digest = inspect.id.clone().ok_or_else(|| { - Error::config(format!( - "docker supervisor image '{image}' inspect response has no Id", - )) - })?; - - let cache_path = supervisor_cache_path(&digest)?; - if cache_path.is_file() { - validate_linux_elf_binary(&cache_path)?; - return Ok(cache_path); - } - - let cache_dir = cache_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - cache_path.display(), - )) - })?; - std::fs::create_dir_all(cache_dir).map_err(|err| { - Error::config(format!( - "failed to create docker supervisor cache dir '{}': {err}", - cache_dir.display(), - )) - })?; - - info!( - image = image, - digest = digest, - cache_path = %cache_path.display(), - "Extracting supervisor binary from image to host cache", - ); - - let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes)?; - validate_linux_elf_binary(&cache_path)?; - Ok(cache_path) -} - -async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { - let mut stream = docker.create_image( - Some(CreateImageOptions { - from_image: Some(image.to_string()), - ..Default::default() - }), - None, - None, - ); - while let Some(result) = stream.next().await { - result.map_err(|err| { - Error::config(format!( - "failed to pull docker supervisor image '{image}': {err}", - )) - })?; - } - Ok(()) -} - -/// Create a short-lived container from `image`, stream out the supervisor -/// binary as a tar archive, and return the untarred file bytes. The -/// container is always removed, even on error paths. -async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { - let container_name = temp_extract_container_name(); - docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - ContainerCreateBody { - image: Some(image.to_string()), - entrypoint: Some(vec![SUPERVISOR_IMAGE_BINARY_PATH.to_string()]), - cmd: Some(Vec::new()), - ..Default::default() - }, - ) - .await - .map_err(|err| { - Error::config(format!( - "failed to create extractor container from '{image}': {err}", - )) - })?; - - // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; - if let Err(remove_err) = docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - warn!( - container = container_name, - error = %remove_err, - "Failed to remove supervisor extractor container", - ); - } - result -} - -async fn download_binary_from_container( - docker: &Docker, - container_name: &str, -) -> CoreResult> { - let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) - .build(); - let mut stream = docker.download_from_container(container_name, Some(options)); - - let mut tar_bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk: Bytes = chunk.map_err(|err| { - Error::config(format!( - "failed to read supervisor binary stream from '{container_name}': {err}", - )) - })?; - tar_bytes.extend_from_slice(&chunk); - } - - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) -} - -/// Extract the payload of the first regular-file entry in a tar archive. -/// Docker's `/containers//archive` endpoint returns a single-file tar -/// when `path` points to a file, so we only need the first entry. -fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { - let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let mut entries = archive - .entries() - .map_err(|err| format!("open tar archive: {err}"))?; - let mut entry = entries - .next() - .ok_or_else(|| "tar archive was empty".to_string())? - .map_err(|err| format!("read tar entry: {err}"))?; - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|err| format!("read tar entry payload: {err}"))?; - Ok(bytes) -} - -fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { - let dir = final_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - final_path.display(), - )) - })?; - let mut temp = tempfile::Builder::new() - .prefix(".openshell-sandbox-") - .tempfile_in(dir) - .map_err(|err| { - Error::config(format!( - "failed to create temp file for supervisor binary in '{}': {err}", - dir.display(), - )) - })?; - std::io::Write::write_all(&mut temp, bytes).map_err(|err| { - Error::config(format!( - "failed to write supervisor binary to temp file: {err}", - )) - })?; - temp.as_file().sync_all().map_err(|err| { - Error::config(format!("failed to sync supervisor binary temp file: {err}")) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( - |err| { - Error::config(format!( - "failed to chmod supervisor binary temp file: {err}", - )) - }, - )?; - } - - temp.persist(final_path).map_err(|err| { - Error::config(format!( - "failed to rename supervisor binary into '{}': {}", - final_path.display(), - err.error, - )) - })?; - Ok(()) -} - -/// Cache path for an extracted supervisor binary, keyed by the image's -/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed -/// directory keeps stale extractions from earlier releases isolated so they -/// can be GC'd without affecting the active binary. -fn supervisor_cache_path(digest: &str) -> CoreResult { - let base = openshell_core::paths::xdg_data_dir() - .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; - Ok(supervisor_cache_path_with_base(&base, digest)) -} - -fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { - let sanitized = digest.replace(':', "-"); - base.join("openshell") - .join("docker-supervisor") - .join(sanitized) - .join("openshell-sandbox") -} - -fn temp_extract_container_name() -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static SEQ: AtomicU64 = AtomicU64::new(0); - let pid = std::process::id(); - let seq = SEQ.fetch_add(1, Ordering::Relaxed); - format!("openshell-supervisor-extract-{pid}-{seq}") -} - -fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { - if !path.is_file() { - return Err(Error::config(format!( - "{description} '{}' does not exist or is not a file", - path.display() - ))); - } - std::fs::canonicalize(path).map_err(|err| { - Error::config(format!( - "failed to resolve {description} '{}': {err}", - path.display() - )) - }) -} - -pub(crate) fn validate_linux_elf_binary(path: &Path) -> CoreResult<()> { - let mut file = std::fs::File::open(path).map_err(|err| { - Error::config(format!( - "failed to open docker supervisor binary '{}': {err}", - path.display() - )) - })?; - let mut magic = [0_u8; 4]; - file.read_exact(&mut magic).map_err(|err| { - Error::config(format!( - "failed to read docker supervisor binary '{}': {err}", - path.display() - )) - })?; - if magic != [0x7f, b'E', b'L', b'F'] { - return Err(Error::config(format!( - "docker supervisor binary '{}' must be a Linux ELF executable", - path.display() - ))); - } - Ok(()) -} - -fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { - docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() -} - -pub(crate) fn docker_guest_tls_paths( - docker_config: &DockerComputeConfig, -) -> CoreResult> { - let tls_flags_provided = docker_config.guest_tls_ca.is_some() - || docker_config.guest_tls_cert.is_some() - || docker_config.guest_tls_key.is_some(); - - if !docker_config.grpc_endpoint.starts_with("https://") { - if tls_flags_provided { - return Err(Error::config(format!( - "guest_tls_ca/guest_tls_cert/guest_tls_key were provided but grpc_endpoint is '{}'; TLS materials require an https:// endpoint", - docker_config.grpc_endpoint, - ))); - } - return Ok(None); - } - - let provided = [ - docker_config.guest_tls_ca.as_ref(), - docker_config.guest_tls_cert.as_ref(), - docker_config.guest_tls_key.as_ref(), - ]; - if provided.iter().all(Option::is_none) { - return Err(Error::config( - "docker compute driver requires guest_tls_ca, guest_tls_cert, and guest_tls_key when grpc_endpoint uses https://", - )); - } - - let Some(ca) = docker_config.guest_tls_ca.clone() else { - return Err(Error::config( - "guest_tls_ca is required when Docker sandbox TLS materials are configured", - )); - }; - let Some(cert) = docker_config.guest_tls_cert.clone() else { - return Err(Error::config( - "guest_tls_cert is required when Docker sandbox TLS materials are configured", - )); - }; - let Some(key) = docker_config.guest_tls_key.clone() else { - return Err(Error::config( - "guest_tls_key is required when Docker sandbox TLS materials are configured", - )); - }; - - Ok(Some(DockerGuestTlsPaths { - ca: canonicalize_existing_file(&ca, "docker TLS CA certificate")?, - cert: canonicalize_existing_file(&cert, "docker TLS client certificate")?, - key: canonicalize_existing_file(&key, "docker TLS client private key")?, - })) -} - -fn is_not_found_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 404, - .. - } - ) -} - -fn is_conflict_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) -} - -fn is_not_modified_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 304, - .. - } - ) -} - -fn create_status_from_docker_error(operation: &str, err: BollardError) -> Status { - if matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) { - Status::already_exists("sandbox already exists") - } else { - internal_status(operation, err) - } -} - -fn internal_status(operation: &str, err: BollardError) -> Status { - Status::internal(format!("{operation} failed: {err}")) -} - -#[cfg(test)] -mod tests; diff --git a/crates/openshell-driver-docker/src/lib_win.rs b/crates/openshell-driver-docker/src/lib_win.rs new file mode 100644 index 0000000000..343ce7c585 --- /dev/null +++ b/crates/openshell-driver-docker/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod windows; + +pub use windows::{DockerComputeConfig, default_docker_supervisor_image}; diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index e72ac4f974..541f526198 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-driver-kubernetes" +autobins = false description = "Kubernetes compute driver for OpenShell" version.workspace = true edition.workspace = true @@ -12,7 +13,7 @@ repository.workspace = true [[bin]] name = "openshell-driver-kubernetes" -path = "src/main.rs" +path = "src/main_entry.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index c151df22c3..c733b8a45b 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -1,11 +1,181 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +use clap::{ArgAction, Parser}; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use tracing::info; +use tracing_subscriber::EnvFilter; -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-driver-kubernetes is unsupported on Windows"); - std::process::exit(1); +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_kubernetes::{ + AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, + SupervisorSideloadMethod, SupervisorTopology, +}; + +#[derive(Parser, Debug)] +#[command(name = "openshell-driver-kubernetes")] +#[command(version = VERSION)] +struct Args { + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] + sandbox_namespace: String, + + #[arg( + long, + env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", + default_value = DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME + )] + sandbox_service_account: String, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] + sandbox_image_pull_policy: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_IMAGE_PULL_SECRETS", + value_delimiter = ',' + )] + sandbox_image_pull_secrets: Vec, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = "/run/openshell/ssh.sock" + )] + sandbox_ssh_socket_path: String, + + #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] + client_tls_secret_name: Option, + + #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] + host_gateway_ip: Option, + + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + supervisor_image: Option, + + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] + supervisor_image_pull_policy: Option, + + #[arg( + long, + env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", + default_value = "image-volume" + )] + supervisor_sideload_method: SupervisorSideloadMethod, + + #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] + topology: SupervisorTopology, + + #[arg( + long = "sidecar-proxy-uid", + alias = "proxy-uid", + env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", + default_value_t = DEFAULT_PROXY_UID + )] + sidecar_proxy_uid: u32, + + #[arg( + long = "sidecar-process-binary-aware-network-policy", + env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", + default_value_t = true, + action = ArgAction::Set + )] + sidecar_process_binary_aware_network_policy: bool, + + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] + enable_user_namespaces: bool, + + #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] + app_armor_profile: Option, + + /// Lifetime (seconds) of the projected `ServiceAccount` token + /// kubelet writes into each sandbox pod for the `IssueSandboxToken` + /// bootstrap exchange. Kubelet enforces a minimum of 600s; the + /// gateway clamps values outside `[600, 86400]`. Default 3600. + #[arg(long, env = "OPENSHELL_K8S_SA_TOKEN_TTL_SECS", default_value_t = 3600)] + sa_token_ttl_secs: i64, + + #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] + provider_spiffe_workload_api_socket_path: Option, + + #[arg(long, env = "OPENSHELL_K8S_SANDBOX_UID")] + sandbox_uid: Option, + + #[arg(long, env = "OPENSHELL_K8S_SANDBOX_GID")] + sandbox_gid: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { + namespace: args.sandbox_namespace, + service_account_name: args.sandbox_service_account, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_secrets: args.sandbox_image_pull_secrets, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_sideload_method: args.supervisor_sideload_method, + topology: args.topology, + sidecar: KubernetesSidecarConfig { + proxy_uid: args.sidecar_proxy_uid, + process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, + }, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + ssh_socket_path: args.sandbox_ssh_socket_path, + client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), + host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), + enable_user_namespaces: args.enable_user_namespaces, + app_armor_profile: args.app_armor_profile, + workspace_default_storage_size: std::env::var( + "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", + ) + .unwrap_or_else(|_| { + openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() + }), + default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") + .unwrap_or_default(), + sa_token_ttl_secs: args.sa_token_ttl_secs, + provider_spiffe_workload_api_socket_path: args + .provider_spiffe_workload_api_socket_path + .unwrap_or_default(), + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, + }) + .await + .into_diagnostic()?; + + info!(address = %args.bind_address, "Starting Kubernetes compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) + .serve(args.bind_address) + .await + .into_diagnostic() } diff --git a/crates/openshell-driver-kubernetes/src/main_entry.rs b/crates/openshell-driver-kubernetes/src/main_entry.rs new file mode 100644 index 0000000000..432402121a --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-driver-kubernetes/src/main_unix.rs b/crates/openshell-driver-kubernetes/src/main_unix.rs deleted file mode 100644 index c733b8a45b..0000000000 --- a/crates/openshell-driver-kubernetes/src/main_unix.rs +++ /dev/null @@ -1,181 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use clap::{ArgAction, Parser}; -use miette::{IntoDiagnostic, Result}; -use std::net::SocketAddr; -use tracing::info; -use tracing_subscriber::EnvFilter; - -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, -}; - -#[derive(Parser, Debug)] -#[command(name = "openshell-driver-kubernetes")] -#[command(version = VERSION)] -struct Args { - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_BIND", - default_value = "127.0.0.1:50061" - )] - bind_address: SocketAddr, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] - sandbox_namespace: String, - - #[arg( - long, - env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", - default_value = DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME - )] - sandbox_service_account: String, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_IMAGE_PULL_SECRETS", - value_delimiter = ',' - )] - sandbox_image_pull_secrets: Vec, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", - default_value = "/run/openshell/ssh.sock" - )] - sandbox_ssh_socket_path: String, - - #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] - client_tls_secret_name: Option, - - #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: Option, - - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, - - #[arg( - long, - env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", - default_value = "image-volume" - )] - supervisor_sideload_method: SupervisorSideloadMethod, - - #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] - topology: SupervisorTopology, - - #[arg( - long = "sidecar-proxy-uid", - alias = "proxy-uid", - env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", - default_value_t = DEFAULT_PROXY_UID - )] - sidecar_proxy_uid: u32, - - #[arg( - long = "sidecar-process-binary-aware-network-policy", - env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", - default_value_t = true, - action = ArgAction::Set - )] - sidecar_process_binary_aware_network_policy: bool, - - #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] - enable_user_namespaces: bool, - - #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] - app_armor_profile: Option, - - /// Lifetime (seconds) of the projected `ServiceAccount` token - /// kubelet writes into each sandbox pod for the `IssueSandboxToken` - /// bootstrap exchange. Kubelet enforces a minimum of 600s; the - /// gateway clamps values outside `[600, 86400]`. Default 3600. - #[arg(long, env = "OPENSHELL_K8S_SA_TOKEN_TTL_SECS", default_value_t = 3600)] - sa_token_ttl_secs: i64, - - #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] - provider_spiffe_workload_api_socket_path: Option, - - #[arg(long, env = "OPENSHELL_K8S_SANDBOX_UID")] - sandbox_uid: Option, - - #[arg(long, env = "OPENSHELL_K8S_SANDBOX_GID")] - sandbox_gid: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { - namespace: args.sandbox_namespace, - service_account_name: args.sandbox_service_account, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), - image_pull_secrets: args.sandbox_image_pull_secrets, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, - }, - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - ssh_socket_path: args.sandbox_ssh_socket_path, - client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), - host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), - enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, - workspace_default_storage_size: std::env::var( - "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", - ) - .unwrap_or_else(|_| { - openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() - }), - default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") - .unwrap_or_default(), - sa_token_ttl_secs: args.sa_token_ttl_secs, - provider_spiffe_workload_api_socket_path: args - .provider_spiffe_workload_api_socket_path - .unwrap_or_default(), - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) - .await - .into_diagnostic()?; - - info!(address = %args.bind_address, "Starting Kubernetes compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve(args.bind_address) - .await - .into_diagnostic() -} diff --git a/crates/openshell-driver-kubernetes/src/main_win.rs b/crates/openshell-driver-kubernetes/src/main_win.rs new file mode 100644 index 0000000000..56cc95ac04 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-driver-kubernetes is unsupported on Windows"); + std::process::exit(1); +} diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 90a5e6d761..9d490d4e9b 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-driver-podman" +autobins = false description = "Podman compute driver for OpenShell" version.workspace = true edition.workspace = true @@ -10,9 +11,12 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +path = "src/lib_entry.rs" + [[bin]] name = "openshell-driver-podman" -path = "src/main.rs" +path = "src/main_entry.rs" [target.'cfg(target_os = "windows")'.dependencies] serde = { workspace = true } diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index b76c9e0fb3..5847a10ea6 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -1,11 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -mod windows; +pub(crate) mod client; +pub mod config; +pub(crate) mod container; +pub mod driver; +pub mod grpc; +#[cfg(test)] +pub(crate) mod test_utils; +pub(crate) mod watcher; -#[cfg(target_os = "windows")] -pub use windows::PodmanComputeConfig; - -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +pub use config::PodmanComputeConfig; +pub use driver::PodmanComputeDriver; +pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-podman/src/lib_entry.rs b/crates/openshell-driver-podman/src/lib_entry.rs new file mode 100644 index 0000000000..4272e7c537 --- /dev/null +++ b/crates/openshell-driver-podman/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-driver-podman/src/lib_unix.rs b/crates/openshell-driver-podman/src/lib_unix.rs deleted file mode 100644 index 5847a10ea6..0000000000 --- a/crates/openshell-driver-podman/src/lib_unix.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) mod client; -pub mod config; -pub(crate) mod container; -pub mod driver; -pub mod grpc; -#[cfg(test)] -pub(crate) mod test_utils; -pub(crate) mod watcher; - -pub use config::PodmanComputeConfig; -pub use driver::PodmanComputeDriver; -pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-podman/src/lib_win.rs b/crates/openshell-driver-podman/src/lib_win.rs new file mode 100644 index 0000000000..514eed1bdd --- /dev/null +++ b/crates/openshell-driver-podman/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod windows; + +pub use windows::PodmanComputeConfig; diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 42060fd75c..4a38643f38 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -1,11 +1,185 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-driver-podman is unsupported on Windows"); - std::process::exit(1); +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use std::path::PathBuf; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_podman::config::{ + DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, + ImagePullPolicy, +}; +use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; + +#[derive(Parser)] +#[command(name = "openshell-driver-podman")] +#[command(version = VERSION)] +struct Args { + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + /// Path to the Podman API Unix socket. + #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] + podman_socket: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", + default_value_t = ImagePullPolicy::Missing + )] + sandbox_image_pull_policy: ImagePullPolicy, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + /// Port the gateway server is listening on. + /// + /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint + /// that sandbox containers dial back to. + #[arg( + long, + env = "OPENSHELL_GATEWAY_PORT", + default_value_t = openshell_core::config::DEFAULT_SERVER_PORT + )] + gateway_port: u16, + + /// Host gateway IP used for sandbox host aliases. + /// + /// Empty uses Podman's `host-gateway` resolver. + #[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")] + host_gateway_ip: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = "/run/openshell/ssh.sock" + )] + sandbox_ssh_socket_path: String, + + /// Podman bridge network name. + #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] + network_name: String, + + /// Container stop timeout in seconds (SIGTERM → SIGKILL). + #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] + stop_timeout: u32, + + /// Container cgroup PID limit for sandbox containers. Set 0 to inherit + /// Podman's runtime/default PID limit. + #[arg( + long, + env = "OPENSHELL_SANDBOX_PIDS_LIMIT", + default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + )] + sandbox_pids_limit: i64, + + /// OCI image containing the openshell-sandbox supervisor binary. + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + supervisor_image: Option, + + /// Host path to the CA certificate for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] + podman_tls_ca: Option, + + /// Host path to the client certificate for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")] + podman_tls_cert: Option, + + /// Host path to the client private key for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] + podman_tls_key: Option, + + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, + /// in explicit `http://host:port` form (scheme and port required). + /// Credentials must not be embedded in the URL; use + /// `--sandbox-proxy-auth-file` instead. + #[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")] + sandbox_https_proxy: Option, + + /// Comma-separated `NO_PROXY` list injected alongside the proxy URL. + #[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")] + sandbox_no_proxy: Option, + + /// Path to a file containing the corporate proxy credentials as + /// `user:pass`. Delivered to the supervisor through a root-only secret + /// mount so the credentials never appear in config or container metadata. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")] + sandbox_proxy_auth_file: Option, + + /// Explicit acknowledgement (`true`) that the proxy credential is sent + /// as cleartext Basic auth over the plain-TCP connection to the http:// + /// proxy. Required when `--sandbox-proxy-auth-file` is set. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")] + sandbox_proxy_auth_allow_insecure: Option, + + /// Send the destination hostname in CONNECT requests to the corporate + /// proxy instead of a validated IP. Only for proxies whose ACLs filter + /// on hostnames: the proxy then resolves the name itself, so sandbox + /// SSRF/`allowed_ips` validation no longer binds the connection. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] + sandbox_proxy_connect_by_hostname: Option, } -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let driver = PodmanComputeDriver::new(PodmanComputeConfig { + socket_path: args.podman_socket, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + gateway_port: args.gateway_port, + host_gateway_ip: args + .host_gateway_ip + .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), + sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + network_name: args.network_name, + stop_timeout_secs: args.stop_timeout, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + guest_tls_ca: args.podman_tls_ca, + guest_tls_cert: args.podman_tls_cert, + guest_tls_key: args.podman_tls_key, + sandbox_pids_limit: args.sandbox_pids_limit, + https_proxy: args.sandbox_https_proxy, + no_proxy: args.sandbox_no_proxy, + proxy_auth_file: args.sandbox_proxy_auth_file, + proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, + ..PodmanComputeConfig::default() + }) + .await + .into_diagnostic()?; + + info!(address = %args.bind_address, "Starting Podman compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) + .serve_with_shutdown(args.bind_address, async { + tokio::signal::ctrl_c().await.ok(); + info!("Received shutdown signal, draining in-flight requests"); + }) + .await + .into_diagnostic() +} diff --git a/crates/openshell-driver-podman/src/main_entry.rs b/crates/openshell-driver-podman/src/main_entry.rs new file mode 100644 index 0000000000..432402121a --- /dev/null +++ b/crates/openshell-driver-podman/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-driver-podman/src/main_unix.rs b/crates/openshell-driver-podman/src/main_unix.rs deleted file mode 100644 index 4a38643f38..0000000000 --- a/crates/openshell-driver-podman/src/main_unix.rs +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use std::net::SocketAddr; -use std::path::PathBuf; -use tracing::info; -use tracing_subscriber::EnvFilter; - -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; -use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; - -#[derive(Parser)] -#[command(name = "openshell-driver-podman")] -#[command(version = VERSION)] -struct Args { - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_BIND", - default_value = "127.0.0.1:50061" - )] - bind_address: SocketAddr, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - /// Path to the Podman API Unix socket. - #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] - podman_socket: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing - )] - sandbox_image_pull_policy: ImagePullPolicy, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - /// Port the gateway server is listening on. - /// - /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint - /// that sandbox containers dial back to. - #[arg( - long, - env = "OPENSHELL_GATEWAY_PORT", - default_value_t = openshell_core::config::DEFAULT_SERVER_PORT - )] - gateway_port: u16, - - /// Host gateway IP used for sandbox host aliases. - /// - /// Empty uses Podman's `host-gateway` resolver. - #[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", - default_value = "/run/openshell/ssh.sock" - )] - sandbox_ssh_socket_path: String, - - /// Podman bridge network name. - #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] - network_name: String, - - /// Container stop timeout in seconds (SIGTERM → SIGKILL). - #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] - stop_timeout: u32, - - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit - /// Podman's runtime/default PID limit. - #[arg( - long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT - )] - sandbox_pids_limit: i64, - - /// OCI image containing the openshell-sandbox supervisor binary. - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: Option, - - /// Host path to the CA certificate for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] - podman_tls_ca: Option, - - /// Host path to the client certificate for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")] - podman_tls_cert: Option, - - /// Host path to the client private key for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] - podman_tls_key: Option, - - /// Corporate forward proxy URL for the supervisor's upstream TLS dials, - /// in explicit `http://host:port` form (scheme and port required). - /// Credentials must not be embedded in the URL; use - /// `--sandbox-proxy-auth-file` instead. - #[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")] - sandbox_https_proxy: Option, - - /// Comma-separated `NO_PROXY` list injected alongside the proxy URL. - #[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")] - sandbox_no_proxy: Option, - - /// Path to a file containing the corporate proxy credentials as - /// `user:pass`. Delivered to the supervisor through a root-only secret - /// mount so the credentials never appear in config or container metadata. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")] - sandbox_proxy_auth_file: Option, - - /// Explicit acknowledgement (`true`) that the proxy credential is sent - /// as cleartext Basic auth over the plain-TCP connection to the http:// - /// proxy. Required when `--sandbox-proxy-auth-file` is set. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")] - sandbox_proxy_auth_allow_insecure: Option, - - /// Send the destination hostname in CONNECT requests to the corporate - /// proxy instead of a validated IP. Only for proxies whose ACLs filter - /// on hostnames: the proxy then resolves the name itself, so sandbox - /// SSRF/`allowed_ips` validation no longer binds the connection. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] - sandbox_proxy_connect_by_hostname: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let driver = PodmanComputeDriver::new(PodmanComputeConfig { - socket_path: args.podman_socket, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy, - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - gateway_port: args.gateway_port, - host_gateway_ip: args - .host_gateway_ip - .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, - network_name: args.network_name, - stop_timeout_secs: args.stop_timeout, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - guest_tls_ca: args.podman_tls_ca, - guest_tls_cert: args.podman_tls_cert, - guest_tls_key: args.podman_tls_key, - sandbox_pids_limit: args.sandbox_pids_limit, - https_proxy: args.sandbox_https_proxy, - no_proxy: args.sandbox_no_proxy, - proxy_auth_file: args.sandbox_proxy_auth_file, - proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, - ..PodmanComputeConfig::default() - }) - .await - .into_diagnostic()?; - - info!(address = %args.bind_address, "Starting Podman compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async { - tokio::signal::ctrl_c().await.ok(); - info!("Received shutdown signal, draining in-flight requests"); - }) - .await - .into_diagnostic() -} diff --git a/crates/openshell-driver-podman/src/main_win.rs b/crates/openshell-driver-podman/src/main_win.rs new file mode 100644 index 0000000000..527dacccb2 --- /dev/null +++ b/crates/openshell-driver-podman/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-driver-podman is unsupported on Windows"); + std::process::exit(1); +} diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 0adf7196d7..ba1b118340 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-driver-vm" +autobins = false description = "MicroVM compute driver for OpenShell" version.workspace = true edition.workspace = true @@ -12,11 +13,11 @@ repository.workspace = true [lib] name = "openshell_driver_vm" -path = "src/lib.rs" +path = "src/lib_entry.rs" [[bin]] name = "openshell-driver-vm" -path = "src/main.rs" +path = "src/main_entry.rs" [target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index e0d663bc44..88e2c3b201 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -1,10 +1,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -pub fn windows_unsupported() -> &'static str { - "openshell-driver-vm is unsupported on Windows" -} +pub mod driver; +mod embedded_runtime; +mod ffi; +pub mod gpu; +pub mod lifecycle; +mod nft_ruleset; +pub mod procguard; +mod rootfs; +mod runtime; -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +pub use driver::{VmDriver, VmDriverConfig}; +pub use lifecycle::{ + BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, + LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, + RestoreContext, +}; +pub use runtime::{ + VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, + configured_runtime_dir, run_vm, +}; diff --git a/crates/openshell-driver-vm/src/lib_entry.rs b/crates/openshell-driver-vm/src/lib_entry.rs new file mode 100644 index 0000000000..4272e7c537 --- /dev/null +++ b/crates/openshell-driver-vm/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-driver-vm/src/lib_unix.rs b/crates/openshell-driver-vm/src/lib_unix.rs deleted file mode 100644 index 88e2c3b201..0000000000 --- a/crates/openshell-driver-vm/src/lib_unix.rs +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -pub mod driver; -mod embedded_runtime; -mod ffi; -pub mod gpu; -pub mod lifecycle; -mod nft_ruleset; -pub mod procguard; -mod rootfs; -mod runtime; - -pub use driver::{VmDriver, VmDriverConfig}; -pub use lifecycle::{ - BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, - LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, - RestoreContext, -}; -pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, - configured_runtime_dir, run_vm, -}; diff --git a/crates/openshell-driver-vm/src/lib_win.rs b/crates/openshell-driver-vm/src/lib_win.rs new file mode 100644 index 0000000000..55f725ceec --- /dev/null +++ b/crates/openshell-driver-vm/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub fn windows_unsupported() -> &'static str { + "openshell-driver-vm is unsupported on Windows" +} diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index c41b94f247..0ae694effa 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -1,11 +1,728 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-driver-vm is unsupported on Windows"); - std::process::exit(1); +use clap::Parser; +use futures::Stream; +use miette::{IntoDiagnostic, Result}; +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +#[cfg(target_os = "macos")] +use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; +use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; +use std::io; +use std::net::SocketAddr; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::net::{UnixListener, UnixStream}; +use tracing::info; +use tracing_subscriber::EnvFilter; + +#[derive(Parser, Debug)] +#[command(name = "openshell-driver-vm")] +#[command(version = VERSION)] +#[allow(clippy::struct_excessive_bools)] +struct Args { + #[arg(long, hide = true, default_value_t = false)] + internal_run_vm: bool, + + #[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")] + vm_root_disk: Option, + + #[arg(long = "vm-overlay-disk", hide = true)] + vm_overlay_disk: Option, + + #[arg(long = "vm-image-disk", hide = true)] + vm_image_disk: Option, + + #[arg(long = "vm-kernel-image", hide = true)] + vm_kernel_image: Option, + + #[arg(long, hide = true)] + vm_exec: Option, + + #[arg(long, hide = true, default_value = "/")] + vm_workdir: String, + + #[arg(long, hide = true)] + vm_env: Vec, + + #[arg(long, hide = true)] + vm_console_output: Option, + + #[arg(long, hide = true, default_value_t = 2)] + vm_vcpus: u8, + + #[arg(long, hide = true, default_value_t = 2048)] + vm_mem_mib: u32, + + #[arg(long, hide = true, default_value_t = 1)] + vm_krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")] + bind_address: Option, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + + #[arg(long, hide = true)] + expected_peer_pid: Option, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP", + default_value_t = false + )] + allow_unauthenticated_tcp: bool, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER", + default_value_t = false + )] + allow_same_uid_peer: bool, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + openshell_endpoint: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] + default_image: String, + + #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] + bootstrap_image: String, + + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_STATE_DIR", + default_value = "target/openshell-vm-driver" + )] + state_dir: PathBuf, + + #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] + guest_tls_ca: Option, + + #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] + guest_tls_cert: Option, + + #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] + guest_tls_key: Option, + + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] + krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] + vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] + mem_mib: u32, + + #[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)] + overlay_disk_mib: u64, + + #[arg(long, env = "OPENSHELL_VM_GPU")] + gpu: bool, + + #[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)] + gpu_mem_mib: u32, + + #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] + gpu_vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")] + sandbox_uid: Option, + + #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] + sandbox_gid: Option, + + #[arg(long, hide = true)] + vm_backend: Option, + + #[arg(long, hide = true)] + vm_gpu_bdf: Option, + + #[arg(long, hide = true)] + vm_tap_device: Option, + + #[arg(long, hide = true)] + vm_guest_ip: Option, + + #[arg(long, hide = true)] + vm_host_ip: Option, + + #[arg(long, hide = true)] + vm_vsock_cid: Option, + + #[arg(long, hide = true)] + vm_guest_mac: Option, + + #[arg(long, hide = true)] + vm_gateway_port: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + if args.internal_run_vm { + // We intentionally defer procguard arming until `run_vm()` so + // that the only arm is the one that knows how to clean up + // gvproxy. Racing two watchers against the same parent-death + // event causes the bare arm's `exit(1)` to win, skipping the + // gvproxy cleanup and leaking the helper. The risk window + // before `run_vm` arms procguard is ~a few syscalls long + // (`build_vm_launch_config`, `configured_runtime_dir`), which + // is negligible next to the parent gRPC server's uptime. + maybe_reexec_internal_vm_with_runtime_env()?; + let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; + run_vm(&config).map_err(|err| miette::miette!("{err}"))?; + return Ok(()); + } + + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; + + // Arm procguard so that if the gateway is killed (SIGKILL or crash) + // we also die. Without this the driver is reparented to init and + // keeps its per-sandbox VM launchers alive forever. Launchers have + // their own procguards (armed in `run_vm`) which cascade cleanup of + // gvproxy and the libkrun worker the moment this driver exits. + if let Err(err) = procguard::die_with_parent() { + tracing::warn!( + error = %err, + "procguard arm failed; gateway crashes may orphan this driver" + ); + } + + let driver = VmDriver::new(VmDriverConfig { + openshell_endpoint: args + .openshell_endpoint + .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, + state_dir: args.state_dir.clone(), + launcher_bin: None, + default_image: args.default_image.clone(), + bootstrap_image: args.bootstrap_image.clone(), + log_level: args.log_level.clone(), + krun_log_level: args.krun_log_level, + vcpus: args.vcpus, + mem_mib: args.mem_mib, + overlay_disk_mib: args.overlay_disk_mib, + guest_tls_ca: args.guest_tls_ca.clone(), + guest_tls_cert: args.guest_tls_cert.clone(), + guest_tls_key: args.guest_tls_key.clone(), + gpu_enabled: args.gpu, + gpu_mem_mib: args.gpu_mem_mib, + gpu_vcpus: args.gpu_vcpus, + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, + }) + .await + .map_err(|err| miette::miette!("{err}"))?; + + match listen_mode { + ComputeDriverListenMode::Unix { + socket_path, + expected_peer_pid, + } => { + prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?; + + info!(socket = %socket_path.display(), "Starting vm compute driver"); + let listener = UnixListener::bind(&socket_path).into_diagnostic()?; + restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?; + let result = tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming(AuthenticatedUnixIncoming::new(listener, expected_peer_pid)) + .await + .into_diagnostic(); + let _ = std::fs::remove_file(&socket_path); + result + } + ComputeDriverListenMode::Tcp(bind_address) => { + info!(address = %bind_address, "Starting unauthenticated dev vm compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve(bind_address) + .await + .into_diagnostic() + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum ComputeDriverListenMode { + Unix { + socket_path: PathBuf, + expected_peer_pid: Option, + }, + Tcp(SocketAddr), +} + +fn compute_driver_listen_mode(args: &Args) -> std::result::Result { + if let Some(socket_path) = args.bind_socket.clone() { + if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer { + return Err( + "--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development" + .to_string(), + ); + } + return Ok(ComputeDriverListenMode::Unix { + socket_path, + expected_peer_pid: args.expected_peer_pid, + }); + } + + if !args.allow_unauthenticated_tcp { + return Err( + "--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development" + .to_string(), + ); + } + + let Some(bind_address) = args.bind_address else { + return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string()); + }; + + Ok(ComputeDriverListenMode::Tcp(bind_address)) +} + +fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> { + let Some(parent) = socket_path.parent() else { + return Err(format!( + "vm compute driver socket path '{}' has no parent directory", + socket_path.display() + )); + }; + let expected_uid = current_euid(); + prepare_private_socket_dir(parent, expected_uid)?; + remove_stale_socket(socket_path, expected_uid) +} + +fn current_euid() -> u32 { + rustix::process::geteuid().as_raw() +} + +fn prepare_private_socket_dir( + socket_dir: &Path, + expected_uid: u32, +) -> std::result::Result<(), String> { + std::fs::create_dir_all(socket_dir) + .map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?; + let metadata = std::fs::symlink_metadata(socket_dir) + .map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(format!( + "socket dir {} is a symlink; refusing to use it", + socket_dir.display() + )); + } + if !file_type.is_dir() { + return Err(format!( + "socket dir {} is not a directory", + socket_dir.display() + )); + } + if metadata.uid() != expected_uid { + return Err(format!( + "socket dir {} is owned by uid {} but current euid is {}", + socket_dir.display(), + metadata.uid(), + expected_uid + )); + } + std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display())) +} + +fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> { + let metadata = match std::fs::symlink_metadata(socket_path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())), + }; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(format!( + "socket {} is a symlink; refusing to remove it", + socket_path.display() + )); + } + if metadata.uid() != expected_uid { + return Err(format!( + "socket {} is owned by uid {} but current euid is {}", + socket_path.display(), + metadata.uid(), + expected_uid + )); + } + if !file_type.is_socket() { + return Err(format!( + "socket path {} exists but is not a Unix socket", + socket_path.display() + )); + } + std::fs::remove_file(socket_path) + .map_err(|err| format!("remove stale socket {}: {err}", socket_path.display())) +} + +fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> { + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) + .map_err(|err| format!("chmod socket {}: {err}", socket_path.display())) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PeerCredentials { + uid: u32, + pid: Option, +} + +fn peer_credentials(stream: &UnixStream) -> std::result::Result { + let credentials = stream + .peer_cred() + .map_err(|err| format!("read peer credentials: {err}"))?; + Ok(PeerCredentials { + uid: credentials.uid(), + pid: credentials.pid(), + }) +} + +fn authorize_peer_credentials( + peer: PeerCredentials, + driver_uid: u32, + gateway_pid: Option, +) -> std::result::Result<(), String> { + if peer.uid != driver_uid { + return Err(format!( + "peer uid {} does not match current euid {}", + peer.uid, driver_uid + )); + } + let Some(gateway_pid) = gateway_pid else { + return Ok(()); + }; + let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else { + return Err(format!( + "peer pid is unavailable; expected gateway pid {gateway_pid}" + )); + }; + if peer_process_id != gateway_pid { + return Err(format!( + "peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}" + )); + } + Ok(()) +} + +struct AuthenticatedUnixIncoming { + listener: UnixListener, + expected_uid: u32, + expected_peer_pid: Option, +} + +impl AuthenticatedUnixIncoming { + fn new(listener: UnixListener, expected_peer_pid: Option) -> Self { + Self { + listener, + expected_uid: current_euid(), + expected_peer_pid, + } + } +} + +impl Stream for AuthenticatedUnixIncoming { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.listener.poll_accept(cx) { + Poll::Ready(Ok((stream, _addr))) => { + let authorized = peer_credentials(&stream).and_then(|peer| { + authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid) + }); + match authorized { + Ok(()) => return Poll::Ready(Some(Ok(stream))), + Err(err) => { + tracing::warn!( + error = %err, + "rejected vm compute driver UDS client" + ); + } + } + } + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + Poll::Pending => return Poll::Pending, + } + } + } } -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +fn build_vm_launch_config(args: &Args) -> std::result::Result { + let root_disk = args + .vm_root_disk + .clone() + .ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?; + let overlay_disk = args + .vm_overlay_disk + .clone() + .ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?; + let image_disk = args.vm_image_disk.clone(); + let exec_path = args + .vm_exec + .clone() + .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; + let console_output = args + .vm_console_output + .clone() + .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; + + let backend = match args.vm_backend.as_deref() { + Some("qemu") => VmBackend::Qemu, + Some("libkrun") | None => VmBackend::Libkrun, + Some(other) => return Err(format!("unknown VM backend: {other}")), + }; + + Ok(VmLaunchConfig { + root_disk, + overlay_disk, + image_disk, + kernel_image: args.vm_kernel_image.clone(), + vcpus: args.vm_vcpus, + mem_mib: args.vm_mem_mib, + exec_path, + args: Vec::new(), + env: args.vm_env.clone(), + workdir: args.vm_workdir.clone(), + log_level: args.vm_krun_log_level, + console_output, + backend, + gpu_bdf: args.vm_gpu_bdf.clone(), + tap_device: args.vm_tap_device.clone(), + guest_ip: args.vm_guest_ip.clone(), + host_ip: args.vm_host_ip.clone(), + vsock_cid: args.vm_vsock_cid, + guest_mac: args.vm_guest_mac.clone(), + gateway_port: args.vm_gateway_port, + }) +} + +#[cfg(target_os = "macos")] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + use std::os::unix::process::CommandExt as _; + + const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; + + if std::env::var_os(REEXEC_ENV).is_some() { + return Ok(()); + } + + let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; + let runtime_str = runtime_dir.to_string_lossy(); + let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") + .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); + if !needs_reexec { + return Ok(()); + } + + let mut dyld_paths = vec![runtime_dir.clone()]; + if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { + dyld_paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(&dyld_paths) + .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; + let exe = std::env::current_exe().into_diagnostic()?; + let args: Vec = std::env::args().skip(1).collect(); + + // Use execvp() so the current process is *replaced* by the re-exec'd + // binary — no wrapper process sits between the compute driver and + // the actually-running VM launcher. That avoids two problems: + // 1. An extra process level that survives SIGKILL of the driver + // (the wrapper was reparenting the re-exec'd child to init). + // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper + // doesn't reach the child unless we hand-roll forwarding. + // After exec, the child inherits our PID and our procguard arming. + let err = std::process::Command::new(exe) + .args(&args) + .env("DYLD_LIBRARY_PATH", &joined) + .env(VM_RUNTIME_DIR_ENV, runtime_dir) + .env(REEXEC_ENV, "1") + .exec(); + // `exec()` only returns on failure. + Err(miette::miette!("failed to re-exec with runtime env: {err}")) +} + +#[cfg(not(target_os = "macos"))] +// Signature must match the macOS variant which can fail. +#[allow(clippy::unnecessary_wraps)] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, + compute_driver_listen_mode, + }; + use clap::Parser; + use std::path::PathBuf; + + #[test] + fn peer_authorization_accepts_matching_uid_and_pid() { + authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: Some(42), + }, + 1000, + Some(42), + ) + .unwrap(); + } + + #[test] + fn peer_authorization_rejects_wrong_pid() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: Some(7), + }, + 1000, + Some(42), + ) + .expect_err("wrong pid should be rejected"); + assert!(err.contains("does not match expected gateway pid")); + } + + #[test] + fn peer_authorization_rejects_wrong_uid() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1001, + pid: Some(42), + }, + 1000, + Some(42), + ) + .expect_err("wrong uid should be rejected"); + assert!(err.contains("does not match current euid")); + } + + #[test] + fn peer_authorization_rejects_missing_pid_when_expected() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: None, + }, + 1000, + Some(42), + ) + .expect_err("missing pid should be rejected"); + assert!(err.contains("peer pid is unavailable")); + } + + #[test] + fn peer_authorization_accepts_matching_uid_without_expected_pid() { + authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: None, + }, + 1000, + None, + ) + .unwrap(); + } + + #[test] + fn listen_mode_rejects_default_tcp() { + let args = Args::parse_from(["openshell-driver-vm"]); + let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled"); + assert!(err.contains("--bind-socket is required")); + } + + #[test] + fn listen_mode_rejects_bind_address_without_tcp_opt_in() { + let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); + let err = + compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in"); + assert!(err.contains("--allow-unauthenticated-tcp")); + } + + #[test] + fn listen_mode_requires_bind_address_with_tcp_opt_in() { + let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]); + let err = + compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address"); + assert!(err.contains("--bind-address is required")); + } + + #[test] + fn listen_mode_accepts_explicit_unauthenticated_tcp() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--allow-unauthenticated-tcp", + "--bind-address", + "127.0.0.1:50061", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap()) + ); + } + + #[test] + fn listen_mode_requires_expected_peer_pid_for_uds() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + ]); + let err = compute_driver_listen_mode(&args) + .expect_err("UDS should require gateway peer pid by default"); + assert!(err.contains("--expected-peer-pid is required")); + } + + #[test] + fn listen_mode_accepts_uds_with_expected_peer_pid() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + "--expected-peer-pid", + "42", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Unix { + socket_path: PathBuf::from("/tmp/compute-driver.sock"), + expected_peer_pid: Some(42), + } + ); + } + + #[test] + fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + "--allow-same-uid-peer", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Unix { + socket_path: PathBuf::from("/tmp/compute-driver.sock"), + expected_peer_pid: None, + } + ); + } +} diff --git a/crates/openshell-driver-vm/src/main_entry.rs b/crates/openshell-driver-vm/src/main_entry.rs new file mode 100644 index 0000000000..432402121a --- /dev/null +++ b/crates/openshell-driver-vm/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-driver-vm/src/main_unix.rs b/crates/openshell-driver-vm/src/main_unix.rs deleted file mode 100644 index 0ae694effa..0000000000 --- a/crates/openshell-driver-vm/src/main_unix.rs +++ /dev/null @@ -1,728 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use clap::Parser; -use futures::Stream; -use miette::{IntoDiagnostic, Result}; -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -#[cfg(target_os = "macos")] -use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; -use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; -use std::io; -use std::net::SocketAddr; -use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::net::{UnixListener, UnixStream}; -use tracing::info; -use tracing_subscriber::EnvFilter; - -#[derive(Parser, Debug)] -#[command(name = "openshell-driver-vm")] -#[command(version = VERSION)] -#[allow(clippy::struct_excessive_bools)] -struct Args { - #[arg(long, hide = true, default_value_t = false)] - internal_run_vm: bool, - - #[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")] - vm_root_disk: Option, - - #[arg(long = "vm-overlay-disk", hide = true)] - vm_overlay_disk: Option, - - #[arg(long = "vm-image-disk", hide = true)] - vm_image_disk: Option, - - #[arg(long = "vm-kernel-image", hide = true)] - vm_kernel_image: Option, - - #[arg(long, hide = true)] - vm_exec: Option, - - #[arg(long, hide = true, default_value = "/")] - vm_workdir: String, - - #[arg(long, hide = true)] - vm_env: Vec, - - #[arg(long, hide = true)] - vm_console_output: Option, - - #[arg(long, hide = true, default_value_t = 2)] - vm_vcpus: u8, - - #[arg(long, hide = true, default_value_t = 2048)] - vm_mem_mib: u32, - - #[arg(long, hide = true, default_value_t = 1)] - vm_krun_log_level: u32, - - #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")] - bind_address: Option, - - #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] - bind_socket: Option, - - #[arg(long, hide = true)] - expected_peer_pid: Option, - - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP", - default_value_t = false - )] - allow_unauthenticated_tcp: bool, - - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER", - default_value_t = false - )] - allow_same_uid_peer: bool, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] - default_image: String, - - #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] - bootstrap_image: String, - - #[arg( - long, - env = "OPENSHELL_VM_DRIVER_STATE_DIR", - default_value = "target/openshell-vm-driver" - )] - state_dir: PathBuf, - - #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] - guest_tls_ca: Option, - - #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] - guest_tls_cert: Option, - - #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] - guest_tls_key: Option, - - #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] - krun_log_level: u32, - - #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] - vcpus: u8, - - #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] - mem_mib: u32, - - #[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)] - overlay_disk_mib: u64, - - #[arg(long, env = "OPENSHELL_VM_GPU")] - gpu: bool, - - #[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)] - gpu_mem_mib: u32, - - #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] - gpu_vcpus: u8, - - #[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")] - sandbox_uid: Option, - - #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] - sandbox_gid: Option, - - #[arg(long, hide = true)] - vm_backend: Option, - - #[arg(long, hide = true)] - vm_gpu_bdf: Option, - - #[arg(long, hide = true)] - vm_tap_device: Option, - - #[arg(long, hide = true)] - vm_guest_ip: Option, - - #[arg(long, hide = true)] - vm_host_ip: Option, - - #[arg(long, hide = true)] - vm_vsock_cid: Option, - - #[arg(long, hide = true)] - vm_guest_mac: Option, - - #[arg(long, hide = true)] - vm_gateway_port: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - if args.internal_run_vm { - // We intentionally defer procguard arming until `run_vm()` so - // that the only arm is the one that knows how to clean up - // gvproxy. Racing two watchers against the same parent-death - // event causes the bare arm's `exit(1)` to win, skipping the - // gvproxy cleanup and leaking the helper. The risk window - // before `run_vm` arms procguard is ~a few syscalls long - // (`build_vm_launch_config`, `configured_runtime_dir`), which - // is negligible next to the parent gRPC server's uptime. - maybe_reexec_internal_vm_with_runtime_env()?; - let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; - run_vm(&config).map_err(|err| miette::miette!("{err}"))?; - return Ok(()); - } - - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; - - // Arm procguard so that if the gateway is killed (SIGKILL or crash) - // we also die. Without this the driver is reparented to init and - // keeps its per-sandbox VM launchers alive forever. Launchers have - // their own procguards (armed in `run_vm`) which cascade cleanup of - // gvproxy and the libkrun worker the moment this driver exits. - if let Err(err) = procguard::die_with_parent() { - tracing::warn!( - error = %err, - "procguard arm failed; gateway crashes may orphan this driver" - ); - } - - let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint - .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, - state_dir: args.state_dir.clone(), - launcher_bin: None, - default_image: args.default_image.clone(), - bootstrap_image: args.bootstrap_image.clone(), - log_level: args.log_level.clone(), - krun_log_level: args.krun_log_level, - vcpus: args.vcpus, - mem_mib: args.mem_mib, - overlay_disk_mib: args.overlay_disk_mib, - guest_tls_ca: args.guest_tls_ca.clone(), - guest_tls_cert: args.guest_tls_cert.clone(), - guest_tls_key: args.guest_tls_key.clone(), - gpu_enabled: args.gpu, - gpu_mem_mib: args.gpu_mem_mib, - gpu_vcpus: args.gpu_vcpus, - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) - .await - .map_err(|err| miette::miette!("{err}"))?; - - match listen_mode { - ComputeDriverListenMode::Unix { - socket_path, - expected_peer_pid, - } => { - prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?; - - info!(socket = %socket_path.display(), "Starting vm compute driver"); - let listener = UnixListener::bind(&socket_path).into_diagnostic()?; - restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?; - let result = tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(driver)) - .serve_with_incoming(AuthenticatedUnixIncoming::new(listener, expected_peer_pid)) - .await - .into_diagnostic(); - let _ = std::fs::remove_file(&socket_path); - result - } - ComputeDriverListenMode::Tcp(bind_address) => { - info!(address = %bind_address, "Starting unauthenticated dev vm compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(driver)) - .serve(bind_address) - .await - .into_diagnostic() - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum ComputeDriverListenMode { - Unix { - socket_path: PathBuf, - expected_peer_pid: Option, - }, - Tcp(SocketAddr), -} - -fn compute_driver_listen_mode(args: &Args) -> std::result::Result { - if let Some(socket_path) = args.bind_socket.clone() { - if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer { - return Err( - "--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development" - .to_string(), - ); - } - return Ok(ComputeDriverListenMode::Unix { - socket_path, - expected_peer_pid: args.expected_peer_pid, - }); - } - - if !args.allow_unauthenticated_tcp { - return Err( - "--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development" - .to_string(), - ); - } - - let Some(bind_address) = args.bind_address else { - return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string()); - }; - - Ok(ComputeDriverListenMode::Tcp(bind_address)) -} - -fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> { - let Some(parent) = socket_path.parent() else { - return Err(format!( - "vm compute driver socket path '{}' has no parent directory", - socket_path.display() - )); - }; - let expected_uid = current_euid(); - prepare_private_socket_dir(parent, expected_uid)?; - remove_stale_socket(socket_path, expected_uid) -} - -fn current_euid() -> u32 { - rustix::process::geteuid().as_raw() -} - -fn prepare_private_socket_dir( - socket_dir: &Path, - expected_uid: u32, -) -> std::result::Result<(), String> { - std::fs::create_dir_all(socket_dir) - .map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?; - let metadata = std::fs::symlink_metadata(socket_dir) - .map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?; - let file_type = metadata.file_type(); - if file_type.is_symlink() { - return Err(format!( - "socket dir {} is a symlink; refusing to use it", - socket_dir.display() - )); - } - if !file_type.is_dir() { - return Err(format!( - "socket dir {} is not a directory", - socket_dir.display() - )); - } - if metadata.uid() != expected_uid { - return Err(format!( - "socket dir {} is owned by uid {} but current euid is {}", - socket_dir.display(), - metadata.uid(), - expected_uid - )); - } - std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700)) - .map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display())) -} - -fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> { - let metadata = match std::fs::symlink_metadata(socket_path) { - Ok(metadata) => metadata, - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())), - }; - let file_type = metadata.file_type(); - if file_type.is_symlink() { - return Err(format!( - "socket {} is a symlink; refusing to remove it", - socket_path.display() - )); - } - if metadata.uid() != expected_uid { - return Err(format!( - "socket {} is owned by uid {} but current euid is {}", - socket_path.display(), - metadata.uid(), - expected_uid - )); - } - if !file_type.is_socket() { - return Err(format!( - "socket path {} exists but is not a Unix socket", - socket_path.display() - )); - } - std::fs::remove_file(socket_path) - .map_err(|err| format!("remove stale socket {}: {err}", socket_path.display())) -} - -fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> { - std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) - .map_err(|err| format!("chmod socket {}: {err}", socket_path.display())) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct PeerCredentials { - uid: u32, - pid: Option, -} - -fn peer_credentials(stream: &UnixStream) -> std::result::Result { - let credentials = stream - .peer_cred() - .map_err(|err| format!("read peer credentials: {err}"))?; - Ok(PeerCredentials { - uid: credentials.uid(), - pid: credentials.pid(), - }) -} - -fn authorize_peer_credentials( - peer: PeerCredentials, - driver_uid: u32, - gateway_pid: Option, -) -> std::result::Result<(), String> { - if peer.uid != driver_uid { - return Err(format!( - "peer uid {} does not match current euid {}", - peer.uid, driver_uid - )); - } - let Some(gateway_pid) = gateway_pid else { - return Ok(()); - }; - let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else { - return Err(format!( - "peer pid is unavailable; expected gateway pid {gateway_pid}" - )); - }; - if peer_process_id != gateway_pid { - return Err(format!( - "peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}" - )); - } - Ok(()) -} - -struct AuthenticatedUnixIncoming { - listener: UnixListener, - expected_uid: u32, - expected_peer_pid: Option, -} - -impl AuthenticatedUnixIncoming { - fn new(listener: UnixListener, expected_peer_pid: Option) -> Self { - Self { - listener, - expected_uid: current_euid(), - expected_peer_pid, - } - } -} - -impl Stream for AuthenticatedUnixIncoming { - type Item = io::Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - loop { - match this.listener.poll_accept(cx) { - Poll::Ready(Ok((stream, _addr))) => { - let authorized = peer_credentials(&stream).and_then(|peer| { - authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid) - }); - match authorized { - Ok(()) => return Poll::Ready(Some(Ok(stream))), - Err(err) => { - tracing::warn!( - error = %err, - "rejected vm compute driver UDS client" - ); - } - } - } - Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), - Poll::Pending => return Poll::Pending, - } - } - } -} - -fn build_vm_launch_config(args: &Args) -> std::result::Result { - let root_disk = args - .vm_root_disk - .clone() - .ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?; - let overlay_disk = args - .vm_overlay_disk - .clone() - .ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?; - let image_disk = args.vm_image_disk.clone(); - let exec_path = args - .vm_exec - .clone() - .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; - let console_output = args - .vm_console_output - .clone() - .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; - - let backend = match args.vm_backend.as_deref() { - Some("qemu") => VmBackend::Qemu, - Some("libkrun") | None => VmBackend::Libkrun, - Some(other) => return Err(format!("unknown VM backend: {other}")), - }; - - Ok(VmLaunchConfig { - root_disk, - overlay_disk, - image_disk, - kernel_image: args.vm_kernel_image.clone(), - vcpus: args.vm_vcpus, - mem_mib: args.vm_mem_mib, - exec_path, - args: Vec::new(), - env: args.vm_env.clone(), - workdir: args.vm_workdir.clone(), - log_level: args.vm_krun_log_level, - console_output, - backend, - gpu_bdf: args.vm_gpu_bdf.clone(), - tap_device: args.vm_tap_device.clone(), - guest_ip: args.vm_guest_ip.clone(), - host_ip: args.vm_host_ip.clone(), - vsock_cid: args.vm_vsock_cid, - guest_mac: args.vm_guest_mac.clone(), - gateway_port: args.vm_gateway_port, - }) -} - -#[cfg(target_os = "macos")] -fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { - use std::os::unix::process::CommandExt as _; - - const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; - - if std::env::var_os(REEXEC_ENV).is_some() { - return Ok(()); - } - - let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; - let runtime_str = runtime_dir.to_string_lossy(); - let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") - .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); - if !needs_reexec { - return Ok(()); - } - - let mut dyld_paths = vec![runtime_dir.clone()]; - if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { - dyld_paths.extend(std::env::split_paths(&existing)); - } - let joined = std::env::join_paths(&dyld_paths) - .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; - let exe = std::env::current_exe().into_diagnostic()?; - let args: Vec = std::env::args().skip(1).collect(); - - // Use execvp() so the current process is *replaced* by the re-exec'd - // binary — no wrapper process sits between the compute driver and - // the actually-running VM launcher. That avoids two problems: - // 1. An extra process level that survives SIGKILL of the driver - // (the wrapper was reparenting the re-exec'd child to init). - // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper - // doesn't reach the child unless we hand-roll forwarding. - // After exec, the child inherits our PID and our procguard arming. - let err = std::process::Command::new(exe) - .args(&args) - .env("DYLD_LIBRARY_PATH", &joined) - .env(VM_RUNTIME_DIR_ENV, runtime_dir) - .env(REEXEC_ENV, "1") - .exec(); - // `exec()` only returns on failure. - Err(miette::miette!("failed to re-exec with runtime env: {err}")) -} - -#[cfg(not(target_os = "macos"))] -// Signature must match the macOS variant which can fail. -#[allow(clippy::unnecessary_wraps)] -fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::{ - Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, - compute_driver_listen_mode, - }; - use clap::Parser; - use std::path::PathBuf; - - #[test] - fn peer_authorization_accepts_matching_uid_and_pid() { - authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: Some(42), - }, - 1000, - Some(42), - ) - .unwrap(); - } - - #[test] - fn peer_authorization_rejects_wrong_pid() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: Some(7), - }, - 1000, - Some(42), - ) - .expect_err("wrong pid should be rejected"); - assert!(err.contains("does not match expected gateway pid")); - } - - #[test] - fn peer_authorization_rejects_wrong_uid() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1001, - pid: Some(42), - }, - 1000, - Some(42), - ) - .expect_err("wrong uid should be rejected"); - assert!(err.contains("does not match current euid")); - } - - #[test] - fn peer_authorization_rejects_missing_pid_when_expected() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: None, - }, - 1000, - Some(42), - ) - .expect_err("missing pid should be rejected"); - assert!(err.contains("peer pid is unavailable")); - } - - #[test] - fn peer_authorization_accepts_matching_uid_without_expected_pid() { - authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: None, - }, - 1000, - None, - ) - .unwrap(); - } - - #[test] - fn listen_mode_rejects_default_tcp() { - let args = Args::parse_from(["openshell-driver-vm"]); - let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled"); - assert!(err.contains("--bind-socket is required")); - } - - #[test] - fn listen_mode_rejects_bind_address_without_tcp_opt_in() { - let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); - let err = - compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in"); - assert!(err.contains("--allow-unauthenticated-tcp")); - } - - #[test] - fn listen_mode_requires_bind_address_with_tcp_opt_in() { - let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]); - let err = - compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address"); - assert!(err.contains("--bind-address is required")); - } - - #[test] - fn listen_mode_accepts_explicit_unauthenticated_tcp() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--allow-unauthenticated-tcp", - "--bind-address", - "127.0.0.1:50061", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap()) - ); - } - - #[test] - fn listen_mode_requires_expected_peer_pid_for_uds() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - ]); - let err = compute_driver_listen_mode(&args) - .expect_err("UDS should require gateway peer pid by default"); - assert!(err.contains("--expected-peer-pid is required")); - } - - #[test] - fn listen_mode_accepts_uds_with_expected_peer_pid() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - "--expected-peer-pid", - "42", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Unix { - socket_path: PathBuf::from("/tmp/compute-driver.sock"), - expected_peer_pid: Some(42), - } - ); - } - - #[test] - fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - "--allow-same-uid-peer", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Unix { - socket_path: PathBuf::from("/tmp/compute-driver.sock"), - expected_peer_pid: None, - } - ); - } -} diff --git a/crates/openshell-driver-vm/src/main_win.rs b/crates/openshell-driver-vm/src/main_win.rs new file mode 100644 index 0000000000..8463707caf --- /dev/null +++ b/crates/openshell-driver-vm/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-driver-vm is unsupported on Windows"); + std::process::exit(1); +} diff --git a/crates/openshell-prover/src/lib_unix.rs b/crates/openshell-prover/src/lib_unix.rs deleted file mode 100644 index 0fb8757577..0000000000 --- a/crates/openshell-prover/src/lib_unix.rs +++ /dev/null @@ -1,293 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Formal policy verification for `OpenShell` sandboxes. -//! -//! Encodes sandbox policies, binary capabilities, and credential scopes as Z3 -//! SMT constraints, then checks reachability queries to detect data exfiltration -//! paths and write-bypass violations. - -pub mod accepted_risks; -pub mod credentials; -pub mod finding; -pub mod model; -pub mod policy; -pub mod queries; -pub mod registry; -pub mod report; - -use std::path::Path; - -use miette::Result; - -use accepted_risks::{apply_accepted_risks, load_accepted_risks}; -use model::build_model; -use policy::parse_policy; -use queries::run_all_queries; -use report::{render_compact, render_report}; - -/// Run the prover end-to-end and return a result containing an exit code. -/// -/// - `Ok(0)` — pass (no findings, or all accepted) -/// - `Ok(1)` — fail (one or more unaccepted findings present) -/// - `Err(_)` — input or registry loading error -/// -/// Binary and API capability registries are embedded at compile time. -/// Pass `registry_dir` to override with a custom filesystem registry. -pub fn prove( - policy_path: &str, - credentials_path: &str, - registry_dir: Option<&str>, - accepted_risks_path: Option<&str>, - compact: bool, -) -> Result { - let policy = parse_policy(Path::new(policy_path))?; - - let (credential_set, binary_registry) = match registry_dir { - Some(dir) => { - let dir = Path::new(dir); - ( - credentials::load_credential_set_from_dir(Path::new(credentials_path), dir)?, - registry::load_binary_registry_from_dir(dir)?, - ) - } - None => ( - credentials::load_credential_set_embedded(Path::new(credentials_path))?, - registry::load_embedded_binary_registry()?, - ), - }; - - let z3_model = build_model(policy, credential_set, binary_registry); - let mut findings = run_all_queries(&z3_model); - - if let Some(ar_path) = accepted_risks_path { - let accepted = load_accepted_risks(Path::new(ar_path))?; - findings = apply_accepted_risks(findings, &accepted); - } - - let exit_code = if compact { - render_compact(&findings, policy_path, credentials_path) - } else { - render_report(&findings, policy_path, credentials_path) - }; - - Ok(exit_code) -} - -// =========================================================================== -// Tests -// =========================================================================== - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - fn testdata_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") - } - - // 1. Parse testdata/policy.yaml, verify structure. - #[test] - fn test_parse_policy() { - let path = testdata_dir().join("policy.yaml"); - let model = parse_policy(&path).expect("failed to parse policy"); - assert_eq!(model.version, 1); - assert!(model.network_policies.contains_key("github_api")); - let rule = &model.network_policies["github_api"]; - assert_eq!(rule.name, "github-api"); - assert_eq!(rule.endpoints.len(), 2); - assert!(rule.binaries.len() >= 4); - } - - // 2. Verify readable_paths. - #[test] - fn test_filesystem_policy() { - let path = testdata_dir().join("policy.yaml"); - let model = parse_policy(&path).expect("failed to parse policy"); - let readable = model.filesystem_policy.readable_paths(); - assert!(readable.contains(&"/usr".to_owned())); - assert!(readable.contains(&"/sandbox".to_owned())); - assert!(readable.contains(&"/tmp".to_owned())); - } - - // 3. Workdir NOT included by default (matches runtime behavior). - #[test] - fn test_include_workdir_default() { - let yaml = r" -version: 1 -filesystem_policy: - read_only: - - /usr -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); - assert!(!readable.contains(&"/sandbox".to_owned())); - } - - // 4. Workdir excluded when include_workdir: false. - #[test] - fn test_include_workdir_false() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); - assert!(!readable.contains(&"/sandbox".to_owned())); - } - - // 5. No duplicate when workdir already in read_write. - #[test] - fn test_include_workdir_no_duplicate() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: true - read_write: - - /sandbox - - /tmp -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); - let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); - assert_eq!(sandbox_count, 1); - } - - // 6. End-to-end: testdata policy with a github credential in scope and a - // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. - // The prover output is categorical, not severity-graded. - #[test] - fn test_findings_for_github_policy() { - use finding::category; - - let policy_path = testdata_dir().join("policy.yaml"); - let creds_path = testdata_dir().join("credentials.yaml"); - - let pol = parse_policy(&policy_path).expect("parse policy"); - let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(pol, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - let categories: std::collections::HashSet<&str> = - findings.iter().map(|f| f.query.as_str()).collect(); - assert!( - categories.contains(category::L7_BYPASS_CREDENTIALED), - "expected l7_bypass_credentialed finding for bypass-L7 binary with credential in scope; \ - got categories: {categories:?}" - ); - // Every emitted category must be one of the four v1 categories. - let allowed: std::collections::HashSet<&str> = [ - category::LINK_LOCAL_REACH, - category::L7_BYPASS_CREDENTIALED, - category::CREDENTIAL_REACH_EXPANSION, - category::CAPABILITY_EXPANSION, - ] - .into_iter() - .collect(); - for c in &categories { - assert!( - allowed.contains(*c), - "unexpected category {c} emitted by the prover" - ); - } - } - - #[test] - fn test_wildcard_endpoint_covering_credential_host_emits_credential_reach() { - use finding::{FindingPath, category}; - - let policy = policy::parse_policy_str( - r#" -version: 1 -network_policies: - github_wildcard: - name: github-wildcard - endpoints: - - host: "*.github.com" - port: 443 - protocol: rest - enforcement: enforce - access: read-write - binaries: - - path: /usr/bin/curl -"#, - ) - .expect("parse policy"); - let cred_set = - credentials::load_credential_set_embedded(&testdata_dir().join("credentials.yaml")) - .expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(policy, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - let reach = findings - .iter() - .find(|finding| finding.query == category::CREDENTIAL_REACH_EXPANSION) - .expect("wildcard host covering api.github.com must be credentialed"); - assert!(reach.paths.iter().any(|path| matches!( - path, - FindingPath::Exfil(exfil) - if exfil.endpoint_host == "*.github.com" && exfil.binary == "/usr/bin/curl" - ))); - } - - #[test] - fn test_known_metadata_hostname_emits_link_local_finding() { - use finding::{FindingPath, category}; - - let policy = policy::parse_policy_str( - r" -version: 1 -network_policies: - metadata: - name: metadata - endpoints: - - host: metadata.google.internal - port: 80 - binaries: - - path: /usr/bin/curl -", - ) - .expect("parse policy"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(policy, credentials::CredentialSet::default(), bin_reg); - let findings = run_all_queries(&z3_model); - - let link_local = findings - .iter() - .find(|finding| finding.query == category::LINK_LOCAL_REACH) - .expect("known metadata hostname must emit link-local/metadata finding"); - assert!(link_local.paths.iter().any(|path| matches!( - path, - FindingPath::Exfil(exfil) - if exfil.endpoint_host == "metadata.google.internal" - ))); - } - - // 7. Empty policy produces no findings. - #[test] - fn test_empty_policy_no_findings() { - let policy_path = testdata_dir().join("empty-policy.yaml"); - let creds_path = testdata_dir().join("credentials.yaml"); - - let pol = parse_policy(&policy_path).expect("parse policy"); - let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(pol, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - assert!( - findings.is_empty(), - "deny-all policy should produce no findings, got: {findings:?}" - ); - } -} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 3fff53439c..dc716d0e58 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-sandbox" +autobins = false description = "OpenShell process sandbox and monitor" version.workspace = true edition.workspace = true @@ -10,9 +11,12 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +path = "src/lib_entry.rs" + [[bin]] name = "openshell-sandbox" -path = "src/main.rs" +path = "src/main_entry.rs" [target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 4d431aa1e6..e696403f39 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1,10 +1,3773 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -pub fn windows_unsupported() -> &'static str { - "openshell-sandbox supervisor is not available on Windows" +//! `OpenShell` Sandbox library. +//! +//! This crate provides process sandboxing and monitoring capabilities. + +mod activity_aggregator; +mod denial_aggregator; +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +mod google_cloud_metadata; +mod mechanistic_mapper; +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +mod metadata_server; +mod sidecar_control; + +use miette::{IntoDiagnostic, Result, WrapErr}; +use std::future::Future; +use std::sync::Arc; +#[cfg(target_os = "linux")] +use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::time::Duration; +use tracing::{debug, info, warn}; + +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, + DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, +}; + +// --------------------------------------------------------------------------- +// OCSF Context +// --------------------------------------------------------------------------- +// +// The following log sites intentionally remain as plain `tracing` macros +// and are NOT migrated to OCSF builders: +// +// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) +// - Transient "about to do X" events where the result is logged separately +// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") +// - Internal SSH channel warnings (unknown channel, PTY resize failures) +// - Denial flush telemetry (the individual denials are already OCSF events) +// - Status reporting failures (sync to gateway, non-actionable) +// - Route refresh interval validation warnings +// +// These are operational plumbing that don't represent security decisions, +// policy changes, or observable sandbox behavior worth structuring. +// --------------------------------------------------------------------------- + +/// Re-export the process-wide OCSF sandbox context getter. +/// +/// The singleton lives in `openshell-ocsf` so both supervisor leaves can +/// reach it without depending on `openshell-sandbox`. Initialised once during +/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; + +use openshell_core::denial::DenialEvent; +use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_supervisor_network::opa::OpaEngine; +use openshell_supervisor_process::process::ProcessEnforcementMode; +pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; +use openshell_supervisor_process::skills; +use tokio::sync::mpsc::UnboundedSender; +#[cfg(any(test, target_os = "linux"))] +use tokio::time::timeout; + +const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; +const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; +const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; +const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; + +/// Run a command in the sandbox. +/// +/// # Errors +/// +/// Returns an error if the command fails to start or encounters a fatal error. +#[allow( + clippy::too_many_arguments, + clippy::similar_names, + clippy::fn_params_excessive_bools +)] +pub async fn run_sandbox( + command: Vec, + workdir: Option, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, + ssh_socket_path: Option, + _health_check: bool, + _health_port: u16, + inference_routes: Option, + ocsf_enabled: Arc, + network_enabled: bool, + process_enabled: bool, + upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, +) -> Result { + let (program, args) = command + .split_first() + .ok_or_else(|| miette::miette!("No command specified"))?; + + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; they are only significant for network + // events which happen after the netns is created. + { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + + if !openshell_ocsf::ctx::set_ctx(SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + }) { + debug!("OCSF context already initialized, keeping existing"); + } + } + + let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); + let process_enforcement_mode = process_enforcement_mode(); + let process_uses_sidecar_control = + process_enabled && !network_enabled && sidecar_network_enforcement; + let mut process_control_connection = None; + let sidecar_bootstrap = if process_uses_sidecar_control { + let socket = sidecar_control_socket().ok_or_else(|| { + miette::miette!( + "{} is required for process-only sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET + ) + })?; + let (bootstrap, connection) = sidecar_control::connect_process_client( + &socket, + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), + ) + .await?; + process_control_connection = Some(connection); + Some(bootstrap) + } else { + None + }; + + // Load policy and initialize OPA engine + let openshell_endpoint_for_proxy = openshell_endpoint.clone(); + let sandbox_name_for_agg = sandbox.clone(); + let ( + mut policy, + opa_engine, + retained_proto, + middleware_registry_status, + loaded_policy_origin, + initial_agent_proposals_enabled, + ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let (policy, opa_engine, retained_proto, loaded_policy_origin) = + load_policy_from_sidecar_bootstrap(bootstrap)?; + ( + policy, + opa_engine, + retained_proto, + MiddlewareRegistryStatus::Synchronized, + loaded_policy_origin, + bootstrap.agent_proposals_enabled, + ) + } else { + load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + ) + .await? + }; + + // Override the policy's process identity with the driver-resolved UID/GID + // from the pod environment. The policy defaults to the name "sandbox" which + // resolves via /etc/passwd, but the driver may have chosen a different + // numeric UID (e.g. from OpenShift SCC annotations). + // Validate overrides against the same rules as the policy layer to prevent + // env-injected values (e.g. GID 0) from bypassing policy restrictions. + if let Ok(uid) = std::env::var(openshell_core::sandbox_env::SANDBOX_UID) + && !uid.is_empty() + { + if !openshell_policy::is_valid_sandbox_identity(&uid) { + return Err(miette::miette!( + "OPENSHELL_SANDBOX_UID contains invalid sandbox identity '{uid}'; \ + expected 'sandbox' or a numeric UID in range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID, + )); + } + policy.process.run_as_user = Some(uid); + } + if let Ok(gid) = std::env::var(openshell_core::sandbox_env::SANDBOX_GID) + && !gid.is_empty() + { + if !openshell_policy::is_valid_sandbox_identity(&gid) { + return Err(miette::miette!( + "OPENSHELL_SANDBOX_GID contains invalid sandbox identity '{gid}'; \ + expected 'sandbox' or a numeric GID in range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID, + )); + } + policy.process.run_as_group = Some(gid); + } + + #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] + let (provider_credentials, mut provider_env) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + bootstrap.provider_env_revision, + bootstrap.provider_child_env.clone(), + ); + (provider_credentials, bootstrap.provider_child_env.clone()) + } else { + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .message(format!( + "Failed to fetch provider environment, continuing without: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + }; + + let provider_credentials = ProviderCredentialState::from_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ); + let provider_env = provider_credentials.child_env_with_gcp_resolved(); + (provider_credentials, provider_env) + }; + + // Shared agent-proposals feature flag. Seed from the same initial settings + // snapshot that produced the policy so networking and process setup agree + // before the poll loop starts reconciling later changes. + let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); + + let process_control_writer = process_control_connection + .as_ref() + .map(|connection| connection.writer.clone()); + let mut process_control_closed = None; + if let Some(connection) = process_control_connection { + process_control_closed = Some(connection.closed); + spawn_sidecar_control_update_watcher( + connection.updates, + provider_credentials.clone(), + agent_proposals.clone(), + ); + } + + // Shared PID: set after process spawn so the proxy can look up + // the entrypoint process's /proc/net/tcp for identity binding. + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + + // Create the workload's network namespace. It is shared infrastructure: + // the proxy binds to its host-side veth IP, the bypass monitor reads + // /dev/kmsg from inside it, and the workload child / SSH sessions enter + // it via setns(). The RAII handle lives in this frame for the duration + // of the sandbox. + #[cfg(target_os = "linux")] + let netns = if network_enabled && !sidecar_network_enforcement { + openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? + } else { + None + }; + + // The denial channel is owned by the orchestrator: the proxy (in the + // networking leaf) and the bypass monitor (in the process leaf) both + // produce DenialEvents that the denial aggregator (orchestrator-side) + // consumes via the matching receiver. Both leaves are pure producers; + // the orchestrator owns the consumer task spawned below. + let (denial_tx, denial_rx, bypass_denial_tx): ( + Option>, + _, + Option>, + ) = if sandbox_id.is_some() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let bypass_tx = tx.clone(); + (Some(tx), Some(rx), Some(bypass_tx)) + } else { + (None, None, None) + }; + #[cfg(not(target_os = "linux"))] + drop(bypass_denial_tx); + + // Anonymous activity channel: same orchestrator-owned pattern as the + // denial channel. The proxy and the bypass monitor both emit per-event + // activity records; the orchestrator-side aggregator drains, sanitizes, + // and flushes anonymous summaries to the gateway. + let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { + let (tx, rx) = + tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); + let bypass_tx = tx.clone(); + (Some(tx), Some(rx), Some(bypass_tx)) + } else { + (None, None, None) + }; + #[cfg(not(target_os = "linux"))] + drop(bypass_activity_tx); + + // Workspace watch: the policy poll loop learns the workspace from + // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local + // API read the current value so proposals target the correct workspace. + let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); + + let networking = if network_enabled { + #[cfg(target_os = "linux")] + let proxy_bind_ip = netns + .as_ref() + .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); + #[cfg(not(target_os = "linux"))] + let proxy_bind_ip: Option = None; + + Some( + openshell_supervisor_network::run::run_networking( + &policy, + proxy_bind_ip, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + process_enabled, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + ) + .await?, + ) + } else { + None + }; + + #[cfg(target_os = "linux")] + let sidecar_control_server = if network_enabled && sidecar_network_enforcement { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Err(miette::miette!( + "sidecar network enforcement requires proxy network mode" + )); + } + let socket = sidecar_control_socket().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET + ) + })?; + let proto = retained_proto.as_ref().ok_or_else(|| { + miette::miette!( + "sidecar topology requires gateway policy data for the process supervisor" + ) + })?; + let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); + Some(sidecar_control::spawn_server( + &socket, + sidecar_control::BootstrapData { + policy_proto: proto.clone(), + provider_env_revision: provider_credentials.snapshot().revision, + provider_child_env: provider_env.clone(), + agent_proposals_enabled: agent_proposals.enabled(), + proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), + proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), + }, + sidecar_expected_peer()?, + )?) + } else { + None + }; + #[cfg(not(target_os = "linux"))] + let sidecar_control_server: Option = None; + + let sidecar_control_publisher = sidecar_control_server + .as_ref() + .map(sidecar_control::ServerHandle::publisher); + + #[cfg(target_os = "linux")] + let mut sidecar_control_task = None; + + #[cfg(target_os = "linux")] + if network_enabled + && sidecar_network_enforcement + && let Some(server) = sidecar_control_server + { + let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar network topology", + openshell_core::sandbox_env::SSH_SOCKET_PATH + ) + })?; + let (entrypoint_rx, connection_task) = server.into_runtime_parts(); + sidecar_control_task = Some(connection_task); + spawn_sidecar_entrypoint_handler( + entrypoint_rx, + entrypoint_pid.clone(), + opa_engine.clone(), + retained_proto.clone(), + openshell_endpoint.clone(), + sandbox_id.clone(), + std::path::PathBuf::from(trusted_ssh_socket_path), + ); + } + + #[cfg(not(target_os = "linux"))] + if network_enabled && sidecar_network_enforcement { + return Err(miette::miette!( + "sidecar network enforcement is only supported on Linux" + )); + } + + // Spawn the denial-aggregator flush task. The aggregator drains denial + // events from the proxy + bypass monitor, batches them, and ships + // summaries to the gateway via `SubmitPolicyAnalysis`. + if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall + // back to the ID when the name isn't set. + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + + let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); + let denial_workspace_gate = workspace_rx.clone(); + let denial_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + |summaries| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = denial_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_proposals_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summaries, + ) + .await + { + warn!(error = %e, "Failed to flush denial summaries to gateway"); + } + } + }, + move || !denial_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn the activity-aggregator flush task. The aggregator drains + // anonymous activity events from the proxy, sanitizes deny groups, + // and ships periodic summaries to the gateway. + if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( + std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") + .ok() + .as_deref(), + ); + + let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); + let activity_workspace_gate = workspace_rx.clone(); + let activity_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + move |summary| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = activity_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_activity_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summary, + ) + .await + { + warn!(error = %e, "Failed to flush activity summary to gateway"); + } + } + }, + move || !activity_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn background policy poll task (gRPC mode only). + if !process_uses_sidecar_control + && let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) + { + let poll_id = id.to_string(); + let poll_endpoint = endpoint.to_string(); + let poll_engine = engine.clone(); + let poll_ocsf_enabled = ocsf_enabled.clone(); + let poll_pid = entrypoint_pid.clone(); + let poll_provider_credentials = provider_credentials.clone(); + let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); + let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let poll_ctx = PolicyPollLoopContext { + endpoint: poll_endpoint, + sandbox_id: poll_id, + opa_engine: poll_engine, + loaded_policy_origin, + entrypoint_pid: poll_pid, + interval_secs: poll_interval_secs, + ocsf_enabled: poll_ocsf_enabled, + provider_credentials: poll_provider_credentials, + policy_local_ctx: poll_policy_local, + agent_proposals: agent_proposals.clone(), + middleware_registry_status, + sidecar_control_publisher: sidecar_control_publisher.clone(), + workspace_tx, + }; + + tokio::spawn(async move { + if let Err(e) = run_policy_poll_loop(poll_ctx).await { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .message(format!("Policy poll loop exited with error: {e}")) + .build() + ); + } + }); + } + + // Start GCE metadata loopback server inside the network namespace so + // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) + // can reach it via direct TCP. Must start before the process leaf so SSH + // sessions also see corrected env vars on bind failure. + #[cfg(target_os = "linux")] + if let Some(ns) = netns.as_ref() + && provider_credentials + .snapshot() + .child_env + .contains_key("GCE_METADATA_HOST") + { + let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + match ns + .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) + .await + { + Ok(listener) => { + tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); + if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { + info!(addr = %addr, "GCE metadata loopback server ready"); + } else { + warn!("GCE metadata server failed to become ready, removing metadata env vars"); + provider_env.remove("GCE_METADATA_HOST"); + provider_env.remove("GCE_METADATA_IP"); + provider_env.remove("METADATA_SERVER_DETECTION"); + provider_credentials.remove_env_key("GCE_METADATA_HOST"); + } + } + Err(e) => { + warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); + provider_env.remove("GCE_METADATA_HOST"); + provider_env.remove("GCE_METADATA_IP"); + provider_env.remove("METADATA_SERVER_DETECTION"); + provider_credentials.remove_env_key("GCE_METADATA_HOST"); + } + } + } + + let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; + let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { + bootstrap + .proxy_ca_cert_path + .clone() + .zip(bootstrap.proxy_ca_bundle_path.clone()) + }); + + let exit_code = if process_enabled { + let ca_file_paths = networking + .as_ref() + .and_then(|n| n.ca_file_paths.clone()) + .or_else(|| { + if sidecar_network_enforcement { + sidecar_bootstrap_ca_file_paths + .clone() + .or_else(sidecar_ca_file_paths) + } else { + None + } + }); + + let entrypoint_started_tx = + if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + match rx.await { + Ok(pid) => { + if let Err(err) = + sidecar_control::send_entrypoint_started(&writer, pid).await + { + warn!(error = %err, "Failed to send sidecar entrypoint event"); + } + } + Err(_closed) => { + debug!("Entrypoint exited before sidecar entrypoint event was sent"); + } + } + }); + Some(tx) + } else { + None + }; + + let process = openshell_supervisor_process::run::run_process( + program, + args, + workdir.as_deref(), + timeout_secs, + interactive, + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path, + sidecar_network_enforcement, + &process_policy, + process_enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + provider_credentials, + provider_env, + ca_file_paths, + agent_proposals.clone(), + #[cfg(target_os = "linux")] + netns.as_ref(), + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + ); + + if let Some(control_closed) = process_control_closed.as_mut() { + tokio::select! { + result = process => result?, + _ = control_closed => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Authoritative network-sidecar control channel closed; terminating process container" + ) + .build() + ); + return Err(miette::miette!( + "authoritative network-sidecar control channel closed" + )); + } + } + } else { + process.await? + } + } else { + // Network-only sidecar mode: keep the proxy and its background + // tasks alive (held via the `networking` value) until shutdown. If the + // sole authenticated process-supervisor control connection closes, + // exit non-zero so Kubernetes restarts the network sidecar and creates + // a fresh one-client bootstrap listener for the restarted agent. + #[cfg(target_os = "linux")] + if let Some(control_task) = sidecar_control_task { + tokio::select! { + () = wait_for_shutdown_signal() => 0, + result = control_task => { + warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); + 1 + } + } + } else { + wait_for_shutdown_signal().await; + 0 + } + #[cfg(not(target_os = "linux"))] + { + wait_for_shutdown_signal().await; + 0 + } + }; + + // Drop networking explicitly so the proxy + bypass monitor RAII + // handles tear down before we return. + drop(networking); + + Ok(exit_code) +} + +/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is +/// no entrypoint child whose lifetime drives the supervisor's exit. +async fn wait_for_shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut sigterm = match signal(SignalKind::terminate()) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + error = %e, + "Failed to install SIGTERM handler; waiting on SIGINT only" + ); + let _ = tokio::signal::ctrl_c().await; + return; + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => { + info!("Received SIGINT, shutting down network-only supervisor"); + } + _ = sigterm.recv() => { + info!("Received SIGTERM, shutting down network-only supervisor"); + } + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + info!("Received Ctrl-C, shutting down network-only supervisor"); + } +} + +fn sidecar_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) + .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) +} + +fn process_enforcement_mode() -> ProcessEnforcementMode { + match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) + .ok() + .as_deref() + { + Some("sidecar") => ProcessEnforcementMode::NetworkOnly, + _ => ProcessEnforcementMode::Full, + } +} + +fn sidecar_control_socket() -> Option { + std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) + .ok() + .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn sidecar_expected_peer() -> Result { + fn required_numeric_env(name: &str) -> Result { + let value = std::env::var(name) + .into_diagnostic() + .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; + value.parse::().into_diagnostic().wrap_err_with(|| { + format!("{name} must be a numeric ID for sidecar control authentication") + }) + } + + Ok(sidecar_control::ExpectedPeer { + uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, + gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, + }) +} + +type LoadedPolicyBundle = ( + SandboxPolicy, + Option>, + Option, + LoadedPolicyOrigin, +); + +fn load_policy_from_sidecar_bootstrap( + bootstrap: &sidecar_control::BootstrapData, +) -> Result { + let proto = bootstrap.policy_proto.clone(); + let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); + let policy = SandboxPolicy::try_from(proto.clone())?; + info!("Loaded sidecar policy from control socket bootstrap"); + Ok(( + policy, + opa_engine, + Some(proto), + LoadedPolicyOrigin::Gateway { revision: None }, + )) +} + +fn spawn_sidecar_control_update_watcher( + mut updates: tokio::sync::mpsc::UnboundedReceiver, + provider_credentials: ProviderCredentialState, + agent_proposals: AgentProposals, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + while let Some(update) = updates.recv().await { + match update { + sidecar_control::ControlUpdate::ProviderEnv { + revision, + provider_child_env, + } => { + if revision <= provider_credentials.snapshot().revision { + continue; + } + let env_count = provider_credentials + .install_child_env_snapshot(revision, provider_child_env); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("provider_env_revision", serde_json::json!(revision)) + .message(format!( + "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" + )) + .build() + ); + } + sidecar_control::ControlUpdate::Policy { + policy_proto, + policy_hash, + config_revision, + } => { + debug!( + version = policy_proto.version, + policy_hash, + config_revision, + "Received sidecar policy update for process supervisor" + ); + } + sidecar_control::ControlUpdate::AgentProposals { + enabled, + config_revision, + } => { + apply_agent_proposals_enabled( + &agent_proposals, + enabled, + "sidecar control", + Some(config_revision), + None, + skills::install_static_skills, + ); + } + } + } + }) +} + +#[cfg(target_os = "linux")] +fn spawn_sidecar_entrypoint_handler( + mut entrypoint_rx: tokio::sync::mpsc::Receiver, + entrypoint_pid: Arc, + opa_engine: Option>, + retained_proto: Option, + openshell_endpoint: Option, + sandbox_id: Option, + trusted_ssh_socket_path: std::path::PathBuf, +) { + tokio::spawn(async move { + let mut session_started = false; + let mut trusted_supervisor_pid = None; + let terminating = Arc::new(AtomicBool::new(false)); + while let Some(started) = entrypoint_rx.recv().await { + entrypoint_pid.store(started.pid, Ordering::Release); + if started.start_session { + info!( + pid = started.pid, + ssh_socket = %trusted_ssh_socket_path.display(), + "Sidecar process supervisor reported entrypoint start" + ); + } else { + trusted_supervisor_pid = Some(started.pid); + info!( + pid = started.pid, + "Sidecar process supervisor reported initial process anchor" + ); + } + + if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { + match engine.reload_from_proto_with_pid(proto, started.pid) { + Ok(()) => info!( + pid = started.pid, + "Policy binary symlink resolution complete for sidecar process anchor" + ), + Err(err) => warn!( + error = %err, + pid = started.pid, + "Failed to rebuild OPA engine with sidecar process anchor PID" + ), + } + } + + if started.start_session + && !session_started + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_ref(), sandbox_id.as_ref()) + { + let Some(supervisor_pid) = trusted_supervisor_pid else { + warn!( + pid = started.pid, + "Ignoring sidecar entrypoint event before authenticated supervisor anchor" + ); + continue; + }; + openshell_supervisor_process::supervisor_session::spawn( + endpoint.clone(), + id.clone(), + trusted_ssh_socket_path.clone(), + None, + Some(supervisor_pid), + Arc::clone(&terminating), + ); + session_started = true; + info!("sidecar supervisor session task spawned"); + } + } + terminating.store(true, Ordering::Release); + }); +} + +fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { + let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) + .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); + let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); + let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); + (cert.exists() && bundle.exists()).then_some((cert, bundle)) +} + +fn process_policy_for_topology( + policy: &SandboxPolicy, + sidecar_network_enforcement: bool, +) -> Result { + let mut process_policy = policy.clone(); + if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { + let proxy = process_policy + .network + .proxy + .get_or_insert(ProxyPolicy { http_addr: None }); + if proxy.http_addr.is_none() { + proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); + } + } + Ok(process_policy) +} + +/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. +async fn flush_proposals_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summaries: Vec, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialSummary, L7RequestSample}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summaries: Vec = summaries + .into_iter() + .map(|s| DenialSummary { + sandbox_id: String::new(), + host: s.host, + port: u32::from(s.port), + binary: s.binary, + ancestors: s.ancestors, + deny_reason: s.deny_reason, + first_seen_ms: s.first_seen_ms, + last_seen_ms: s.last_seen_ms, + count: s.count, + suppressed_count: 0, + total_count: s.count, + sample_cmdlines: s.sample_cmdlines, + binary_sha256: String::new(), + persistent: false, + denial_stage: s.denial_stage, + l7_request_samples: s + .l7_samples + .into_iter() + .map(|l| L7RequestSample { + method: l.method, + path: l.path, + decision: "deny".to_string(), + count: l.count, + }) + .collect(), + l7_inspection_active: false, + }) + .collect(); + + // Run the mechanistic mapper sandbox-side to generate proposals. + // The gateway is a thin persistence + validation layer — it never + // generates proposals itself. + let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); + + info!( + sandbox_name = %sandbox_name, + summaries = proto_summaries.len(), + proposals = proposals.len(), + "Flushed denial analysis to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + proto_summaries, + proposals, + Vec::new(), + "mechanistic", + ) + .await?; + + Ok(()) +} + +/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. +async fn flush_activity_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summary: activity_aggregator::FlushableActivitySummary, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summary = NetworkActivitySummary { + network_activity_count: summary.network_activity_count, + denied_action_count: summary.denied_action_count, + denials_by_group: summary + .denials_by_group + .into_iter() + .map(|(group, count)| DenialGroupCount { + deny_group: group, + denied_count: count, + }) + .collect(), + }; + + info!( + sandbox_name = %sandbox_name, + network_activity_count = proto_summary.network_activity_count, + denied_action_count = proto_summary.denied_action_count, + "Flushed activity summary to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + Vec::new(), + Vec::new(), + vec![proto_summary], + "activity", + ) + .await?; + + Ok(()) +} + +// ============================================================================ +// Baseline filesystem path enrichment +// ============================================================================ + +/// Minimum read-only paths required for a proxy-mode sandbox child process to +/// function: dynamic linker, shared libraries, DNS resolution, CA certs, +/// Python venv, openshell logs, process info, and random bytes. +/// +/// `/proc` and `/dev/urandom` are included here for the same reasons they +/// appear in `restrictive_default_policy()`: virtually every process needs +/// them. Before the Landlock per-path fix (#677) these were effectively free +/// because a missing path silently disabled the entire ruleset; now they must +/// be explicit. +const PROXY_BASELINE_READ_ONLY: &[&str] = &[ + "/usr", + "/lib", + "/etc", + "/app", + "/var/log", + "/proc", + "/dev/urandom", +]; + +/// Minimum read-write paths required for a proxy-mode sandbox child process: +/// user working directory and temporary files. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; + +/// GPU read-only paths. +/// +/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced +/// socket at init time. If the directory exists but Landlock denies traversal +/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` +/// even though the daemon is optional. Only read/traversal access is needed. +/// +/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, +/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` +/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may +/// not be covered by the parent-directory Landlock rule when the mount crosses +/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal +/// is permitted regardless of Landlock's cross-mount behaviour. +const GPU_BASELINE_READ_ONLY: &[&str] = &[ + "/run/nvidia-persistenced", + "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory +]; + +/// GPU read-write paths (static). +/// +/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, +/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native +/// Linux. Landlock restricts `open(2)` on device files even when DAC allows +/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. +/// These devices do not exist on WSL2 and will be skipped by the existence +/// check in `enrich_proto_baseline_paths()`. +/// +/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver +/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects +/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux +/// and will be skipped there by the existence check. +/// +/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` +/// to set thread names. Without write access, `cuInit()` returns error 304. +/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to +/// inodes and child processes have different procfs inodes than the parent. +/// +/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by +/// `enumerate_gpu_device_nodes()` since the count varies. +const GPU_BASELINE_READ_WRITE: &[&str] = &[ + "/dev/nvidiactl", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia-modeset", + "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) + "/proc", +]; + +/// Returns true if GPU devices are present in the container. +/// +/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and +/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these +/// depending on the host kernel; the other will not exist. +fn has_gpu_devices() -> bool { + std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() +} + +/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). +fn enumerate_gpu_device_nodes() -> Vec { + let mut paths = Vec::new(); + if let Ok(entries) = std::fs::read_dir("/dev") { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(suffix) = name.strip_prefix("nvidia") { + if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { + continue; + } + paths.push(entry.path().to_string_lossy().into_owned()); + } + } + } + paths +} + +fn push_unique(paths: &mut Vec, path: String) { + if !paths.iter().any(|p| p == &path) { + paths.push(path); + } +} + +fn collect_baseline_enrichment_paths( + include_proxy: bool, + include_gpu: bool, + gpu_device_nodes: Vec, +) -> (Vec, Vec) { + let mut ro = Vec::new(); + let mut rw = Vec::new(); + + if include_proxy { + for &path in PROXY_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in PROXY_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + } + + if include_gpu { + for &path in GPU_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in GPU_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + for path in gpu_device_nodes { + push_unique(&mut rw, path); + } + } + + // A path promoted to read_write (e.g. /proc for GPU) should not also + // appear in read_only — Landlock handles the overlap correctly but the + // duplicate is confusing when inspecting the effective policy. + ro.retain(|p| !rw.contains(p)); + + (ro, rw) +} + +fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { + let include_gpu = has_gpu_devices(); + let gpu_device_nodes = if include_gpu { + enumerate_gpu_device_nodes() + } else { + Vec::new() + }; + collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) +} + +/// Collect all active baseline paths for tests and diagnostics. +/// Returns `(read_only, read_write)` as owned `String` vecs. +#[cfg(test)] +fn baseline_enrichment_paths() -> (Vec, Vec) { + active_baseline_enrichment_paths(true) +} + +fn enrich_proto_baseline_paths_with( + proto: &mut openshell_core::proto::SandboxPolicy, + ro: &[String], + rw: &[String], + path_exists: F, +) -> bool +where + F: Fn(&str) -> bool, +{ + if ro.is_empty() && rw.is_empty() { + return false; + } + + let fs = proto + .filesystem + .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { + include_workdir: true, + ..Default::default() + }); + + let mut modified = false; + for path in ro { + if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { + if !path_exists(path) { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + fs.read_only.push(path.clone()); + modified = true; + } + } + for path in rw { + if fs.read_write.iter().any(|p| p == path) { + continue; + } + if !path_exists(path) { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + if fs.read_only.iter().any(|p| p == path) { + if path == "/proc" { + info!( + path, + "Promoting /proc from read-only to read-write for GPU runtime compatibility" + ); + fs.read_only.retain(|p| p != path); + fs.read_write.push(path.clone()); + modified = true; + } + continue; + } + fs.read_write.push(path.clone()); + modified = true; + } + + modified +} + +/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths +/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if +/// missing; user-specified paths are never removed. +/// +/// Returns `true` if the policy was modified (caller may want to sync back). +fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); + + // Baseline paths are system-injected, not user-specified. Skip paths + // that do not exist in this container image to avoid noisy warnings from + // Landlock and, more critically, to prevent a single missing baseline + // path from abandoning the entire Landlock ruleset under best-effort + // mode (see issue #664). + let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { + std::path::Path::new(path).exists() + }); + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } + + modified +} + +fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + openshell_policy::strip_provider_rule_names(proto) +} + +fn proto_sync_payload_for_enriched_policy( + proto: &openshell_core::proto::SandboxPolicy, + enriched: bool, +) -> Option { + if !enriched { + return None; + } + + let mut sync_policy = proto.clone(); + strip_proto_provider_policy_entries(&mut sync_policy); + Some(sync_policy) +} + +/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem +/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the +/// local-file code path where no proto is available. +fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { + let (ro, rw) = + active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); + if ro.is_empty() && rw.is_empty() { + return; + } + + let mut modified = false; + for path in &ro { + let p = std::path::PathBuf::from(path); + if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { + if !p.exists() { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_only.push(p); + modified = true; + } + } + for path in &rw { + let p = std::path::PathBuf::from(path); + if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { + continue; + } + if !p.exists() { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_write.push(p); + modified = true; + } + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod baseline_tests { + use super::*; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; + + #[test] + fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { + // When GPU devices are present, /proc is promoted to read_write + // (CUDA needs to write /proc//task//comm). It should + // NOT also appear in read_only. + if !has_gpu_devices() { + // Can't test GPU dedup without GPU devices; skip silently. + return; + } + let (ro, rw) = baseline_enrichment_paths(); + assert!( + rw.contains(&"/proc".to_string()), + "/proc should be in read_write when GPU is present" + ); + assert!( + !ro.contains(&"/proc".to_string()), + "/proc should NOT be in read_only when it is already in read_write" + ); + } + + #[test] + fn proc_in_read_only_without_gpu() { + if has_gpu_devices() { + // On a GPU host we can't test the non-GPU path; skip silently. + return; + } + let (ro, _rw) = baseline_enrichment_paths(); + assert!( + ro.contains(&"/proc".to_string()), + "/proc should be in read_only when GPU is not present" + ); + } + + #[test] + fn baseline_read_write_always_includes_sandbox_and_tmp() { + let (_ro, rw) = baseline_enrichment_paths(); + assert!(rw.contains(&"/sandbox".to_string())); + assert!(rw.contains(&"/tmp".to_string())); + } + + #[test] + fn enumerate_gpu_device_nodes_skips_bare_nvidia() { + // "nvidia" (without a trailing digit) is a valid /dev entry on some + // systems but is not a per-GPU device node. The enumerator must + // not match it. + let nodes = enumerate_gpu_device_nodes(); + assert!( + !nodes.contains(&"/dev/nvidia".to_string()), + "bare /dev/nvidia should not be enumerated: {nodes:?}" + ); + } + + #[test] + fn no_duplicate_paths_in_baseline() { + let (ro, rw) = baseline_enrichment_paths(); + // No path should appear in both lists. + for path in &ro { + assert!( + !rw.contains(path), + "path {path} appears in both read_only and read_write" + ); + } + } + + #[test] + fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + read_only: vec!["/tmp".to_string()], + read_write: vec![], + include_workdir: false, + }); + policy.network_policies.insert( + "test".into(), + openshell_core::proto::NetworkPolicyRule { + name: "test-rule".into(), + endpoints: vec![openshell_core::proto::NetworkEndpoint { + host: "example.com".into(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + + enrich_proto_baseline_paths(&mut policy); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + filesystem.read_only.contains(&"/tmp".to_string()), + "explicit read_only baseline path should be preserved" + ); + assert!( + !filesystem.read_write.contains(&"/tmp".to_string()), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + assert!(strip_proto_provider_policy_entries(&mut policy)); + assert!( + !policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(policy.network_policies.contains_key("sandbox_only")); + assert!(!strip_proto_provider_policy_entries(&mut policy)); + } + + #[test] + fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + + assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "provider-derived rules alone must not trigger sync or mutate runtime policy" + ); + } + + #[test] + fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + runtime_policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) + .expect("enrichment should create a sync payload"); + + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "runtime policy must retain provider-derived rules for OPA input" + ); + assert!( + !sync_policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(sync_policy.network_policies.contains_key("sandbox_only")); + } + + #[test] + fn proto_gpu_enrichment_promotes_proc_without_network_policy() { + let mut policy = openshell_policy::restrictive_default_policy(); + assert!( + policy.network_policies.is_empty(), + "regression setup must exercise the no-network default path" + ); + let (ro, rw) = + collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); + + let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { + matches!(path, "/proc" | "/dev/nvidia0") + }); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + enriched, + "GPU enrichment should not require network policies" + ); + assert!( + filesystem.read_write.contains(&"/dev/nvidia0".to_string()), + "GPU enrichment should add enumerated device nodes without network policies" + ); + assert!( + !filesystem.read_only.contains(&"/proc".to_string()), + "GPU enrichment should remove /proc from read_only" + ); + assert!( + filesystem.read_write.contains(&"/proc".to_string()), + "GPU enrichment should promote /proc to read_write" + ); + } + + #[test] + fn gpu_baseline_read_write_contains_dxg() { + // /dev/dxg must be present so WSL2 sandboxes get the Landlock + // read-write rule for the CDI-injected DXG device. The existence + // check in enrich_proto_baseline_paths() skips it on native Linux. + assert!( + GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), + "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" + ); + } + + #[test] + fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![std::path::PathBuf::from("/tmp")], + read_write: vec![], + include_workdir: false, + }, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }; + + enrich_sandbox_baseline_paths(&mut policy); + + assert!( + policy + .filesystem + .read_only + .contains(&std::path::PathBuf::from("/tmp")), + "explicit read_only baseline path should be preserved" + ); + assert!( + !policy + .filesystem + .read_write + .contains(&std::path::PathBuf::from("/tmp")), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn gpu_baseline_read_only_contains_usr_lib_wsl() { + // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library + // bind-mounts are accessible under Landlock. Skipped on native Linux. + assert!( + GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), + "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" + ); + } + + #[test] + fn has_gpu_devices_reflects_dxg_or_nvidiactl() { + // Verify the OR logic: result must match the manual disjunction of + // the two path checks. Passes in all environments. + let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); + let dxg = std::path::Path::new("/dev/dxg").exists(); + assert_eq!( + has_gpu_devices(), + nvidiactl || dxg, + "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" + ); + } +} + +/// Returns `true` if the error is transient and worth retrying. +/// +/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If +/// found, only the gRPC codes that represent transient failures are retryable. +/// If no `tonic::Status` is present (e.g. a raw connection error), assume the +/// failure is transient. +fn is_retryable_error(err: &miette::Report) -> bool { + let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); + while let Some(e) = source { + if let Some(status) = e.downcast_ref::() { + return matches!( + status.code(), + tonic::Code::Unavailable + | tonic::Code::DeadlineExceeded + | tonic::Code::ResourceExhausted + | tonic::Code::Aborted + | tonic::Code::Internal + | tonic::Code::Unknown + ); + } + source = e.source(); + } + true +} + +/// Retry a gRPC operation with exponential backoff (capped at 4 s). +/// +/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, +/// `PERMISSION_DENIED`) are returned immediately without retrying. +async fn grpc_retry(op_name: &str, f: F) -> Result +where + F: Fn() -> Fut, + Fut: Future>, +{ + let mut last_err = None; + for attempt in 1..=5u32 { + match f().await { + Ok(val) => return Ok(val), + Err(e) => { + if !is_retryable_error(&e) { + return Err(e); + } + if attempt < 5 { + warn!( + attempt, + max_attempts = 5, + error = %e, + "{op_name} failed, retrying" + ); + let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); + tokio::time::sleep(backoff).await; + } + last_err = Some(e); + } + } + } + Err(miette::miette!( + "{op_name} failed after 5 attempts: {}", + last_err.expect("loop executed at least once") + )) +} + +/// Load sandbox policy from local files or gRPC. +/// +/// Priority: +/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files +/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC +/// 3. If the server returns no policy, discover from disk or use restrictive default +/// 4. Otherwise, return an error +/// +/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto +/// policy. The proto is retained so the OPA engine can be rebuilt with symlink +/// resolution after the container entrypoint starts. +async fn load_policy( + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, +) -> Result<( + SandboxPolicy, + Option>, + Option, + MiddlewareRegistryStatus, + LoadedPolicyOrigin, + bool, +)> { + // File mode: load OPA engine from rego rules + YAML data (dev override) + if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "loading") + .unmapped("policy_rules", serde_json::json!(policy_file)) + .unmapped("policy_data", serde_json::json!(data_file)) + .message(format!( + "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" + )) + .build()); + let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { + openshell_supervisor_middleware_builtins::validate_config(implementation, config) + .map_err(|error| error.to_string()) + }; + let engine = OpaEngine::from_files_with_middleware_config( + std::path::Path::new(policy_file), + std::path::Path::new(data_file), + Some(&validate_middleware_config), + )?; + let middleware_registry = + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + engine.replace_middleware_registry(middleware_registry)?; + let config = engine.query_sandbox_config()?; + let mut policy = SandboxPolicy { + version: 1, + filesystem: config.filesystem, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: config.landlock, + process: config.process, + }; + enrich_sandbox_baseline_paths(&mut policy); + // File mode has no operator-registered middleware to connect. + return Ok(( + policy, + Some(Arc::new(engine)), + None, + MiddlewareRegistryStatus::Synchronized, + LoadedPolicyOrigin::LocalOverride, + false, + )); + } + + // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data + if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + info!( + sandbox_id = %id, + endpoint = %endpoint, + "Fetching sandbox policy via gRPC" + ); + let mut snapshot = grpc_retry("Policy fetch", || { + openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) + }) + .await?; + + let mut proto_policy = if let Some(p) = snapshot.policy.clone() { + p + } else { + // No policy configured on the server. Discover from disk or + // fall back to the restrictive default, then sync to the + // gateway so it becomes the authoritative baseline. + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "discovery") + .message("Server returned no policy; attempting local discovery") + .build() + ); + let mut discovered = discover_policy_from_disk_or_default(); + // Enrich before syncing so the gateway baseline includes + // baseline paths from the start. + enrich_proto_baseline_paths(&mut discovered); + strip_proto_provider_policy_entries(&mut discovered); + let sandbox = sandbox.as_deref().ok_or_else(|| { + miette::miette!( + "Cannot sync discovered policy: sandbox not available.\n\ + Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." + ) + })?; + + // Sync and re-fetch over a single connection to avoid extra + // TLS handshakes. + let ws = snapshot.workspace.clone(); + snapshot = grpc_retry("Policy discovery sync", || { + openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox, + &discovered, + &ws, + ) + }) + .await?; + snapshot.policy.clone().ok_or_else(|| { + miette::miette!("Server still returned no policy after sync — this is a bug") + })? + }; + + // True only while `snapshot` describes the exact policy that will be + // constructed below. If enrichment cannot be synced and re-fetched, + // the policy remains enforceable but cannot be acknowledged by + // inferred structural equality. + let mut policy_bound_to_snapshot = true; + + // Ensure baseline filesystem paths are present for proxy-mode + // sandboxes. If the policy was enriched, sync the updated version + // back to the gateway so users can see the effective policy. + let enriched = enrich_proto_baseline_paths(&mut proto_policy); + let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); + if let Some(sync_policy) = sync_policy { + if let Some(sandbox_name) = sandbox.as_deref() { + match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox_name, + &sync_policy, + &snapshot.workspace, + ) + .await + { + Ok(canonical) => { + if let Some(policy) = canonical.policy.clone() { + proto_policy = policy; + snapshot = canonical; + } else { + policy_bound_to_snapshot = false; + warn!( + "Gateway returned no policy after enrichment sync; initial revision will be reconciled" + ); + } + } + Err(e) => { + policy_bound_to_snapshot = false; + warn!( + error = %e, + "Failed to sync enriched policy back to gateway; initial revision will be reconciled" + ); + } + } + } else { + policy_bound_to_snapshot = false; + } + } + + let loaded_policy_revision = + policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); + + // Build OPA engine from baked-in rules + typed proto data. + // In cluster mode, proxy networking is always enabled so OPA is + // always required for allow/deny decisions. + // The initial load uses pid=0 (no symlink resolution) because the + // container hasn't started yet. After the entrypoint spawns, the + // engine is rebuilt with the real PID for symlink resolution. + info!("Creating OPA engine from proto policy data"); + let engine = match OpaEngine::from_proto(&proto_policy) { + Ok(engine) => engine, + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + return Err(e); + } + }; + + // Install the in-process catalog before any external connection can + // fail. A newly started sandbox must always be able to resolve built-in + // bindings, even while operator-run services are unavailable. + install_builtin_middleware_registry(&engine).await?; + + // Connect operator-registered middleware services. A connect/describe + // failure keeps the built-in registry active so each request's + // `on_error` policy governs matched traffic. The policy poll loop + // retries the install without waiting for a config change. + let middleware_services = snapshot.supervisor_middleware_services.clone(); + let middleware_registry_status = if middleware_services.is_empty() { + MiddlewareRegistryStatus::Synchronized + } else if let Err(error) = grpc_retry("Middleware connect", || { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + middleware_services.clone(), + ) + }) + .await + .and_then(|registry| engine.replace_middleware_registry(registry)) + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(middleware_services.len()) + ) + .message(format!( + "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" + )) + .build() + ); + MiddlewareRegistryStatus::NeedsReconciliation + } else { + MiddlewareRegistryStatus::Synchronized + }; + let opa_engine = Some(Arc::new(engine)); + + let policy = match SandboxPolicy::try_from(proto_policy.clone()) { + Ok(policy) => policy, + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + return Err(e); + } + }; + return Ok(( + policy, + opa_engine, + Some(proto_policy), + middleware_registry_status, + LoadedPolicyOrigin::Gateway { + revision: loaded_policy_revision, + }, + agent_proposals_enabled_from_settings(&snapshot.settings), + )); + } + + // No policy source available + Err(miette::miette!( + "Sandbox policy required. Provide one of:\n\ + - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ + - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" + )) +} + +/// Try to discover a sandbox policy from the well-known disk path, falling +/// back to the legacy path, then to the hardcoded restrictive default. +fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { + let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); + if primary.exists() { + return discover_policy_from_path(primary); + } + let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); + if legacy.exists() { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "legacy_path", + serde_json::json!(legacy.display().to_string()) + ) + .unmapped("new_path", serde_json::json!(primary.display().to_string())) + .message(format!( + "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", + legacy.display(), + primary.display() + )) + .build() + ); + return discover_policy_from_path(legacy); + } + discover_policy_from_path(primary) +} + +/// Try to read a sandbox policy YAML from `path`, falling back to the +/// hardcoded restrictive default if the file is missing or invalid. +fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { + use openshell_policy::{ + parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, + }; + + let Ok(yaml) = std::fs::read_to_string(path) else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "default") + .message(format!( + "No policy file on disk, using restrictive default [path:{}]", + path.display() + )) + .build() + ); + return restrictive_default_policy(); + }; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Loaded sandbox policy from container disk [path:{}]", + path.display() + )) + .build() + ); + match parse_sandbox_policy(&yaml) { + Ok(policy) => { + // Validate the disk-loaded policy for safety. + if let Err(violations) = validate_sandbox_policy(&policy) { + let messages: Vec = violations.iter().map(ToString::to_string).collect(); + ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Medium) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .finding_info( + FindingInfo::new( + "unsafe-disk-policy", + "Unsafe Disk Policy Content", + ) + .with_desc(&format!( + "Disk policy at {} contains unsafe content: {}", + path.display(), + messages.join("; "), + )), + ) + .message(format!( + "Disk policy contains unsafe content, using restrictive default [path:{}]", + path.display() + )) + .build()); + return restrictive_default_policy(); + } + policy + } + Err(e) => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "fallback") + .message(format!( + "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", + path.display() + )) + .build()); + restrictive_default_policy() + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MiddlewareRegistryStatus { + Synchronized, + NeedsReconciliation, +} + +/// True when the installed middleware registry no longer matches the desired +/// service set and must be rebuilt (reconnecting every delivered service). +/// +/// A policy-only change never requires a rebuild: middleware configs were +/// validated at gateway admission and the installed registry's manifests +/// already cover the unchanged service set, so requiring the services to be +/// reachable would only let a middleware outage block the policy update. +fn middleware_registry_needs_rebuild( + registry_status: MiddlewareRegistryStatus, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> bool { + registry_status == MiddlewareRegistryStatus::NeedsReconciliation + || current_services != desired_services +} + +fn gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy: bool, + current_policy_hash: &str, + desired_policy_hash: &str, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + registry_status: MiddlewareRegistryStatus, +) -> bool { + reloads_gateway_policy + && (current_policy_hash != desired_policy_hash + || middleware_registry_needs_rebuild( + registry_status, + current_services, + desired_services, + )) +} + +/// Identity returned with the exact policy snapshot used to construct OPA. +#[derive(Clone, Debug, PartialEq, Eq)] +struct LoadedPolicyRevision { + version: u32, + policy_hash: String, + config_revision: u64, + policy_source: openshell_core::proto::PolicySource, +} + +/// Identifies where the policy currently loaded into OPA came from. +/// +/// A missing gateway revision means the policy was loaded from the gateway but +/// could not be bound to an authoritative snapshot (for example, enrichment +/// sync failed). That state must reconcile on the first successful poll. A +/// local-file override is different: gateway policy revisions are observed for +/// settings/provider refreshes but must never replace the explicit local OPA +/// policy. +#[derive(Clone, Debug, PartialEq, Eq)] +enum LoadedPolicyOrigin { + LocalOverride, + Gateway { + revision: Option, + }, +} + +impl LoadedPolicyOrigin { + fn allows_gateway_policy_reload(&self) -> bool { + matches!(self, Self::Gateway { .. }) + } +} + +impl LoadedPolicyRevision { + fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { + Self { + version: snapshot.version, + policy_hash: snapshot.policy_hash.clone(), + config_revision: snapshot.config_revision, + policy_source: snapshot.policy_source, + } + } +} + +/// A sandbox-scoped policy revision that was constructed successfully at +/// startup and must be acknowledged to the gateway exactly once. +#[derive(Clone, Debug, PartialEq, Eq)] +struct InitialPolicyAck { + version: u32, + policy_hash: String, + config_revision: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PolicyStatusUpdate { + version: u32, + loaded: bool, + error: String, + initial_policy_hash: Option, +} + +impl PolicyStatusUpdate { + fn initial_loaded(ack: &InitialPolicyAck) -> Self { + Self { + version: ack.version, + loaded: true, + error: String::new(), + initial_policy_hash: Some(ack.policy_hash.clone()), + } + } + + fn loaded(version: u32) -> Self { + Self { + version, + loaded: true, + error: String::new(), + initial_policy_hash: None, + } + } + + fn failed(version: u32, error: String) -> Self { + Self { + version, + loaded: false, + error, + initial_policy_hash: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum InitialPollDisposition { + Acknowledge(InitialPolicyAck), + Reconcile, + TrackOnly, +} + +/// Determine whether the initially loaded policy corresponds to an +/// authoritative sandbox-scoped revision that must be acknowledged. +/// +/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose +/// captured gateway identity matches the current version and hash. Global +/// policies, local-file development policies, version zero, and changed +/// identities yield `None`, so those paths never emit a sandbox-revision +/// acknowledgement. +fn initial_policy_ack_candidate( + loaded: Option<&LoadedPolicyRevision>, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> Option { + let loaded = loaded?; + if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox + || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox + { + return None; + } + if loaded.version == 0 || canonical.version == 0 { + return None; + } + if loaded.version != canonical.version + || loaded.policy_hash != canonical.policy_hash + || canonical.config_revision < loaded.config_revision + { + return None; + } + Some(InitialPolicyAck { + version: loaded.version, + policy_hash: loaded.policy_hash.clone(), + config_revision: canonical.config_revision, + }) +} + +fn initial_poll_disposition( + origin: &LoadedPolicyOrigin, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> InitialPollDisposition { + match origin { + LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, + LoadedPolicyOrigin::Gateway { revision } => { + initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( + InitialPollDisposition::Reconcile, + InitialPollDisposition::Acknowledge, + ) + } + } +} + +/// Deliver policy status updates independently from policy reconciliation. +/// +/// The channel is FIFO, so a delayed older status can never arrive after a +/// newer status and move the gateway's active version backward. Delivery uses +/// the existing bounded retry, but failures never delay policy enforcement. +async fn run_policy_status_reporter( + client: openshell_core::grpc_client::CachedOpenShellClient, + sandbox_id: String, + mut updates: tokio::sync::mpsc::UnboundedReceiver, +) { + 'updates: while let Some(update) = updates.recv().await { + let operation = if update.initial_policy_hash.is_some() { + "Initial policy acknowledgement" + } else { + "Policy status report" + }; + let mut attempt = 1_u32; + loop { + let sandbox_id = sandbox_id.clone(); + let error = update.error.clone(); + let client = client.clone(); + match client + .report_policy_status(&sandbox_id, update.version, update.loaded, &error) + .await + { + Ok(()) => break, + Err(error) if is_retryable_error(&error) => { + let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); + warn!( + %error, + attempt, + version = update.version, + loaded = update.loaded, + retry_in_secs = backoff.as_secs(), + "{operation} failed transiently; retaining ordered update" + ); + tokio::time::sleep(backoff).await; + attempt = attempt.saturating_add(1); + } + Err(error) => { + warn!( + %error, + version = update.version, + loaded = update.loaded, + "Discarding terminal policy status update" + ); + continue 'updates; + } + } + } + + if let Some(policy_hash) = update.initial_policy_hash { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("version", serde_json::json!(update.version)) + .unmapped("policy_hash", serde_json::json!(policy_hash)) + .message(format!( + "Acknowledged initial policy revision as loaded [version:{}]", + update.version + )) + .build() + ); + } + } +} + +fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { + let version = update.version; + if let Err(error) = sender.send(update) { + warn!( + %error, + version, + "Policy status reporter unavailable during shutdown" + ); + } +} + +/// Best-effort `FAILED` acknowledgement when initial policy construction or +/// conversion fails. +/// +/// Uses the revision identity captured with the policy that failed to build, +/// and preserves the original construction error as the reported message. A +/// delivery failure here is swallowed so it can never mask that error. +async fn report_initial_policy_failure( + endpoint: &str, + sandbox_id: &str, + revision: Option<&LoadedPolicyRevision>, + error: &miette::Report, +) { + let Some(revision) = revision.filter(|revision| { + revision.version > 0 + && revision.policy_source == openshell_core::proto::PolicySource::Sandbox + }) else { + return; + }; + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + Ok(client) => client, + Err(e) => { + warn!(error = %e, "Failed to connect to report initial policy failure"); + return; + } + }; + let message = error.to_string(); + if let Err(e) = grpc_retry("Initial policy failure report", || { + let client = client.clone(); + let message = message.clone(); + async move { + client + .report_policy_status(sandbox_id, revision.version, false, &message) + .await + } + }) + .await + { + warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); + } } -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +/// Background loop that polls the server for policy updates. +/// +/// When a new version is detected, attempts to reload the OPA engine via +/// `reload_from_proto_with_pid()`. Reports load success/failure back to the +/// server. On failure, the previous engine is untouched (LKG behavior). +/// +/// When the entrypoint PID is available, policy reloads include symlink +/// resolution for binary paths via the container filesystem. +struct PolicyPollLoopContext { + endpoint: String, + sandbox_id: String, + opa_engine: Arc, + /// Source of the policy currently loaded into OPA. This distinguishes an + /// explicit local-file override from an unbound gateway revision so the + /// former is never replaced by policy polling. + loaded_policy_origin: LoadedPolicyOrigin, + entrypoint_pid: Arc, + interval_secs: u64, + ocsf_enabled: Arc, + provider_credentials: ProviderCredentialState, + policy_local_ctx: Option>, + agent_proposals: AgentProposals, + middleware_registry_status: MiddlewareRegistryStatus, + sidecar_control_publisher: Option, + workspace_tx: tokio::sync::watch::Sender, +} + +async fn connect_middleware_registry( + services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> Result { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + ) + .await +} + +async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + opa_engine.replace_middleware_registry(registry) +} + +async fn reconcile_middleware_registry( + opa_engine: &OpaEngine, + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + current_services: &mut Vec, + status: &mut MiddlewareRegistryStatus, +) { + if *status == MiddlewareRegistryStatus::Synchronized + && desired_services == current_services.as_slice() + { + return; + } + + match connect_middleware_registry(desired_services) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + current_services.clear(); + current_services.extend_from_slice(desired_services); + *status = MiddlewareRegistryStatus::Synchronized; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(current_services.len()) + ) + .message(format!( + "Supervisor middleware registry reloaded [service_count:{}]", + current_services.len() + )) + .build() + ); + } + Err(error) => { + // Emit only on the transition into the failed state to avoid + // repeating the same finding on every poll during an outage. + if *status == MiddlewareRegistryStatus::Synchronized { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + } + *status = MiddlewareRegistryStatus::NeedsReconciliation; + } + } +} + +async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::PolicySource; + use std::sync::atomic::Ordering; + + let client = CachedOpenShellClient::connect(&ctx.endpoint).await?; + let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(run_policy_status_reporter( + client.clone(), + ctx.sandbox_id.clone(), + status_receiver, + )); + + let mut current_config_revision: u64 = 0; + let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; + let mut current_policy_hash = String::new(); + let mut current_middleware_services = Vec::new(); + let mut middleware_registry_status = ctx.middleware_registry_status; + let mut current_settings: std::collections::HashMap< + String, + openshell_core::proto::EffectiveSetting, + > = std::collections::HashMap::new(); + let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); + let mut last_failed_runtime_revision: Option<(u64, String)> = None; + + // A first poll that does not match the policy already loaded into OPA must + // pass through the normal reconciliation path immediately. It must never + // seed the applied-state trackers before OPA actually loads it. + let mut pending_result = None; + + // Initialize revision from the first poll and acknowledge the initial + // policy revision the supervisor actually loaded. A mismatched result is + // reconciled below instead of being recorded as already applied. + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { + InitialPollDisposition::Acknowledge(candidate) => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(candidate.config_revision), + ctx.sidecar_control_publisher.as_ref(), + skills::install_static_skills, + ); + current_config_revision = candidate.config_revision; + current_policy_hash.clone_from(&candidate.policy_hash); + current_middleware_services = result.supervisor_middleware_services; + current_settings = result.settings; + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::initial_loaded(&candidate), + ); + debug!( + config_revision = current_config_revision, + "Settings poll: initial policy matches loaded revision" + ); + } + InitialPollDisposition::Reconcile => pending_result = Some(result), + InitialPollDisposition::TrackOnly => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(result.config_revision), + ctx.sidecar_control_publisher.as_ref(), + skills::install_static_skills, + ); + current_config_revision = result.config_revision; + current_policy_hash = result.policy_hash.clone(); + current_middleware_services = result.supervisor_middleware_services; + current_settings = result.settings; + debug!( + config_revision = current_config_revision, + "Settings poll: tracking gateway config while preserving local policy override" + ); + } + } + } + Err(e) => { + warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); + } + } + + let interval = Duration::from_secs(ctx.interval_secs); + loop { + let result = if let Some(result) = pending_result.take() { + result + } else { + tokio::time::sleep(interval).await; + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + result + } + Err(e) => { + debug!(error = %e, "Settings poll: server unreachable, will retry"); + continue; + } + } + }; + + let config_changed = result.config_revision != current_config_revision; + let provider_env_changed = result.provider_env_revision != current_provider_env_revision; + let policy_changed = result.policy_hash != current_policy_hash; + let middleware_registry_changed = middleware_registry_needs_rebuild( + middleware_registry_status, + ¤t_middleware_services, + &result.supervisor_middleware_services, + ); + let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); + + // A local policy override is not coupled to the gateway policy + // snapshot, so its service registry can still be reconciled alone. + // Gateway policy snapshots, however, must install policy and registry + // as one generation below. + if !reloads_gateway_policy { + reconcile_middleware_registry( + &ctx.opa_engine, + &result.supervisor_middleware_services, + &mut current_middleware_services, + &mut middleware_registry_status, + ) + .await; + } + + if !config_changed && !provider_env_changed && !policy_runtime_changed { + continue; + } + + if config_changed || provider_env_changed { + // Log which settings changed. + log_setting_changes(¤t_settings, &result.settings); + + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "detected") + .unmapped("old_config_revision", serde_json::json!(current_config_revision)) + .unmapped("new_config_revision", serde_json::json!(result.config_revision)) + .unmapped("policy_changed", serde_json::json!(policy_changed)) + .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) + .message(format!( + "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", + result.config_revision + )) + .build()); + } + + if provider_env_changed { + match openshell_core::grpc_client::fetch_provider_environment( + &ctx.endpoint, + &ctx.sandbox_id, + ) + .await + { + Ok(env_result) => { + ctx.provider_credentials.install_environment( + env_result.provider_env_revision, + env_result.environment, + env_result.credential_expires_at_ms, + env_result.dynamic_credentials, + ); + let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); + let env_count = child_env.len(); + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_provider_env( + env_result.provider_env_revision, + child_env.clone(), + ); + } + current_provider_env_revision = env_result.provider_env_revision; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "provider_env_revision", + serde_json::json!(env_result.provider_env_revision) + ) + .message(format!( + "Provider environment refreshed [revision:{} env_count:{env_count}]", + env_result.provider_env_revision + )) + .build() + ); + } + Err(e) => { + warn!( + error = %e, + provider_env_revision = result.provider_env_revision, + "Settings poll: failed to refresh provider environment" + ); + } + } + } + + if policy_runtime_changed { + let pid = ctx.entrypoint_pid.load(Ordering::Acquire); + let runtime_result = match result.policy.as_ref() { + Some(policy) if middleware_registry_changed => { + match connect_middleware_registry(&result.supervisor_middleware_services).await + { + Ok(registry) => ctx + .opa_engine + .reload_policy_and_middleware_from_proto_with_pid( + policy, pid, registry, + ), + Err(error) => Err(error), + } + } + // Policy-only change: the installed registry already matches + // the delivered service set, so swap the engine alone. This + // must not require middleware reachability. + Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), + None => Err(miette::miette!( + "runtime reload requires a policy payload but none was returned" + )), + }; + + match runtime_result { + Ok(()) => { + let policy = result + .policy + .as_ref() + .expect("successful runtime reload requires a policy payload"); + if policy_changed { + if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { + policy_local_ctx.set_current_policy(policy.clone()).await; + } + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_policy( + policy.clone(), + result.policy_hash.clone(), + result.config_revision, + ); + } + if result.global_policy_version > 0 { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .unmapped("global_version", serde_json::json!(result.global_policy_version)) + .message(format!( + "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", + result.policy_hash, + result.global_policy_version + )) + .build()); + } else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + } + if result.version > 0 && result.policy_source == PolicySource::Sandbox { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + } + } + + if middleware_registry_changed { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(result.supervisor_middleware_services.len()) + ) + .message(format!( + "Supervisor policy runtime reloaded atomically [service_count:{}]", + result.supervisor_middleware_services.len() + )) + .build()); + } + + current_policy_hash.clone_from(&result.policy_hash); + current_middleware_services.clone_from(&result.supervisor_middleware_services); + middleware_registry_status = MiddlewareRegistryStatus::Synchronized; + last_failed_runtime_revision = None; + } + Err(e) => { + let failed_revision = (result.config_revision, result.policy_hash.clone()); + if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .unmapped("version", serde_json::json!(result.version)) + .unmapped("error", serde_json::json!(e.to_string())) + .message(format!( + "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", + result.version + )) + .build()); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, e.to_string()), + ); + } + } + last_failed_runtime_revision = Some(failed_revision); + // Nothing was installed, so the registry status still + // describes the live registry. The retry is driven by the + // persisting hash/service-set mismatch (or an existing + // NeedsReconciliation), not by degrading the status here. + } + } + } + + // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + + // Apply the agent-proposals feature toggle. On a false→true transition + // we lazily install the skill so a sandbox that started with the flag + // off picks up the surface without a recreate. We never uninstall on + // a true→false transition: stale skill content on disk is harmless + // because route_request and agent_next_steps both gate on the live + // shared flag, so the agent that reads the skill will see 404s and an + // empty `next_steps` array regardless. + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "settings poll", + Some(result.config_revision), + ctx.sidecar_control_publisher.as_ref(), + skills::install_static_skills, + ); + + current_config_revision = result.config_revision; + if !reloads_gateway_policy { + current_policy_hash = result.policy_hash; + } + current_settings = result.settings; + } +} + +fn apply_ocsf_json_setting( + enabled: &AtomicBool, + settings: &std::collections::HashMap, +) { + use std::sync::atomic::Ordering; + + let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); + let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); + if new_ocsf != prev_ocsf { + info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); + } +} + +/// Extract a bool value from an effective setting, if present. +fn extract_bool_setting( + settings: &std::collections::HashMap, + key: &str, +) -> Option { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|v| match v { + setting_value::Value::BoolValue(b) => Some(*b), + _ => None, + }) +} + +fn agent_proposals_enabled_from_settings( + settings: &std::collections::HashMap, +) -> bool { + extract_bool_setting( + settings, + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, + ) + .unwrap_or(false) +} + +fn apply_agent_proposals_enabled( + agent_proposals: &AgentProposals, + enabled: bool, + source: &'static str, + config_revision: Option, + sidecar_control_publisher: Option<&sidecar_control::Publisher>, + install_static_skills: impl FnOnce() -> Result, +) { + let previously_enabled = agent_proposals.swap_enabled(enabled); + if enabled == previously_enabled { + return; + } + + info!( + agent_policy_proposals_enabled = enabled, + source, config_revision, "agent-driven policy proposals toggled" + ); + + if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { + publisher.publish_agent_proposals(enabled, config_revision); + } + + if enabled && !previously_enabled { + match install_static_skills() { + Ok(installed) => info!( + path = %installed.policy_advisor.display(), + "Installed sandbox agent skill on toggle-on" + ), + Err(error) => warn!( + error = %error, + "Failed to install sandbox agent skill on toggle-on" + ), + } + } +} + +/// Log individual setting changes between two snapshots. +fn log_setting_changes( + old: &std::collections::HashMap, + new: &std::collections::HashMap, +) { + for (key, new_es) in new { + let new_val = format_setting_value(new_es); + match old.get(key) { + Some(old_es) => { + let old_val = format_setting_value(old_es); + if old_val != new_val { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "updated") + .unmapped("key", serde_json::json!(key)) + .unmapped("old", serde_json::json!(old_val.clone())) + .unmapped("new", serde_json::json!(new_val.clone())) + .message(format!( + "Setting changed [key:{key} old:{old_val} new:{new_val}]" + )) + .build() + ); + } + } + None => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enabled") + .unmapped("key", serde_json::json!(key)) + .unmapped("value", serde_json::json!(new_val.clone())) + .message(format!("Setting added [key:{key} value:{new_val}]")) + .build() + ); + } + } + } + for key in old.keys() { + if !new.contains_key(key) { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Disabled, "disabled") + .unmapped("key", serde_json::json!(key)) + .message(format!("Setting removed [key:{key}]")) + .build() + ); + } + } +} + +/// Format an `EffectiveSetting` value for log display. +fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { + use openshell_core::proto::setting_value; + match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { + None => "".to_string(), + Some(setting_value::Value::StringValue(v)) => v.clone(), + Some(setting_value::Value::BoolValue(v)) => v.to_string(), + Some(setting_value::Value::IntValue(v)) => v.to_string(), + Some(setting_value::Value::BytesValue(_)) => "".to_string(), + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, + }; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + fn proxy_policy(http_addr: Option) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { + openshell_core::proto::EffectiveSetting { + value: Some(openshell_core::proto::SettingValue { + value: Some(openshell_core::proto::setting_value::Value::BoolValue( + value, + )), + }), + scope: openshell_core::proto::SettingScope::Global.into(), + } + } + + #[test] + fn sidecar_process_policy_sets_loopback_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, true).unwrap(); + + let http_addr = process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .expect("sidecar process policy should set proxy address"); + assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); + assert!( + policy + .network + .proxy + .as_ref() + .expect("original policy should keep proxy config") + .http_addr + .is_none(), + "process policy normalization must not mutate the network policy" + ); + } + + #[test] + fn non_sidecar_process_policy_preserves_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, false).unwrap(); + + assert!( + process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .is_none() + ); + } + + #[tokio::test] + async fn sidecar_control_provider_env_update_installs_newer_revision() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + 1, + std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), + ); + let handle = spawn_sidecar_control_update_watcher( + rx, + provider_credentials.clone(), + AgentProposals::default(), + ); + + tx.send(sidecar_control::ControlUpdate::ProviderEnv { + revision: 2, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "new".to_string(), + )]), + }) + .unwrap(); + + timeout(Duration::from_secs(1), async { + loop { + if provider_credentials.snapshot().revision == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let snapshot = provider_credentials.snapshot(); + assert_eq!(snapshot.revision, 2); + assert_eq!( + snapshot.child_env.get("TOKEN").map(String::as_str), + Some("new") + ); + + tx.send(sidecar_control::ControlUpdate::ProviderEnv { + revision: 1, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "stale".to_string(), + )]), + }) + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + provider_credentials + .snapshot() + .child_env + .get("TOKEN") + .map(String::as_str), + Some("new") + ); + handle.abort(); + } + + #[tokio::test] + async fn sidecar_control_agent_proposals_update_flips_shared_state() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let provider_credentials = + ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); + let agent_proposals = AgentProposals::new(true); + let handle = + spawn_sidecar_control_update_watcher(rx, provider_credentials, agent_proposals.clone()); + + tx.send(sidecar_control::ControlUpdate::AgentProposals { + enabled: false, + config_revision: 5, + }) + .unwrap(); + + timeout(Duration::from_secs(1), async { + loop { + if !agent_proposals.enabled() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + handle.abort(); + } + + #[test] + fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { + let agent_proposals = AgentProposals::default(); + let installs = AtomicUsize::new(0); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(!agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + } + + #[test] + fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { + let enabled = AtomicBool::new(false); + let mut settings = std::collections::HashMap::new(); + settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(enabled.load(Ordering::Relaxed)); + } + + #[test] + fn apply_ocsf_json_setting_disables_when_setting_is_unset() { + let enabled = AtomicBool::new(true); + let settings = std::collections::HashMap::new(); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(!enabled.load(Ordering::Relaxed)); + } + + #[test] + fn agent_proposals_setting_enables_from_initial_settings_snapshot() { + let mut settings = std::collections::HashMap::new(); + settings.insert( + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), + effective_bool(true), + ); + + assert!(agent_proposals_enabled_from_settings(&settings)); + } + + #[test] + fn agent_proposals_setting_defaults_false_when_unset() { + let settings = std::collections::HashMap::new(); + + assert!(!agent_proposals_enabled_from_settings(&settings)); + } + + // ---- Policy disk discovery tests ---- + + #[test] + fn discover_policy_from_nonexistent_path_returns_restrictive_default() { + let path = std::path::Path::new("/nonexistent/policy.yaml"); + let policy = discover_policy_from_path(path); + // Restrictive default has no network policies. + assert!(policy.network_policies.is_empty()); + // But does have filesystem and process policies. + assert!(policy.filesystem.is_some()); + assert!(policy.process.is_some()); + } + + #[test] + fn discover_policy_from_valid_yaml_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +filesystem_policy: + include_workdir: false + read_only: + - /usr + read_write: + - /tmp +network_policies: + test: + name: test + endpoints: + - { host: example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + assert_eq!(policy.network_policies.len(), 1); + assert!(policy.network_policies.contains_key("test")); + let fs = policy.filesystem.unwrap(); + assert!(!fs.include_workdir); + } + + #[test] + fn discover_policy_from_invalid_yaml_returns_restrictive_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default. + assert!(policy.network_policies.is_empty()); + assert!(policy.filesystem.is_some()); + } + + #[test] + fn discover_policy_from_unsafe_yaml_falls_back_to_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +process: + run_as_user: root + run_as_group: root +filesystem_policy: + include_workdir: true + read_only: + - /usr + read_write: + - /tmp +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default because of root user. + let proc = policy.process.unwrap(); + assert_eq!(proc.run_as_user, "sandbox"); + assert_eq!(proc.run_as_group, "sandbox"); + } + + #[test] + fn discover_policy_restrictive_default_blocks_network() { + // In cluster mode we keep proxy mode enabled so `inference.local` + // can always be routed through proxy/OPA controls. + let proto = openshell_policy::restrictive_default_policy(); + let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); + assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); + } + + // ---- Initial policy acknowledgement tests ---- + + fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::restrictive_default_policy() + } + + fn settings_poll_result( + policy: Option, + version: u32, + source: openshell_core::proto::PolicySource, + ) -> openshell_core::grpc_client::SettingsPollResult { + openshell_core::grpc_client::SettingsPollResult { + policy, + version, + policy_hash: format!("hash-v{version}"), + config_revision: u64::from(version) * 100, + policy_source: source, + settings: std::collections::HashMap::new(), + global_policy_version: 0, + provider_env_revision: 0, + supervisor_middleware_services: Vec::new(), + workspace: String::new(), + } + } + + #[tokio::test] + async fn failed_external_startup_registry_build_preserves_installed_builtins() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let builtins_generation = engine.current_generation(); + assert_eq!(builtins_generation, 1); + + let invalid_external = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_body_bytes: 1024, + ..Default::default() + }; + connect_middleware_registry(&[invalid_external]) + .await + .expect_err("unavailable external service must not replace built-ins"); + + assert_eq!(engine.current_generation(), builtins_generation); + } + + #[test] + fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { + let services = Vec::new(); + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + } + + #[test] + fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &no_services, + &no_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &no_services, + &desired_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + false, + "local-policy", + "hash-v2", + &no_services, + &desired_services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + } + + #[test] + fn policy_only_change_does_not_rebuild_middleware_registry() { + let services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + // The runtime must reconcile, but the registry (and therefore + // middleware reachability) is not part of that reconciliation. + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &services, + &services, + )); + } + + #[test] + fn registry_rebuild_requires_service_set_change_or_degraded_registry() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &no_services, + &desired_services, + )); + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::NeedsReconciliation, + &desired_services, + &desired_services, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &desired_services, + &desired_services, + )); + } + + #[test] + fn initial_ack_candidate_matches_sandbox_revision() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) + .expect("sandbox-sourced matching revision should be acknowledged"); + + assert_eq!(ack.version, 2); + assert_eq!(ack.policy_hash, "hash-v2"); + assert_eq!(ack.config_revision, 200); + } + + #[test] + fn initial_ack_candidate_ignores_global_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Global, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_version_zero() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 0, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_local_file_mode() { + // Local-file mode retains no proto policy, so there is nothing to + // acknowledge to the gateway. + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(None, &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_rejects_mismatched_identity() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let mut newer = proto_policy_fixture(); + newer.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule::default(), + ); + let canonical = + settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); + let canonical = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "hash-provider-change".to_string(), + config_revision: loaded.config_revision + 1, + ..canonical + }; + + assert_eq!( + initial_poll_disposition( + &LoadedPolicyOrigin::Gateway { + revision: Some(loaded), + }, + &canonical, + ), + InitialPollDisposition::Reconcile + ); + } + + #[test] + fn initial_poll_tracks_local_override_without_reconciliation() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert_eq!( + initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), + InitialPollDisposition::TrackOnly + ); + assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); + } + + #[test] + fn initial_poll_reconciles_unbound_gateway_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let origin = LoadedPolicyOrigin::Gateway { revision: None }; + + assert_eq!( + initial_poll_disposition(&origin, &canonical), + InitialPollDisposition::Reconcile + ); + assert!(origin.allows_gateway_policy_reload()); + } + + #[test] + fn policy_status_outbox_preserves_all_revision_order() { + let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); + for version in 1..=128 { + enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); + } + + for version in 1..=128 { + assert_eq!( + receiver.try_recv().unwrap(), + PolicyStatusUpdate::loaded(version) + ); + } + } + + #[test] + fn settings_snapshot_carries_workspace_for_policy_sync() { + let mut snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + snapshot.workspace = "beta".to_string(); + + let revision = LoadedPolicyRevision::from_snapshot(&snapshot); + assert_eq!(revision.version, 1); + assert_eq!( + snapshot.workspace, "beta", + "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" + ); + } +} diff --git a/crates/openshell-sandbox/src/lib_entry.rs b/crates/openshell-sandbox/src/lib_entry.rs new file mode 100644 index 0000000000..4272e7c537 --- /dev/null +++ b/crates/openshell-sandbox/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-sandbox/src/lib_unix.rs b/crates/openshell-sandbox/src/lib_unix.rs deleted file mode 100644 index e696403f39..0000000000 --- a/crates/openshell-sandbox/src/lib_unix.rs +++ /dev/null @@ -1,3773 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! `OpenShell` Sandbox library. -//! -//! This crate provides process sandboxing and monitoring capabilities. - -mod activity_aggregator; -mod denial_aggregator; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod google_cloud_metadata; -mod mechanistic_mapper; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod metadata_server; -mod sidecar_control; - -use miette::{IntoDiagnostic, Result, WrapErr}; -use std::future::Future; -use std::sync::Arc; -#[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicBool, AtomicU32}; -use std::time::Duration; -use tracing::{debug, info, warn}; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, - DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, -}; - -// --------------------------------------------------------------------------- -// OCSF Context -// --------------------------------------------------------------------------- -// -// The following log sites intentionally remain as plain `tracing` macros -// and are NOT migrated to OCSF builders: -// -// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) -// - Transient "about to do X" events where the result is logged separately -// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") -// - Internal SSH channel warnings (unknown channel, PTY resize failures) -// - Denial flush telemetry (the individual denials are already OCSF events) -// - Status reporting failures (sync to gateway, non-actionable) -// - Route refresh interval validation warnings -// -// These are operational plumbing that don't represent security decisions, -// policy changes, or observable sandbox behavior worth structuring. -// --------------------------------------------------------------------------- - -/// Re-export the process-wide OCSF sandbox context getter. -/// -/// The singleton lives in `openshell-ocsf` so both supervisor leaves can -/// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. -pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; - -use openshell_core::denial::DenialEvent; -use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_supervisor_network::opa::OpaEngine; -use openshell_supervisor_process::process::ProcessEnforcementMode; -pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; -use openshell_supervisor_process::skills; -use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] -use tokio::time::timeout; - -const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; -const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; -const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; -const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; -const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; -const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; - -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, - inference_routes: Option, - ocsf_enabled: Arc, - network_enabled: bool, - process_enabled: bool, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, -) -> Result { - let (program, args) = command - .split_first() - .ok_or_else(|| miette::miette!("No command specified"))?; - - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; they are only significant for network - // events which happen after the netns is created. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(SandboxContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); - let process_uses_sidecar_control = - process_enabled && !network_enabled && sidecar_network_enforcement; - let mut process_control_connection = None; - let sidecar_bootstrap = if process_uses_sidecar_control { - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let (bootstrap, connection) = sidecar_control::connect_process_client( - &socket, - Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), - ) - .await?; - process_control_connection = Some(connection); - Some(bootstrap) - } else { - None - }; - - // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); - let sandbox_name_for_agg = sandbox.clone(); - let ( - mut policy, - opa_engine, - retained_proto, - middleware_registry_status, - loaded_policy_origin, - initial_agent_proposals_enabled, - ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let (policy, opa_engine, retained_proto, loaded_policy_origin) = - load_policy_from_sidecar_bootstrap(bootstrap)?; - ( - policy, - opa_engine, - retained_proto, - MiddlewareRegistryStatus::Synchronized, - loaded_policy_origin, - bootstrap.agent_proposals_enabled, - ) - } else { - load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - ) - .await? - }; - - // Override the policy's process identity with the driver-resolved UID/GID - // from the pod environment. The policy defaults to the name "sandbox" which - // resolves via /etc/passwd, but the driver may have chosen a different - // numeric UID (e.g. from OpenShift SCC annotations). - // Validate overrides against the same rules as the policy layer to prevent - // env-injected values (e.g. GID 0) from bypassing policy restrictions. - if let Ok(uid) = std::env::var(openshell_core::sandbox_env::SANDBOX_UID) - && !uid.is_empty() - { - if !openshell_policy::is_valid_sandbox_identity(&uid) { - return Err(miette::miette!( - "OPENSHELL_SANDBOX_UID contains invalid sandbox identity '{uid}'; \ - expected 'sandbox' or a numeric UID in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - policy.process.run_as_user = Some(uid); - } - if let Ok(gid) = std::env::var(openshell_core::sandbox_env::SANDBOX_GID) - && !gid.is_empty() - { - if !openshell_policy::is_valid_sandbox_identity(&gid) { - return Err(miette::miette!( - "OPENSHELL_SANDBOX_GID contains invalid sandbox identity '{gid}'; \ - expected 'sandbox' or a numeric GID in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - policy.process.run_as_group = Some(gid); - } - - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = - if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - bootstrap.provider_env_revision, - bootstrap.provider_child_env.clone(), - ); - (provider_credentials, bootstrap.provider_child_env.clone()) - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .message(format!( - "Failed to fetch provider environment, continuing without: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - } - } - } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - }; - - let provider_credentials = ProviderCredentialState::from_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ); - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - - // Shared agent-proposals feature flag. Seed from the same initial settings - // snapshot that produced the policy so networking and process setup agree - // before the poll loop starts reconciling later changes. - let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); - - let process_control_writer = process_control_connection - .as_ref() - .map(|connection| connection.writer.clone()); - let mut process_control_closed = None; - if let Some(connection) = process_control_connection { - process_control_closed = Some(connection.closed); - spawn_sidecar_control_update_watcher( - connection.updates, - provider_credentials.clone(), - agent_proposals.clone(), - ); - } - - // Shared PID: set after process spawn so the proxy can look up - // the entrypoint process's /proc/net/tcp for identity binding. - let entrypoint_pid = Arc::new(AtomicU32::new(0)); - - // Create the workload's network namespace. It is shared infrastructure: - // the proxy binds to its host-side veth IP, the bypass monitor reads - // /dev/kmsg from inside it, and the workload child / SSH sessions enter - // it via setns(). The RAII handle lives in this frame for the duration - // of the sandbox. - #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { - openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? - } else { - None - }; - - // The denial channel is owned by the orchestrator: the proxy (in the - // networking leaf) and the bypass monitor (in the process leaf) both - // produce DenialEvents that the denial aggregator (orchestrator-side) - // consumes via the matching receiver. Both leaves are pure producers; - // the orchestrator owns the consumer task spawned below. - let (denial_tx, denial_rx, bypass_denial_tx): ( - Option>, - _, - Option>, - ) = if sandbox_id.is_some() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_denial_tx); - - // Anonymous activity channel: same orchestrator-owned pattern as the - // denial channel. The proxy and the bypass monitor both emit per-event - // activity records; the orchestrator-side aggregator drains, sanitizes, - // and flushes anonymous summaries to the gateway. - let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { - let (tx, rx) = - tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_activity_tx); - - // Workspace watch: the policy poll loop learns the workspace from - // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local - // API read the current value so proposals target the correct workspace. - let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - - let networking = if network_enabled { - #[cfg(target_os = "linux")] - let proxy_bind_ip = netns - .as_ref() - .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); - #[cfg(not(target_os = "linux"))] - let proxy_bind_ip: Option = None; - - Some( - openshell_supervisor_network::run::run_networking( - &policy, - proxy_bind_ip, - opa_engine.as_ref(), - retained_proto.as_ref(), - entrypoint_pid.clone(), - process_enabled, - &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), - inference_routes.as_deref(), - denial_tx, - activity_tx, - agent_proposals.clone(), - workspace_rx.clone(), - &upstream_proxy_args, - ) - .await?, - ) - } else { - None - }; - - #[cfg(target_os = "linux")] - let sidecar_control_server = if network_enabled && sidecar_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" - )); - } - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let proto = retained_proto.as_ref().ok_or_else(|| { - miette::miette!( - "sidecar topology requires gateway policy data for the process supervisor" - ) - })?; - let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); - Some(sidecar_control::spawn_server( - &socket, - sidecar_control::BootstrapData { - policy_proto: proto.clone(), - provider_env_revision: provider_credentials.snapshot().revision, - provider_child_env: provider_env.clone(), - agent_proposals_enabled: agent_proposals.enabled(), - proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), - proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), - }, - sidecar_expected_peer()?, - )?) - } else { - None - }; - #[cfg(not(target_os = "linux"))] - let sidecar_control_server: Option = None; - - let sidecar_control_publisher = sidecar_control_server - .as_ref() - .map(sidecar_control::ServerHandle::publisher); - - #[cfg(target_os = "linux")] - let mut sidecar_control_task = None; - - #[cfg(target_os = "linux")] - if network_enabled - && sidecar_network_enforcement - && let Some(server) = sidecar_control_server - { - let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar network topology", - openshell_core::sandbox_env::SSH_SOCKET_PATH - ) - })?; - let (entrypoint_rx, connection_task) = server.into_runtime_parts(); - sidecar_control_task = Some(connection_task); - spawn_sidecar_entrypoint_handler( - entrypoint_rx, - entrypoint_pid.clone(), - opa_engine.clone(), - retained_proto.clone(), - openshell_endpoint.clone(), - sandbox_id.clone(), - std::path::PathBuf::from(trusted_ssh_socket_path), - ); - } - - #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { - return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" - )); - } - - // Spawn the denial-aggregator flush task. The aggregator drains denial - // events from the proxy + bypass monitor, batches them, and ships - // summaries to the gateway via `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { - // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall - // back to the ID when the name isn't set. - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - - let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); - let denial_workspace_gate = workspace_rx.clone(); - let denial_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - |summaries| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = denial_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_proposals_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summaries, - ) - .await - { - warn!(error = %e, "Failed to flush denial summaries to gateway"); - } - } - }, - move || !denial_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn the activity-aggregator flush task. The aggregator drains - // anonymous activity events from the proxy, sanitizes deny groups, - // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( - std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") - .ok() - .as_deref(), - ); - - let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); - let activity_workspace_gate = workspace_rx.clone(); - let activity_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - move |summary| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = activity_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_activity_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summary, - ) - .await - { - warn!(error = %e, "Failed to flush activity summary to gateway"); - } - } - }, - move || !activity_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn background policy poll task (gRPC mode only). - if !process_uses_sidecar_control - && let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) - { - let poll_id = id.to_string(); - let poll_endpoint = endpoint.to_string(); - let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_pid = entrypoint_pid.clone(); - let poll_provider_credentials = provider_credentials.clone(); - let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); - let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, - sandbox_id: poll_id, - opa_engine: poll_engine, - loaded_policy_origin, - entrypoint_pid: poll_pid, - interval_secs: poll_interval_secs, - ocsf_enabled: poll_ocsf_enabled, - provider_credentials: poll_provider_credentials, - policy_local_ctx: poll_policy_local, - agent_proposals: agent_proposals.clone(), - middleware_registry_status, - sidecar_control_publisher: sidecar_control_publisher.clone(), - workspace_tx, - }; - - tokio::spawn(async move { - if let Err(e) = run_policy_poll_loop(poll_ctx).await { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .message(format!("Policy poll loop exited with error: {e}")) - .build() - ); - } - }); - } - - // Start GCE metadata loopback server inside the network namespace so - // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) - // can reach it via direct TCP. Must start before the process leaf so SSH - // sessions also see corrected env vars on bind failure. - #[cfg(target_os = "linux")] - if let Some(ns) = netns.as_ref() - && provider_credentials - .snapshot() - .child_env - .contains_key("GCE_METADATA_HOST") - { - let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); - let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); - match ns - .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) - .await - { - Ok(listener) => { - tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); - if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { - info!(addr = %addr, "GCE metadata loopback server ready"); - } else { - warn!("GCE metadata server failed to become ready, removing metadata env vars"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - Err(e) => { - warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - } - - let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { - bootstrap - .proxy_ca_cert_path - .clone() - .zip(bootstrap.proxy_ca_bundle_path.clone()) - }); - - let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None - } - }); - - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok(pid) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid).await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); - } - } - }); - Some(tx) - } else { - None - }; - - let process = openshell_supervisor_process::run::run_process( - program, - args, - workdir.as_deref(), - timeout_secs, - interactive, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - &process_policy, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - provider_credentials, - provider_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); - - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - } - } else { - process.await? - } - } else { - // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until shutdown. If the - // sole authenticated process-supervisor control connection closes, - // exit non-zero so Kubernetes restarts the network sidecar and creates - // a fresh one-client bootstrap listener for the restarted agent. - #[cfg(target_os = "linux")] - if let Some(control_task) = sidecar_control_task { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - result = control_task => { - warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); - 1 - } - } - } else { - wait_for_shutdown_signal().await; - 0 - } - #[cfg(not(target_os = "linux"))] - { - wait_for_shutdown_signal().await; - 0 - } - }; - - // Drop networking explicitly so the proxy + bypass monitor RAII - // handles tear down before we return. - drop(networking); - - Ok(exit_code) -} - -/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is -/// no entrypoint child whose lifetime drives the supervisor's exit. -async fn wait_for_shutdown_signal() { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(s) => s, - Err(e) => { - tracing::warn!( - error = %e, - "Failed to install SIGTERM handler; waiting on SIGINT only" - ); - let _ = tokio::signal::ctrl_c().await; - return; - } - }; - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Received SIGINT, shutting down network-only supervisor"); - } - _ = sigterm.recv() => { - info!("Received SIGTERM, shutting down network-only supervisor"); - } - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - info!("Received Ctrl-C, shutting down network-only supervisor"); - } -} - -fn sidecar_network_enforcement_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) - .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) -} - -fn process_enforcement_mode() -> ProcessEnforcementMode { - match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .ok() - .as_deref() - { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, - _ => ProcessEnforcementMode::Full, - } -} - -fn sidecar_control_socket() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) - .ok() - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn sidecar_expected_peer() -> Result { - fn required_numeric_env(name: &str) -> Result { - let value = std::env::var(name) - .into_diagnostic() - .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; - value.parse::().into_diagnostic().wrap_err_with(|| { - format!("{name} must be a numeric ID for sidecar control authentication") - }) - } - - Ok(sidecar_control::ExpectedPeer { - uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, - gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, - }) -} - -type LoadedPolicyBundle = ( - SandboxPolicy, - Option>, - Option, - LoadedPolicyOrigin, -); - -fn load_policy_from_sidecar_bootstrap( - bootstrap: &sidecar_control::BootstrapData, -) -> Result { - let proto = bootstrap.policy_proto.clone(); - let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); - let policy = SandboxPolicy::try_from(proto.clone())?; - info!("Loaded sidecar policy from control socket bootstrap"); - Ok(( - policy, - opa_engine, - Some(proto), - LoadedPolicyOrigin::Gateway { revision: None }, - )) -} - -fn spawn_sidecar_control_update_watcher( - mut updates: tokio::sync::mpsc::UnboundedReceiver, - provider_credentials: ProviderCredentialState, - agent_proposals: AgentProposals, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - while let Some(update) = updates.recv().await { - match update { - sidecar_control::ControlUpdate::ProviderEnv { - revision, - provider_child_env, - } => { - if revision <= provider_credentials.snapshot().revision { - continue; - } - let env_count = provider_credentials - .install_child_env_snapshot(revision, provider_child_env); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("provider_env_revision", serde_json::json!(revision)) - .message(format!( - "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" - )) - .build() - ); - } - sidecar_control::ControlUpdate::Policy { - policy_proto, - policy_hash, - config_revision, - } => { - debug!( - version = policy_proto.version, - policy_hash, - config_revision, - "Received sidecar policy update for process supervisor" - ); - } - sidecar_control::ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - apply_agent_proposals_enabled( - &agent_proposals, - enabled, - "sidecar control", - Some(config_revision), - None, - skills::install_static_skills, - ); - } - } - } - }) -} - -#[cfg(target_os = "linux")] -fn spawn_sidecar_entrypoint_handler( - mut entrypoint_rx: tokio::sync::mpsc::Receiver, - entrypoint_pid: Arc, - opa_engine: Option>, - retained_proto: Option, - openshell_endpoint: Option, - sandbox_id: Option, - trusted_ssh_socket_path: std::path::PathBuf, -) { - tokio::spawn(async move { - let mut session_started = false; - let mut trusted_supervisor_pid = None; - let terminating = Arc::new(AtomicBool::new(false)); - while let Some(started) = entrypoint_rx.recv().await { - entrypoint_pid.store(started.pid, Ordering::Release); - if started.start_session { - info!( - pid = started.pid, - ssh_socket = %trusted_ssh_socket_path.display(), - "Sidecar process supervisor reported entrypoint start" - ); - } else { - trusted_supervisor_pid = Some(started.pid); - info!( - pid = started.pid, - "Sidecar process supervisor reported initial process anchor" - ); - } - - if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { - match engine.reload_from_proto_with_pid(proto, started.pid) { - Ok(()) => info!( - pid = started.pid, - "Policy binary symlink resolution complete for sidecar process anchor" - ), - Err(err) => warn!( - error = %err, - pid = started.pid, - "Failed to rebuild OPA engine with sidecar process anchor PID" - ), - } - } - - if started.start_session - && !session_started - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let Some(supervisor_pid) = trusted_supervisor_pid else { - warn!( - pid = started.pid, - "Ignoring sidecar entrypoint event before authenticated supervisor anchor" - ); - continue; - }; - openshell_supervisor_process::supervisor_session::spawn( - endpoint.clone(), - id.clone(), - trusted_ssh_socket_path.clone(), - None, - Some(supervisor_pid), - Arc::clone(&terminating), - ); - session_started = true; - info!("sidecar supervisor session task spawned"); - } - } - terminating.store(true, Ordering::Release); - }); -} - -fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) - .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); - let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); - let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); - (cert.exists() && bundle.exists()).then_some((cert, bundle)) -} - -fn process_policy_for_topology( - policy: &SandboxPolicy, - sidecar_network_enforcement: bool, -) -> Result { - let mut process_policy = policy.clone(); - if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { - let proxy = process_policy - .network - .proxy - .get_or_insert(ProxyPolicy { http_addr: None }); - if proxy.http_addr.is_none() { - proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); - } - } - Ok(process_policy) -} - -/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. -async fn flush_proposals_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summaries: Vec, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialSummary, L7RequestSample}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summaries: Vec = summaries - .into_iter() - .map(|s| DenialSummary { - sandbox_id: String::new(), - host: s.host, - port: u32::from(s.port), - binary: s.binary, - ancestors: s.ancestors, - deny_reason: s.deny_reason, - first_seen_ms: s.first_seen_ms, - last_seen_ms: s.last_seen_ms, - count: s.count, - suppressed_count: 0, - total_count: s.count, - sample_cmdlines: s.sample_cmdlines, - binary_sha256: String::new(), - persistent: false, - denial_stage: s.denial_stage, - l7_request_samples: s - .l7_samples - .into_iter() - .map(|l| L7RequestSample { - method: l.method, - path: l.path, - decision: "deny".to_string(), - count: l.count, - }) - .collect(), - l7_inspection_active: false, - }) - .collect(); - - // Run the mechanistic mapper sandbox-side to generate proposals. - // The gateway is a thin persistence + validation layer — it never - // generates proposals itself. - let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); - - info!( - sandbox_name = %sandbox_name, - summaries = proto_summaries.len(), - proposals = proposals.len(), - "Flushed denial analysis to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - proto_summaries, - proposals, - Vec::new(), - "mechanistic", - ) - .await?; - - Ok(()) -} - -/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. -async fn flush_activity_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summary: activity_aggregator::FlushableActivitySummary, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summary = NetworkActivitySummary { - network_activity_count: summary.network_activity_count, - denied_action_count: summary.denied_action_count, - denials_by_group: summary - .denials_by_group - .into_iter() - .map(|(group, count)| DenialGroupCount { - deny_group: group, - denied_count: count, - }) - .collect(), - }; - - info!( - sandbox_name = %sandbox_name, - network_activity_count = proto_summary.network_activity_count, - denied_action_count = proto_summary.denied_action_count, - "Flushed activity summary to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - Vec::new(), - Vec::new(), - vec![proto_summary], - "activity", - ) - .await?; - - Ok(()) -} - -// ============================================================================ -// Baseline filesystem path enrichment -// ============================================================================ - -/// Minimum read-only paths required for a proxy-mode sandbox child process to -/// function: dynamic linker, shared libraries, DNS resolution, CA certs, -/// Python venv, openshell logs, process info, and random bytes. -/// -/// `/proc` and `/dev/urandom` are included here for the same reasons they -/// appear in `restrictive_default_policy()`: virtually every process needs -/// them. Before the Landlock per-path fix (#677) these were effectively free -/// because a missing path silently disabled the entire ruleset; now they must -/// be explicit. -const PROXY_BASELINE_READ_ONLY: &[&str] = &[ - "/usr", - "/lib", - "/etc", - "/app", - "/var/log", - "/proc", - "/dev/urandom", -]; - -/// Minimum read-write paths required for a proxy-mode sandbox child process: -/// user working directory and temporary files. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; - -/// GPU read-only paths. -/// -/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced -/// socket at init time. If the directory exists but Landlock denies traversal -/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` -/// even though the daemon is optional. Only read/traversal access is needed. -/// -/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, -/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` -/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may -/// not be covered by the parent-directory Landlock rule when the mount crosses -/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal -/// is permitted regardless of Landlock's cross-mount behaviour. -const GPU_BASELINE_READ_ONLY: &[&str] = &[ - "/run/nvidia-persistenced", - "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory -]; - -/// GPU read-write paths (static). -/// -/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, -/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native -/// Linux. Landlock restricts `open(2)` on device files even when DAC allows -/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. -/// These devices do not exist on WSL2 and will be skipped by the existence -/// check in `enrich_proto_baseline_paths()`. -/// -/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver -/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects -/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux -/// and will be skipped there by the existence check. -/// -/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` -/// to set thread names. Without write access, `cuInit()` returns error 304. -/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to -/// inodes and child processes have different procfs inodes than the parent. -/// -/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by -/// `enumerate_gpu_device_nodes()` since the count varies. -const GPU_BASELINE_READ_WRITE: &[&str] = &[ - "/dev/nvidiactl", - "/dev/nvidia-uvm", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-modeset", - "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) - "/proc", -]; - -/// Returns true if GPU devices are present in the container. -/// -/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and -/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these -/// depending on the host kernel; the other will not exist. -fn has_gpu_devices() -> bool { - std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() -} - -/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). -fn enumerate_gpu_device_nodes() -> Vec { - let mut paths = Vec::new(); - if let Ok(entries) = std::fs::read_dir("/dev") { - for entry in entries.flatten() { - let name = entry.file_name(); - let name = name.to_string_lossy(); - if let Some(suffix) = name.strip_prefix("nvidia") { - if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { - continue; - } - paths.push(entry.path().to_string_lossy().into_owned()); - } - } - } - paths -} - -fn push_unique(paths: &mut Vec, path: String) { - if !paths.iter().any(|p| p == &path) { - paths.push(path); - } -} - -fn collect_baseline_enrichment_paths( - include_proxy: bool, - include_gpu: bool, - gpu_device_nodes: Vec, -) -> (Vec, Vec) { - let mut ro = Vec::new(); - let mut rw = Vec::new(); - - if include_proxy { - for &path in PROXY_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in PROXY_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - } - - if include_gpu { - for &path in GPU_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in GPU_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - for path in gpu_device_nodes { - push_unique(&mut rw, path); - } - } - - // A path promoted to read_write (e.g. /proc for GPU) should not also - // appear in read_only — Landlock handles the overlap correctly but the - // duplicate is confusing when inspecting the effective policy. - ro.retain(|p| !rw.contains(p)); - - (ro, rw) -} - -fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { - let include_gpu = has_gpu_devices(); - let gpu_device_nodes = if include_gpu { - enumerate_gpu_device_nodes() - } else { - Vec::new() - }; - collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) -} - -/// Collect all active baseline paths for tests and diagnostics. -/// Returns `(read_only, read_write)` as owned `String` vecs. -#[cfg(test)] -fn baseline_enrichment_paths() -> (Vec, Vec) { - active_baseline_enrichment_paths(true) -} - -fn enrich_proto_baseline_paths_with( - proto: &mut openshell_core::proto::SandboxPolicy, - ro: &[String], - rw: &[String], - path_exists: F, -) -> bool -where - F: Fn(&str) -> bool, -{ - if ro.is_empty() && rw.is_empty() { - return false; - } - - let fs = proto - .filesystem - .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { - include_workdir: true, - ..Default::default() - }); - - let mut modified = false; - for path in ro { - if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { - if !path_exists(path) { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - fs.read_only.push(path.clone()); - modified = true; - } - } - for path in rw { - if fs.read_write.iter().any(|p| p == path) { - continue; - } - if !path_exists(path) { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - if fs.read_only.iter().any(|p| p == path) { - if path == "/proc" { - info!( - path, - "Promoting /proc from read-only to read-write for GPU runtime compatibility" - ); - fs.read_only.retain(|p| p != path); - fs.read_write.push(path.clone()); - modified = true; - } - continue; - } - fs.read_write.push(path.clone()); - modified = true; - } - - modified -} - -/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths -/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if -/// missing; user-specified paths are never removed. -/// -/// Returns `true` if the policy was modified (caller may want to sync back). -fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); - - // Baseline paths are system-injected, not user-specified. Skip paths - // that do not exist in this container image to avoid noisy warnings from - // Landlock and, more critically, to prevent a single missing baseline - // path from abandoning the entire Landlock ruleset under best-effort - // mode (see issue #664). - let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { - std::path::Path::new(path).exists() - }); - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } - - modified -} - -fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - openshell_policy::strip_provider_rule_names(proto) -} - -fn proto_sync_payload_for_enriched_policy( - proto: &openshell_core::proto::SandboxPolicy, - enriched: bool, -) -> Option { - if !enriched { - return None; - } - - let mut sync_policy = proto.clone(); - strip_proto_provider_policy_entries(&mut sync_policy); - Some(sync_policy) -} - -/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem -/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the -/// local-file code path where no proto is available. -fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { - let (ro, rw) = - active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); - if ro.is_empty() && rw.is_empty() { - return; - } - - let mut modified = false; - for path in &ro { - let p = std::path::PathBuf::from(path); - if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { - if !p.exists() { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_only.push(p); - modified = true; - } - } - for path in &rw { - let p = std::path::PathBuf::from(path); - if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { - continue; - } - if !p.exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_write.push(p); - modified = true; - } - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod baseline_tests { - use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; - - #[test] - fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { - // When GPU devices are present, /proc is promoted to read_write - // (CUDA needs to write /proc//task//comm). It should - // NOT also appear in read_only. - if !has_gpu_devices() { - // Can't test GPU dedup without GPU devices; skip silently. - return; - } - let (ro, rw) = baseline_enrichment_paths(); - assert!( - rw.contains(&"/proc".to_string()), - "/proc should be in read_write when GPU is present" - ); - assert!( - !ro.contains(&"/proc".to_string()), - "/proc should NOT be in read_only when it is already in read_write" - ); - } - - #[test] - fn proc_in_read_only_without_gpu() { - if has_gpu_devices() { - // On a GPU host we can't test the non-GPU path; skip silently. - return; - } - let (ro, _rw) = baseline_enrichment_paths(); - assert!( - ro.contains(&"/proc".to_string()), - "/proc should be in read_only when GPU is not present" - ); - } - - #[test] - fn baseline_read_write_always_includes_sandbox_and_tmp() { - let (_ro, rw) = baseline_enrichment_paths(); - assert!(rw.contains(&"/sandbox".to_string())); - assert!(rw.contains(&"/tmp".to_string())); - } - - #[test] - fn enumerate_gpu_device_nodes_skips_bare_nvidia() { - // "nvidia" (without a trailing digit) is a valid /dev entry on some - // systems but is not a per-GPU device node. The enumerator must - // not match it. - let nodes = enumerate_gpu_device_nodes(); - assert!( - !nodes.contains(&"/dev/nvidia".to_string()), - "bare /dev/nvidia should not be enumerated: {nodes:?}" - ); - } - - #[test] - fn no_duplicate_paths_in_baseline() { - let (ro, rw) = baseline_enrichment_paths(); - // No path should appear in both lists. - for path in &ro { - assert!( - !rw.contains(path), - "path {path} appears in both read_only and read_write" - ); - } - } - - #[test] - fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { - read_only: vec!["/tmp".to_string()], - read_write: vec![], - include_workdir: false, - }); - policy.network_policies.insert( - "test".into(), - openshell_core::proto::NetworkPolicyRule { - name: "test-rule".into(), - endpoints: vec![openshell_core::proto::NetworkEndpoint { - host: "example.com".into(), - port: 443, - ..Default::default() - }], - ..Default::default() - }, - ); - - enrich_proto_baseline_paths(&mut policy); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - filesystem.read_only.contains(&"/tmp".to_string()), - "explicit read_only baseline path should be preserved" - ); - assert!( - !filesystem.read_write.contains(&"/tmp".to_string()), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - assert!(strip_proto_provider_policy_entries(&mut policy)); - assert!( - !policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(policy.network_policies.contains_key("sandbox_only")); - assert!(!strip_proto_provider_policy_entries(&mut policy)); - } - - #[test] - fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - - assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "provider-derived rules alone must not trigger sync or mutate runtime policy" - ); - } - - #[test] - fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - runtime_policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) - .expect("enrichment should create a sync payload"); - - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "runtime policy must retain provider-derived rules for OPA input" - ); - assert!( - !sync_policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(sync_policy.network_policies.contains_key("sandbox_only")); - } - - #[test] - fn proto_gpu_enrichment_promotes_proc_without_network_policy() { - let mut policy = openshell_policy::restrictive_default_policy(); - assert!( - policy.network_policies.is_empty(), - "regression setup must exercise the no-network default path" - ); - let (ro, rw) = - collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); - - let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { - matches!(path, "/proc" | "/dev/nvidia0") - }); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - enriched, - "GPU enrichment should not require network policies" - ); - assert!( - filesystem.read_write.contains(&"/dev/nvidia0".to_string()), - "GPU enrichment should add enumerated device nodes without network policies" - ); - assert!( - !filesystem.read_only.contains(&"/proc".to_string()), - "GPU enrichment should remove /proc from read_only" - ); - assert!( - filesystem.read_write.contains(&"/proc".to_string()), - "GPU enrichment should promote /proc to read_write" - ); - } - - #[test] - fn gpu_baseline_read_write_contains_dxg() { - // /dev/dxg must be present so WSL2 sandboxes get the Landlock - // read-write rule for the CDI-injected DXG device. The existence - // check in enrich_proto_baseline_paths() skips it on native Linux. - assert!( - GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), - "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" - ); - } - - #[test] - fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only: vec![std::path::PathBuf::from("/tmp")], - read_write: vec![], - include_workdir: false, - }, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - }; - - enrich_sandbox_baseline_paths(&mut policy); - - assert!( - policy - .filesystem - .read_only - .contains(&std::path::PathBuf::from("/tmp")), - "explicit read_only baseline path should be preserved" - ); - assert!( - !policy - .filesystem - .read_write - .contains(&std::path::PathBuf::from("/tmp")), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn gpu_baseline_read_only_contains_usr_lib_wsl() { - // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library - // bind-mounts are accessible under Landlock. Skipped on native Linux. - assert!( - GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), - "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" - ); - } - - #[test] - fn has_gpu_devices_reflects_dxg_or_nvidiactl() { - // Verify the OR logic: result must match the manual disjunction of - // the two path checks. Passes in all environments. - let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); - let dxg = std::path::Path::new("/dev/dxg").exists(); - assert_eq!( - has_gpu_devices(), - nvidiactl || dxg, - "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" - ); - } -} - -/// Returns `true` if the error is transient and worth retrying. -/// -/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If -/// found, only the gRPC codes that represent transient failures are retryable. -/// If no `tonic::Status` is present (e.g. a raw connection error), assume the -/// failure is transient. -fn is_retryable_error(err: &miette::Report) -> bool { - let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); - while let Some(e) = source { - if let Some(status) = e.downcast_ref::() { - return matches!( - status.code(), - tonic::Code::Unavailable - | tonic::Code::DeadlineExceeded - | tonic::Code::ResourceExhausted - | tonic::Code::Aborted - | tonic::Code::Internal - | tonic::Code::Unknown - ); - } - source = e.source(); - } - true -} - -/// Retry a gRPC operation with exponential backoff (capped at 4 s). -/// -/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, -/// `PERMISSION_DENIED`) are returned immediately without retrying. -async fn grpc_retry(op_name: &str, f: F) -> Result -where - F: Fn() -> Fut, - Fut: Future>, -{ - let mut last_err = None; - for attempt in 1..=5u32 { - match f().await { - Ok(val) => return Ok(val), - Err(e) => { - if !is_retryable_error(&e) { - return Err(e); - } - if attempt < 5 { - warn!( - attempt, - max_attempts = 5, - error = %e, - "{op_name} failed, retrying" - ); - let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); - tokio::time::sleep(backoff).await; - } - last_err = Some(e); - } - } - } - Err(miette::miette!( - "{op_name} failed after 5 attempts: {}", - last_err.expect("loop executed at least once") - )) -} - -/// Load sandbox policy from local files or gRPC. -/// -/// Priority: -/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files -/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC -/// 3. If the server returns no policy, discover from disk or use restrictive default -/// 4. Otherwise, return an error -/// -/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto -/// policy. The proto is retained so the OPA engine can be rebuilt with symlink -/// resolution after the container entrypoint starts. -async fn load_policy( - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, -) -> Result<( - SandboxPolicy, - Option>, - Option, - MiddlewareRegistryStatus, - LoadedPolicyOrigin, - bool, -)> { - // File mode: load OPA engine from rego rules + YAML data (dev override) - if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "loading") - .unmapped("policy_rules", serde_json::json!(policy_file)) - .unmapped("policy_data", serde_json::json!(data_file)) - .message(format!( - "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" - )) - .build()); - let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { - openshell_supervisor_middleware_builtins::validate_config(implementation, config) - .map_err(|error| error.to_string()) - }; - let engine = OpaEngine::from_files_with_middleware_config( - std::path::Path::new(policy_file), - std::path::Path::new(data_file), - Some(&validate_middleware_config), - )?; - let middleware_registry = - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - engine.replace_middleware_registry(middleware_registry)?; - let config = engine.query_sandbox_config()?; - let mut policy = SandboxPolicy { - version: 1, - filesystem: config.filesystem, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: config.landlock, - process: config.process, - }; - enrich_sandbox_baseline_paths(&mut policy); - // File mode has no operator-registered middleware to connect. - return Ok(( - policy, - Some(Arc::new(engine)), - None, - MiddlewareRegistryStatus::Synchronized, - LoadedPolicyOrigin::LocalOverride, - false, - )); - } - - // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - info!( - sandbox_id = %id, - endpoint = %endpoint, - "Fetching sandbox policy via gRPC" - ); - let mut snapshot = grpc_retry("Policy fetch", || { - openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) - }) - .await?; - - let mut proto_policy = if let Some(p) = snapshot.policy.clone() { - p - } else { - // No policy configured on the server. Discover from disk or - // fall back to the restrictive default, then sync to the - // gateway so it becomes the authoritative baseline. - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "discovery") - .message("Server returned no policy; attempting local discovery") - .build() - ); - let mut discovered = discover_policy_from_disk_or_default(); - // Enrich before syncing so the gateway baseline includes - // baseline paths from the start. - enrich_proto_baseline_paths(&mut discovered); - strip_proto_provider_policy_entries(&mut discovered); - let sandbox = sandbox.as_deref().ok_or_else(|| { - miette::miette!( - "Cannot sync discovered policy: sandbox not available.\n\ - Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." - ) - })?; - - // Sync and re-fetch over a single connection to avoid extra - // TLS handshakes. - let ws = snapshot.workspace.clone(); - snapshot = grpc_retry("Policy discovery sync", || { - openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox, - &discovered, - &ws, - ) - }) - .await?; - snapshot.policy.clone().ok_or_else(|| { - miette::miette!("Server still returned no policy after sync — this is a bug") - })? - }; - - // True only while `snapshot` describes the exact policy that will be - // constructed below. If enrichment cannot be synced and re-fetched, - // the policy remains enforceable but cannot be acknowledged by - // inferred structural equality. - let mut policy_bound_to_snapshot = true; - - // Ensure baseline filesystem paths are present for proxy-mode - // sandboxes. If the policy was enriched, sync the updated version - // back to the gateway so users can see the effective policy. - let enriched = enrich_proto_baseline_paths(&mut proto_policy); - let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); - if let Some(sync_policy) = sync_policy { - if let Some(sandbox_name) = sandbox.as_deref() { - match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox_name, - &sync_policy, - &snapshot.workspace, - ) - .await - { - Ok(canonical) => { - if let Some(policy) = canonical.policy.clone() { - proto_policy = policy; - snapshot = canonical; - } else { - policy_bound_to_snapshot = false; - warn!( - "Gateway returned no policy after enrichment sync; initial revision will be reconciled" - ); - } - } - Err(e) => { - policy_bound_to_snapshot = false; - warn!( - error = %e, - "Failed to sync enriched policy back to gateway; initial revision will be reconciled" - ); - } - } - } else { - policy_bound_to_snapshot = false; - } - } - - let loaded_policy_revision = - policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); - - // Build OPA engine from baked-in rules + typed proto data. - // In cluster mode, proxy networking is always enabled so OPA is - // always required for allow/deny decisions. - // The initial load uses pid=0 (no symlink resolution) because the - // container hasn't started yet. After the entrypoint spawns, the - // engine is rebuilt with the real PID for symlink resolution. - info!("Creating OPA engine from proto policy data"); - let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => engine, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - - // Install the in-process catalog before any external connection can - // fail. A newly started sandbox must always be able to resolve built-in - // bindings, even while operator-run services are unavailable. - install_builtin_middleware_registry(&engine).await?; - - // Connect operator-registered middleware services. A connect/describe - // failure keeps the built-in registry active so each request's - // `on_error` policy governs matched traffic. The policy poll loop - // retries the install without waiting for a config change. - let middleware_services = snapshot.supervisor_middleware_services.clone(); - let middleware_registry_status = if middleware_services.is_empty() { - MiddlewareRegistryStatus::Synchronized - } else if let Err(error) = grpc_retry("Middleware connect", || { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - middleware_services.clone(), - ) - }) - .await - .and_then(|registry| engine.replace_middleware_registry(registry)) - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(middleware_services.len()) - ) - .message(format!( - "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" - )) - .build() - ); - MiddlewareRegistryStatus::NeedsReconciliation - } else { - MiddlewareRegistryStatus::Synchronized - }; - let opa_engine = Some(Arc::new(engine)); - - let policy = match SandboxPolicy::try_from(proto_policy.clone()) { - Ok(policy) => policy, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - return Ok(( - policy, - opa_engine, - Some(proto_policy), - middleware_registry_status, - LoadedPolicyOrigin::Gateway { - revision: loaded_policy_revision, - }, - agent_proposals_enabled_from_settings(&snapshot.settings), - )); - } - - // No policy source available - Err(miette::miette!( - "Sandbox policy required. Provide one of:\n\ - - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ - - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" - )) -} - -/// Try to discover a sandbox policy from the well-known disk path, falling -/// back to the legacy path, then to the hardcoded restrictive default. -fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { - let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); - if primary.exists() { - return discover_policy_from_path(primary); - } - let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); - if legacy.exists() { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "legacy_path", - serde_json::json!(legacy.display().to_string()) - ) - .unmapped("new_path", serde_json::json!(primary.display().to_string())) - .message(format!( - "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", - legacy.display(), - primary.display() - )) - .build() - ); - return discover_policy_from_path(legacy); - } - discover_policy_from_path(primary) -} - -/// Try to read a sandbox policy YAML from `path`, falling back to the -/// hardcoded restrictive default if the file is missing or invalid. -fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { - use openshell_policy::{ - parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, - }; - - let Ok(yaml) = std::fs::read_to_string(path) else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "default") - .message(format!( - "No policy file on disk, using restrictive default [path:{}]", - path.display() - )) - .build() - ); - return restrictive_default_policy(); - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Loaded sandbox policy from container disk [path:{}]", - path.display() - )) - .build() - ); - match parse_sandbox_policy(&yaml) { - Ok(policy) => { - // Validate the disk-loaded policy for safety. - if let Err(violations) = validate_sandbox_policy(&policy) { - let messages: Vec = violations.iter().map(ToString::to_string).collect(); - ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Medium) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .finding_info( - FindingInfo::new( - "unsafe-disk-policy", - "Unsafe Disk Policy Content", - ) - .with_desc(&format!( - "Disk policy at {} contains unsafe content: {}", - path.display(), - messages.join("; "), - )), - ) - .message(format!( - "Disk policy contains unsafe content, using restrictive default [path:{}]", - path.display() - )) - .build()); - return restrictive_default_policy(); - } - policy - } - Err(e) => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "fallback") - .message(format!( - "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", - path.display() - )) - .build()); - restrictive_default_policy() - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum MiddlewareRegistryStatus { - Synchronized, - NeedsReconciliation, -} - -/// True when the installed middleware registry no longer matches the desired -/// service set and must be rebuilt (reconnecting every delivered service). -/// -/// A policy-only change never requires a rebuild: middleware configs were -/// validated at gateway admission and the installed registry's manifests -/// already cover the unchanged service set, so requiring the services to be -/// reachable would only let a middleware outage block the policy update. -fn middleware_registry_needs_rebuild( - registry_status: MiddlewareRegistryStatus, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> bool { - registry_status == MiddlewareRegistryStatus::NeedsReconciliation - || current_services != desired_services -} - -fn gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy: bool, - current_policy_hash: &str, - desired_policy_hash: &str, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - registry_status: MiddlewareRegistryStatus, -) -> bool { - reloads_gateway_policy - && (current_policy_hash != desired_policy_hash - || middleware_registry_needs_rebuild( - registry_status, - current_services, - desired_services, - )) -} - -/// Identity returned with the exact policy snapshot used to construct OPA. -#[derive(Clone, Debug, PartialEq, Eq)] -struct LoadedPolicyRevision { - version: u32, - policy_hash: String, - config_revision: u64, - policy_source: openshell_core::proto::PolicySource, -} - -/// Identifies where the policy currently loaded into OPA came from. -/// -/// A missing gateway revision means the policy was loaded from the gateway but -/// could not be bound to an authoritative snapshot (for example, enrichment -/// sync failed). That state must reconcile on the first successful poll. A -/// local-file override is different: gateway policy revisions are observed for -/// settings/provider refreshes but must never replace the explicit local OPA -/// policy. -#[derive(Clone, Debug, PartialEq, Eq)] -enum LoadedPolicyOrigin { - LocalOverride, - Gateway { - revision: Option, - }, -} - -impl LoadedPolicyOrigin { - fn allows_gateway_policy_reload(&self) -> bool { - matches!(self, Self::Gateway { .. }) - } -} - -impl LoadedPolicyRevision { - fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { - Self { - version: snapshot.version, - policy_hash: snapshot.policy_hash.clone(), - config_revision: snapshot.config_revision, - policy_source: snapshot.policy_source, - } - } -} - -/// A sandbox-scoped policy revision that was constructed successfully at -/// startup and must be acknowledged to the gateway exactly once. -#[derive(Clone, Debug, PartialEq, Eq)] -struct InitialPolicyAck { - version: u32, - policy_hash: String, - config_revision: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PolicyStatusUpdate { - version: u32, - loaded: bool, - error: String, - initial_policy_hash: Option, -} - -impl PolicyStatusUpdate { - fn initial_loaded(ack: &InitialPolicyAck) -> Self { - Self { - version: ack.version, - loaded: true, - error: String::new(), - initial_policy_hash: Some(ack.policy_hash.clone()), - } - } - - fn loaded(version: u32) -> Self { - Self { - version, - loaded: true, - error: String::new(), - initial_policy_hash: None, - } - } - - fn failed(version: u32, error: String) -> Self { - Self { - version, - loaded: false, - error, - initial_policy_hash: None, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum InitialPollDisposition { - Acknowledge(InitialPolicyAck), - Reconcile, - TrackOnly, -} - -/// Determine whether the initially loaded policy corresponds to an -/// authoritative sandbox-scoped revision that must be acknowledged. -/// -/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose -/// captured gateway identity matches the current version and hash. Global -/// policies, local-file development policies, version zero, and changed -/// identities yield `None`, so those paths never emit a sandbox-revision -/// acknowledgement. -fn initial_policy_ack_candidate( - loaded: Option<&LoadedPolicyRevision>, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - let loaded = loaded?; - if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox - || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox - { - return None; - } - if loaded.version == 0 || canonical.version == 0 { - return None; - } - if loaded.version != canonical.version - || loaded.policy_hash != canonical.policy_hash - || canonical.config_revision < loaded.config_revision - { - return None; - } - Some(InitialPolicyAck { - version: loaded.version, - policy_hash: loaded.policy_hash.clone(), - config_revision: canonical.config_revision, - }) -} - -fn initial_poll_disposition( - origin: &LoadedPolicyOrigin, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> InitialPollDisposition { - match origin { - LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision } => { - initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( - InitialPollDisposition::Reconcile, - InitialPollDisposition::Acknowledge, - ) - } - } -} - -/// Deliver policy status updates independently from policy reconciliation. -/// -/// The channel is FIFO, so a delayed older status can never arrive after a -/// newer status and move the gateway's active version backward. Delivery uses -/// the existing bounded retry, but failures never delay policy enforcement. -async fn run_policy_status_reporter( - client: openshell_core::grpc_client::CachedOpenShellClient, - sandbox_id: String, - mut updates: tokio::sync::mpsc::UnboundedReceiver, -) { - 'updates: while let Some(update) = updates.recv().await { - let operation = if update.initial_policy_hash.is_some() { - "Initial policy acknowledgement" - } else { - "Policy status report" - }; - let mut attempt = 1_u32; - loop { - let sandbox_id = sandbox_id.clone(); - let error = update.error.clone(); - let client = client.clone(); - match client - .report_policy_status(&sandbox_id, update.version, update.loaded, &error) - .await - { - Ok(()) => break, - Err(error) if is_retryable_error(&error) => { - let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); - warn!( - %error, - attempt, - version = update.version, - loaded = update.loaded, - retry_in_secs = backoff.as_secs(), - "{operation} failed transiently; retaining ordered update" - ); - tokio::time::sleep(backoff).await; - attempt = attempt.saturating_add(1); - } - Err(error) => { - warn!( - %error, - version = update.version, - loaded = update.loaded, - "Discarding terminal policy status update" - ); - continue 'updates; - } - } - } - - if let Some(policy_hash) = update.initial_policy_hash { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("version", serde_json::json!(update.version)) - .unmapped("policy_hash", serde_json::json!(policy_hash)) - .message(format!( - "Acknowledged initial policy revision as loaded [version:{}]", - update.version - )) - .build() - ); - } - } -} - -fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { - let version = update.version; - if let Err(error) = sender.send(update) { - warn!( - %error, - version, - "Policy status reporter unavailable during shutdown" - ); - } -} - -/// Best-effort `FAILED` acknowledgement when initial policy construction or -/// conversion fails. -/// -/// Uses the revision identity captured with the policy that failed to build, -/// and preserves the original construction error as the reported message. A -/// delivery failure here is swallowed so it can never mask that error. -async fn report_initial_policy_failure( - endpoint: &str, - sandbox_id: &str, - revision: Option<&LoadedPolicyRevision>, - error: &miette::Report, -) { - let Some(revision) = revision.filter(|revision| { - revision.version > 0 - && revision.policy_source == openshell_core::proto::PolicySource::Sandbox - }) else { - return; - }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { - Ok(client) => client, - Err(e) => { - warn!(error = %e, "Failed to connect to report initial policy failure"); - return; - } - }; - let message = error.to_string(); - if let Err(e) = grpc_retry("Initial policy failure report", || { - let client = client.clone(); - let message = message.clone(); - async move { - client - .report_policy_status(sandbox_id, revision.version, false, &message) - .await - } - }) - .await - { - warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); - } -} - -/// Background loop that polls the server for policy updates. -/// -/// When a new version is detected, attempts to reload the OPA engine via -/// `reload_from_proto_with_pid()`. Reports load success/failure back to the -/// server. On failure, the previous engine is untouched (LKG behavior). -/// -/// When the entrypoint PID is available, policy reloads include symlink -/// resolution for binary paths via the container filesystem. -struct PolicyPollLoopContext { - endpoint: String, - sandbox_id: String, - opa_engine: Arc, - /// Source of the policy currently loaded into OPA. This distinguishes an - /// explicit local-file override from an unbound gateway revision so the - /// former is never replaced by policy polling. - loaded_policy_origin: LoadedPolicyOrigin, - entrypoint_pid: Arc, - interval_secs: u64, - ocsf_enabled: Arc, - provider_credentials: ProviderCredentialState, - policy_local_ctx: Option>, - agent_proposals: AgentProposals, - middleware_registry_status: MiddlewareRegistryStatus, - sidecar_control_publisher: Option, - workspace_tx: tokio::sync::watch::Sender, -} - -async fn connect_middleware_registry( - services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> Result { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - ) - .await -} - -async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - opa_engine.replace_middleware_registry(registry) -} - -async fn reconcile_middleware_registry( - opa_engine: &OpaEngine, - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - current_services: &mut Vec, - status: &mut MiddlewareRegistryStatus, -) { - if *status == MiddlewareRegistryStatus::Synchronized - && desired_services == current_services.as_slice() - { - return; - } - - match connect_middleware_registry(desired_services) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - current_services.clear(); - current_services.extend_from_slice(desired_services); - *status = MiddlewareRegistryStatus::Synchronized; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(current_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - current_services.len() - )) - .build() - ); - } - Err(error) => { - // Emit only on the transition into the failed state to avoid - // repeating the same finding on every poll during an outage. - if *status == MiddlewareRegistryStatus::Synchronized { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - } - *status = MiddlewareRegistryStatus::NeedsReconciliation; - } - } -} - -async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::PolicySource; - use std::sync::atomic::Ordering; - - let client = CachedOpenShellClient::connect(&ctx.endpoint).await?; - let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); - tokio::spawn(run_policy_status_reporter( - client.clone(), - ctx.sandbox_id.clone(), - status_receiver, - )); - - let mut current_config_revision: u64 = 0; - let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; - let mut current_policy_hash = String::new(); - let mut current_middleware_services = Vec::new(); - let mut middleware_registry_status = ctx.middleware_registry_status; - let mut current_settings: std::collections::HashMap< - String, - openshell_core::proto::EffectiveSetting, - > = std::collections::HashMap::new(); - let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); - let mut last_failed_runtime_revision: Option<(u64, String)> = None; - - // A first poll that does not match the policy already loaded into OPA must - // pass through the normal reconciliation path immediately. It must never - // seed the applied-state trackers before OPA actually loads it. - let mut pending_result = None; - - // Initialize revision from the first poll and acknowledge the initial - // policy revision the supervisor actually loaded. A mismatched result is - // reconciled below instead of being recorded as already applied. - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { - InitialPollDisposition::Acknowledge(candidate) => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(candidate.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = candidate.config_revision; - current_policy_hash.clone_from(&candidate.policy_hash); - current_middleware_services = result.supervisor_middleware_services; - current_settings = result.settings; - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::initial_loaded(&candidate), - ); - debug!( - config_revision = current_config_revision, - "Settings poll: initial policy matches loaded revision" - ); - } - InitialPollDisposition::Reconcile => pending_result = Some(result), - InitialPollDisposition::TrackOnly => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash.clone(); - current_middleware_services = result.supervisor_middleware_services; - current_settings = result.settings; - debug!( - config_revision = current_config_revision, - "Settings poll: tracking gateway config while preserving local policy override" - ); - } - } - } - Err(e) => { - warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); - } - } - - let interval = Duration::from_secs(ctx.interval_secs); - loop { - let result = if let Some(result) = pending_result.take() { - result - } else { - tokio::time::sleep(interval).await; - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - result - } - Err(e) => { - debug!(error = %e, "Settings poll: server unreachable, will retry"); - continue; - } - } - }; - - let config_changed = result.config_revision != current_config_revision; - let provider_env_changed = result.provider_env_revision != current_provider_env_revision; - let policy_changed = result.policy_hash != current_policy_hash; - let middleware_registry_changed = middleware_registry_needs_rebuild( - middleware_registry_status, - ¤t_middleware_services, - &result.supervisor_middleware_services, - ); - let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); - - // A local policy override is not coupled to the gateway policy - // snapshot, so its service registry can still be reconciled alone. - // Gateway policy snapshots, however, must install policy and registry - // as one generation below. - if !reloads_gateway_policy { - reconcile_middleware_registry( - &ctx.opa_engine, - &result.supervisor_middleware_services, - &mut current_middleware_services, - &mut middleware_registry_status, - ) - .await; - } - - if !config_changed && !provider_env_changed && !policy_runtime_changed { - continue; - } - - if config_changed || provider_env_changed { - // Log which settings changed. - log_setting_changes(¤t_settings, &result.settings); - - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "detected") - .unmapped("old_config_revision", serde_json::json!(current_config_revision)) - .unmapped("new_config_revision", serde_json::json!(result.config_revision)) - .unmapped("policy_changed", serde_json::json!(policy_changed)) - .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) - .message(format!( - "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", - result.config_revision - )) - .build()); - } - - if provider_env_changed { - match openshell_core::grpc_client::fetch_provider_environment( - &ctx.endpoint, - &ctx.sandbox_id, - ) - .await - { - Ok(env_result) => { - ctx.provider_credentials.install_environment( - env_result.provider_env_revision, - env_result.environment, - env_result.credential_expires_at_ms, - env_result.dynamic_credentials, - ); - let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); - let env_count = child_env.len(); - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_provider_env( - env_result.provider_env_revision, - child_env.clone(), - ); - } - current_provider_env_revision = env_result.provider_env_revision; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "provider_env_revision", - serde_json::json!(env_result.provider_env_revision) - ) - .message(format!( - "Provider environment refreshed [revision:{} env_count:{env_count}]", - env_result.provider_env_revision - )) - .build() - ); - } - Err(e) => { - warn!( - error = %e, - provider_env_revision = result.provider_env_revision, - "Settings poll: failed to refresh provider environment" - ); - } - } - } - - if policy_runtime_changed { - let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = match result.policy.as_ref() { - Some(policy) if middleware_registry_changed => { - match connect_middleware_registry(&result.supervisor_middleware_services).await - { - Ok(registry) => ctx - .opa_engine - .reload_policy_and_middleware_from_proto_with_pid( - policy, pid, registry, - ), - Err(error) => Err(error), - } - } - // Policy-only change: the installed registry already matches - // the delivered service set, so swap the engine alone. This - // must not require middleware reachability. - Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), - None => Err(miette::miette!( - "runtime reload requires a policy payload but none was returned" - )), - }; - - match runtime_result { - Ok(()) => { - let policy = result - .policy - .as_ref() - .expect("successful runtime reload requires a policy payload"); - if policy_changed { - if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { - policy_local_ctx.set_current_policy(policy.clone()).await; - } - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_policy( - policy.clone(), - result.policy_hash.clone(), - result.config_revision, - ); - } - if result.global_policy_version > 0 { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .unmapped("global_version", serde_json::json!(result.global_policy_version)) - .message(format!( - "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", - result.policy_hash, - result.global_policy_version - )) - .build()); - } else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - } - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - } - } - - if middleware_registry_changed { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(result.supervisor_middleware_services.len()) - ) - .message(format!( - "Supervisor policy runtime reloaded atomically [service_count:{}]", - result.supervisor_middleware_services.len() - )) - .build()); - } - - current_policy_hash.clone_from(&result.policy_hash); - current_middleware_services.clone_from(&result.supervisor_middleware_services); - middleware_registry_status = MiddlewareRegistryStatus::Synchronized; - last_failed_runtime_revision = None; - } - Err(e) => { - let failed_revision = (result.config_revision, result.policy_hash.clone()); - if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(e.to_string())) - .message(format!( - "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", - result.version - )) - .build()); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, e.to_string()), - ); - } - } - last_failed_runtime_revision = Some(failed_revision); - // Nothing was installed, so the registry status still - // describes the live registry. The retry is driven by the - // persisting hash/service-set mismatch (or an existing - // NeedsReconciliation), not by degrading the status here. - } - } - } - - // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - - // Apply the agent-proposals feature toggle. On a false→true transition - // we lazily install the skill so a sandbox that started with the flag - // off picks up the surface without a recreate. We never uninstall on - // a true→false transition: stale skill content on disk is harmless - // because route_request and agent_next_steps both gate on the live - // shared flag, so the agent that reads the skill will see 404s and an - // empty `next_steps` array regardless. - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - - current_config_revision = result.config_revision; - if !reloads_gateway_policy { - current_policy_hash = result.policy_hash; - } - current_settings = result.settings; - } -} - -fn apply_ocsf_json_setting( - enabled: &AtomicBool, - settings: &std::collections::HashMap, -) { - use std::sync::atomic::Ordering; - - let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); - let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); - if new_ocsf != prev_ocsf { - info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); - } -} - -/// Extract a bool value from an effective setting, if present. -fn extract_bool_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) -} - -fn agent_proposals_enabled_from_settings( - settings: &std::collections::HashMap, -) -> bool { - extract_bool_setting( - settings, - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, - ) - .unwrap_or(false) -} - -fn apply_agent_proposals_enabled( - agent_proposals: &AgentProposals, - enabled: bool, - source: &'static str, - config_revision: Option, - sidecar_control_publisher: Option<&sidecar_control::Publisher>, - install_static_skills: impl FnOnce() -> Result, -) { - let previously_enabled = agent_proposals.swap_enabled(enabled); - if enabled == previously_enabled { - return; - } - - info!( - agent_policy_proposals_enabled = enabled, - source, config_revision, "agent-driven policy proposals toggled" - ); - - if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { - publisher.publish_agent_proposals(enabled, config_revision); - } - - if enabled && !previously_enabled { - match install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill on toggle-on" - ), - Err(error) => warn!( - error = %error, - "Failed to install sandbox agent skill on toggle-on" - ), - } - } -} - -/// Log individual setting changes between two snapshots. -fn log_setting_changes( - old: &std::collections::HashMap, - new: &std::collections::HashMap, -) { - for (key, new_es) in new { - let new_val = format_setting_value(new_es); - match old.get(key) { - Some(old_es) => { - let old_val = format_setting_value(old_es); - if old_val != new_val { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "updated") - .unmapped("key", serde_json::json!(key)) - .unmapped("old", serde_json::json!(old_val.clone())) - .unmapped("new", serde_json::json!(new_val.clone())) - .message(format!( - "Setting changed [key:{key} old:{old_val} new:{new_val}]" - )) - .build() - ); - } - } - None => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enabled") - .unmapped("key", serde_json::json!(key)) - .unmapped("value", serde_json::json!(new_val.clone())) - .message(format!("Setting added [key:{key} value:{new_val}]")) - .build() - ); - } - } - } - for key in old.keys() { - if !new.contains_key(key) { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Disabled, "disabled") - .unmapped("key", serde_json::json!(key)) - .message(format!("Setting removed [key:{key}]")) - .build() - ); - } - } -} - -/// Format an `EffectiveSetting` value for log display. -fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { - use openshell_core::proto::setting_value; - match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { - None => "".to_string(), - Some(setting_value::Value::StringValue(v)) => v.clone(), - Some(setting_value::Value::BoolValue(v)) => v.to_string(), - Some(setting_value::Value::IntValue(v)) => v.to_string(), - Some(setting_value::Value::BytesValue(_)) => "".to_string(), - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod tests { - use super::*; - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - fn proxy_policy(http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { - openshell_core::proto::EffectiveSetting { - value: Some(openshell_core::proto::SettingValue { - value: Some(openshell_core::proto::setting_value::Value::BoolValue( - value, - )), - }), - scope: openshell_core::proto::SettingScope::Global.into(), - } - } - - #[test] - fn sidecar_process_policy_sets_loopback_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, true).unwrap(); - - let http_addr = process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .expect("sidecar process policy should set proxy address"); - assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); - assert!( - policy - .network - .proxy - .as_ref() - .expect("original policy should keep proxy config") - .http_addr - .is_none(), - "process policy normalization must not mutate the network policy" - ); - } - - #[test] - fn non_sidecar_process_policy_preserves_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, false).unwrap(); - - assert!( - process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .is_none() - ); - } - - #[tokio::test] - async fn sidecar_control_provider_env_update_installs_newer_revision() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - 1, - std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), - ); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials.clone(), - AgentProposals::default(), - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "new".to_string(), - )]), - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 2 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 2); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 1, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "stale".to_string(), - )]), - }) - .unwrap(); - tokio::time::sleep(Duration::from_millis(20)).await; - assert_eq!( - provider_credentials - .snapshot() - .child_env - .get("TOKEN") - .map(String::as_str), - Some("new") - ); - handle.abort(); - } - - #[tokio::test] - async fn sidecar_control_agent_proposals_update_flips_shared_state() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = - ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); - let agent_proposals = AgentProposals::new(true); - let handle = - spawn_sidecar_control_update_watcher(rx, provider_credentials, agent_proposals.clone()); - - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 5, - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if !agent_proposals.enabled() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - handle.abort(); - } - - #[test] - fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { - let agent_proposals = AgentProposals::default(); - let installs = AtomicUsize::new(0); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(!agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - } - - #[test] - fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { - let enabled = AtomicBool::new(false); - let mut settings = std::collections::HashMap::new(); - settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(enabled.load(Ordering::Relaxed)); - } - - #[test] - fn apply_ocsf_json_setting_disables_when_setting_is_unset() { - let enabled = AtomicBool::new(true); - let settings = std::collections::HashMap::new(); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(!enabled.load(Ordering::Relaxed)); - } - - #[test] - fn agent_proposals_setting_enables_from_initial_settings_snapshot() { - let mut settings = std::collections::HashMap::new(); - settings.insert( - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), - effective_bool(true), - ); - - assert!(agent_proposals_enabled_from_settings(&settings)); - } - - #[test] - fn agent_proposals_setting_defaults_false_when_unset() { - let settings = std::collections::HashMap::new(); - - assert!(!agent_proposals_enabled_from_settings(&settings)); - } - - // ---- Policy disk discovery tests ---- - - #[test] - fn discover_policy_from_nonexistent_path_returns_restrictive_default() { - let path = std::path::Path::new("/nonexistent/policy.yaml"); - let policy = discover_policy_from_path(path); - // Restrictive default has no network policies. - assert!(policy.network_policies.is_empty()); - // But does have filesystem and process policies. - assert!(policy.filesystem.is_some()); - assert!(policy.process.is_some()); - } - - #[test] - fn discover_policy_from_valid_yaml_file() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr - read_write: - - /tmp -network_policies: - test: - name: test - endpoints: - - { host: example.com, port: 443 } - binaries: - - { path: /usr/bin/curl } -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - assert_eq!(policy.network_policies.len(), 1); - assert!(policy.network_policies.contains_key("test")); - let fs = policy.filesystem.unwrap(); - assert!(!fs.include_workdir); - } - - #[test] - fn discover_policy_from_invalid_yaml_returns_restrictive_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default. - assert!(policy.network_policies.is_empty()); - assert!(policy.filesystem.is_some()); - } - - #[test] - fn discover_policy_from_unsafe_yaml_falls_back_to_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -process: - run_as_user: root - run_as_group: root -filesystem_policy: - include_workdir: true - read_only: - - /usr - read_write: - - /tmp -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default because of root user. - let proc = policy.process.unwrap(); - assert_eq!(proc.run_as_user, "sandbox"); - assert_eq!(proc.run_as_group, "sandbox"); - } - - #[test] - fn discover_policy_restrictive_default_blocks_network() { - // In cluster mode we keep proxy mode enabled so `inference.local` - // can always be routed through proxy/OPA controls. - let proto = openshell_policy::restrictive_default_policy(); - let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); - assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); - } - - // ---- Initial policy acknowledgement tests ---- - - fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::restrictive_default_policy() - } - - fn settings_poll_result( - policy: Option, - version: u32, - source: openshell_core::proto::PolicySource, - ) -> openshell_core::grpc_client::SettingsPollResult { - openshell_core::grpc_client::SettingsPollResult { - policy, - version, - policy_hash: format!("hash-v{version}"), - config_revision: u64::from(version) * 100, - policy_source: source, - settings: std::collections::HashMap::new(), - global_policy_version: 0, - provider_env_revision: 0, - supervisor_middleware_services: Vec::new(), - workspace: String::new(), - } - } - - #[tokio::test] - async fn failed_external_startup_registry_build_preserves_installed_builtins() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let builtins_generation = engine.current_generation(); - assert_eq!(builtins_generation, 1); - - let invalid_external = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_body_bytes: 1024, - ..Default::default() - }; - connect_middleware_registry(&[invalid_external]) - .await - .expect_err("unavailable external service must not replace built-ins"); - - assert_eq!(engine.current_generation(), builtins_generation); - } - - #[test] - fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { - let services = Vec::new(); - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - } - - #[test] - fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &no_services, - &no_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &no_services, - &desired_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - false, - "local-policy", - "hash-v2", - &no_services, - &desired_services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - } - - #[test] - fn policy_only_change_does_not_rebuild_middleware_registry() { - let services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - // The runtime must reconcile, but the registry (and therefore - // middleware reachability) is not part of that reconciliation. - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &services, - &services, - )); - } - - #[test] - fn registry_rebuild_requires_service_set_change_or_degraded_registry() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &no_services, - &desired_services, - )); - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::NeedsReconciliation, - &desired_services, - &desired_services, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &desired_services, - &desired_services, - )); - } - - #[test] - fn initial_ack_candidate_matches_sandbox_revision() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) - .expect("sandbox-sourced matching revision should be acknowledged"); - - assert_eq!(ack.version, 2); - assert_eq!(ack.policy_hash, "hash-v2"); - assert_eq!(ack.config_revision, 200); - } - - #[test] - fn initial_ack_candidate_ignores_global_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Global, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_version_zero() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 0, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_local_file_mode() { - // Local-file mode retains no proto policy, so there is nothing to - // acknowledge to the gateway. - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(None, &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_rejects_mismatched_identity() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let mut newer = proto_policy_fixture(); - newer.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule::default(), - ); - let canonical = - settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); - let canonical = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "hash-provider-change".to_string(), - config_revision: loaded.config_revision + 1, - ..canonical - }; - - assert_eq!( - initial_poll_disposition( - &LoadedPolicyOrigin::Gateway { - revision: Some(loaded), - }, - &canonical, - ), - InitialPollDisposition::Reconcile - ); - } - - #[test] - fn initial_poll_tracks_local_override_without_reconciliation() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert_eq!( - initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), - InitialPollDisposition::TrackOnly - ); - assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); - } - - #[test] - fn initial_poll_reconciles_unbound_gateway_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let origin = LoadedPolicyOrigin::Gateway { revision: None }; - - assert_eq!( - initial_poll_disposition(&origin, &canonical), - InitialPollDisposition::Reconcile - ); - assert!(origin.allows_gateway_policy_reload()); - } - - #[test] - fn policy_status_outbox_preserves_all_revision_order() { - let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - for version in 1..=128 { - enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); - } - - for version in 1..=128 { - assert_eq!( - receiver.try_recv().unwrap(), - PolicyStatusUpdate::loaded(version) - ); - } - } - - #[test] - fn settings_snapshot_carries_workspace_for_policy_sync() { - let mut snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - snapshot.workspace = "beta".to_string(); - - let revision = LoadedPolicyRevision::from_snapshot(&snapshot); - assert_eq!(revision.version, 1); - assert_eq!( - snapshot.workspace, "beta", - "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" - ); - } -} diff --git a/crates/openshell-sandbox/src/lib_win.rs b/crates/openshell-sandbox/src/lib_win.rs new file mode 100644 index 0000000000..394226e1a7 --- /dev/null +++ b/crates/openshell-sandbox/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub fn windows_unsupported() -> &'static str { + "openshell-sandbox supervisor is not available on Windows" +} diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 948e0e7108..62ae37b5a1 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -1,11 +1,749 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-sandbox supervisor is not available on Windows"); - std::process::exit(1); +//! `OpenShell` Sandbox - process sandbox and monitor. + +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; + +use openshell_sandbox::run_sandbox; + +/// Subcommand name used to self-copy the supervisor binary into a shared volume. +/// +/// Init containers invoke the binary directly instead of relying on `sh`/`cp` +/// to copy the binary out. Invoking the binary itself with this argument +/// performs the copy in pure Rust. +const COPY_SELF_SUBCOMMAND: &str = "copy-self"; + +/// Subcommand for one-shot debug RPCs from inside a sandbox container. +/// +/// Reads the same token sources as the supervisor (env, file, K8s SA +/// bootstrap) and issues a single gRPC call against the gateway. Useful +/// for end-to-end verification: e.g. `docker exec` into a sandbox, then +/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` +/// to confirm the cross-sandbox IDOR guard fires. +const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; + +/// Default `--mode` value: run both supervisor leaves in a single binary. +const DEFAULT_MODE: &str = "network,process"; +const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +#[cfg(target_os = "linux")] +const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; + +/// Which supervisor leaves are enabled in this process. +/// +/// Parsed from a comma-separated `--mode` value, e.g. `network`, +/// `process`, or `network,process`. `network-init` is a one-shot setup mode +/// used by the Kubernetes sidecar topology and cannot be combined with other +/// mode components. At least one must be set. +#[derive(Clone, Copy, Debug)] +struct Mode { + network: bool, + process: bool, + network_init: bool, +} + +impl std::str::FromStr for Mode { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut mode = Self { + network: false, + process: false, + network_init: false, + }; + for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { + match part { + "network" => mode.network = true, + "process" => mode.process = true, + "network-init" => mode.network_init = true, + other => { + return Err(format!( + "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" + )); + } + } + } + if mode.network_init && (mode.network || mode.process) { + return Err("--mode=network-init cannot be combined with other components".into()); + } + if !mode.network && !mode.process && !mode.network_init { + return Err( + "--mode must enable at least one of: network, process, network-init".into(), + ); + } + Ok(mode) + } +} + +/// `OpenShell` Sandbox - process isolation and monitoring. +// CLI flags are naturally boolean switches; grouping them into structs would +// only obscure the clap definition. +#[allow(clippy::struct_excessive_bools)] +#[derive(Parser, Debug)] +#[command(name = "openshell-sandbox")] +#[command(version = openshell_core::VERSION)] +#[command(about = "Process sandbox and monitor", long_about = None)] +struct Args { + /// Command to execute in the sandbox. + /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. + /// Defaults to `/bin/bash` if neither is provided. + #[arg(trailing_var_arg = true)] + command: Vec, + + /// Working directory for the sandboxed process. + #[arg(long, short)] + workdir: Option, + + /// Timeout in seconds (0 = no timeout). + #[arg(long, short, default_value = "0")] + timeout: u64, + + /// Run in interactive mode (inherit process group for terminal control). + #[arg(long, short = 'i')] + interactive: bool, + + /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. + /// Requires --openshell-endpoint to be set. + #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] + sandbox_id: Option, + + /// Sandbox (used for policy sync when the sandbox discovers policy + /// from disk or falls back to the restrictive default). + #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] + sandbox: Option, + + /// `OpenShell` server gRPC endpoint for fetching policy. + /// Required when using --sandbox-id. + #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] + openshell_endpoint: Option, + + /// Path to Rego policy file for OPA-based network access control. + /// Requires --policy-data to also be set. + #[arg(long, env = "OPENSHELL_POLICY_RULES")] + policy_rules: Option, + + /// Path to YAML data file containing network policies and sandbox config. + /// Requires --policy-rules to also be set. + #[arg(long, env = "OPENSHELL_POLICY_DATA")] + policy_data: Option, + + /// Log level (trace, debug, info, warn, error). + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, + + /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning + /// with `@` selects an abstract socket in the network namespace. + /// The supervisor bridges `RelayStream` traffic from the gateway onto + /// this socket; nothing else should connect to it. + #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] + ssh_socket_path: Option, + + /// Path to YAML inference routes for standalone routing. + /// When set, inference routes are loaded from this file instead of + /// fetching a bundle from the gateway. + #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] + inference_routes: Option, + + /// Enable health check endpoint. + #[arg(long)] + health_check: bool, + + /// Port for health check endpoint. + #[arg(long, default_value = "8080")] + health_port: u16, + + /// Which supervisor components to run. Comma-separated list of + /// "network" and/or "process". Defaults to both (single-binary + /// topology). Use --mode=network for a network-only sidecar, or + /// --mode=process for a process-only supervisor when network + /// enforcement runs in another pod. Use --mode=network-init only in + /// the Kubernetes init container that prepares sidecar nftables. + #[arg(long, default_value = DEFAULT_MODE)] + mode: Mode, + + /// UID that the long-running Kubernetes network sidecar will run as. + /// `--mode=network-init` installs nftables rules that exempt this UID. + #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] + proxy_uid: u32, + + /// GID assigned to shared sidecar state directories. Defaults to + /// `--proxy-uid` when omitted. + #[arg(long, env = "OPENSHELL_PROXY_GID")] + proxy_gid: Option, + + /// Shared state directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] + sidecar_state_dir: String, + + /// Shared TLS work directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] + sidecar_tls_dir: String, + + // Corporate upstream proxy. Operator-owned egress boundary: accepted + // only as command-line arguments (no `env =`), because the driver + // controls the supervisor's argv while a sandbox image could bake + // matching `ENV` values. + /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. + #[arg(long)] + upstream_proxy: Option, + + /// Comma-separated `NO_PROXY` list for the corporate proxy. + #[arg(long)] + upstream_no_proxy: Option, + + /// Path to the root-only file holding corporate proxy credentials (`user:pass`). + #[arg(long)] + upstream_proxy_auth_file: Option, + + /// Acknowledge that proxy credentials travel as cleartext Basic auth over + /// the plain-TCP connection to the `http://` proxy. + #[arg(long)] + upstream_proxy_auth_allow_insecure: bool, + + /// Send the destination hostname in CONNECT instead of a validated IP + /// (for proxies whose ACLs filter on hostnames). + #[arg(long)] + upstream_proxy_connect_by_hostname: bool, +} + +/// Copy the running executable to `dest`, creating parent directories as +/// needed and ensuring the result is executable (mode `0755`). +/// +/// If `dest` already exists as a directory, the binary is placed inside it +/// using the source executable's file name. This mirrors `cp` semantics so +/// callers can pass either a full target path or a directory. +fn copy_self(dest: &str) -> Result<()> { + let exe = std::env::current_exe().into_diagnostic()?; + + let dest_path = Path::new(dest); + let final_path = if dest_path.is_dir() { + let file_name = exe + .file_name() + .ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?; + dest_path.join(file_name) + } else { + dest_path.to_path_buf() + }; + + if let Some(parent) = final_path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + + std::fs::copy(&exe, &final_path).into_diagnostic()?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&final_path) + .into_diagnostic()? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&final_path, perms).into_diagnostic()?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {uid}:{gid}", + path.display() + ) + })?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + let uid = Uid::current(); + let gid = Gid::current(); + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + chown(path, Some(uid), Some(gid)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {}:{}", + path.display(), + uid.as_raw(), + gid.as_raw() + ) + })?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + Ok(()) } -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +#[cfg(target_os = "linux")] +fn copy_sidecar_client_tls_if_present( + source_dir: &Path, + sidecar_tls_dir: &Path, + uid: u32, + gid: u32, +) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + if !source_dir.exists() { + return Ok(()); + } + + let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); + prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; + for file_name in CLIENT_TLS_FILES { + let source = source_dir.join(file_name); + if !source.exists() { + return Err(miette::miette!( + "client TLS source file is missing: {}", + source.display() + )); + } + let dest = dest_dir.join(file_name); + if dest.exists() { + std::fs::remove_file(&dest) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to remove stale client TLS file {}", dest.display()) + })?; + } + std::fs::copy(&source, &dest) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to copy client TLS file {} to {}", + source.display(), + dest.display() + ) + })?; + let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); + perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); + std::fs::set_permissions(&dest, perms) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to chmod copied client TLS file {}", dest.display()) + })?; + chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown copied client TLS file {} to {uid}:{gid}", + dest.display() + ) + })?; + } + + prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn run_network_init( + proxy_user_id: u32, + proxy_primary_group_id: u32, + sidecar_state_dir: &str, + sidecar_tls_dir: &str, +) -> Result<()> { + validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; + + let sidecar_state_dir = Path::new(sidecar_state_dir); + let sidecar_tls_dir = Path::new(sidecar_tls_dir); + prepare_sidecar_directory( + sidecar_state_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_STATE_DIR_MODE, + )?; + // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the + // TLS work directory owned by the init user until the client cert copy is + // complete, then hand it to the long-running proxy UID. + prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; + copy_sidecar_client_tls_if_present( + Path::new(CLIENT_TLS_DIR), + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + )?; + prepare_sidecar_directory( + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_TLS_DIR_MODE, + )?; + openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) +} + +#[cfg(target_os = "linux")] +fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { + if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-uid must be 0 or at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn run_network_init( + _proxy_uid: u32, + _proxy_gid: u32, + _sidecar_state_dir: &str, + _sidecar_tls_dir: &str, +) -> Result<()> { + Err(miette::miette!( + "--mode=network-init is only supported on Linux" + )) +} + +fn main() -> Result<()> { + // Handle `copy-self ` before clap so it works without any of the + // sandbox flags. Kubernetes init containers invoke this path to seed an + // emptyDir volume that the agent container then executes from. + let raw_args: Vec = std::env::args().collect(); + if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { + let dest = raw_args.get(2).ok_or_else(|| { + miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") + })?; + return copy_self(dest); + } + + // Handle `debug-rpc [args]` before clap. Uses a small + // dedicated runtime so we don't pay the supervisor's full startup cost. + if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .into_diagnostic()?; + return runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; + std::process::exit(exit); + }); + } + + let args = Args::parse(); + + if args.mode.network_init { + let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + return run_network_init( + args.proxy_uid, + proxy_gid, + &args.sidecar_state_dir, + &args.sidecar_tls_dir, + ); + } + + // Try to open a rolling log file; fall back to stderr-only logging if it fails + // (e.g., /var/log is not writable in custom workload images). + // Rotates daily, keeps the 3 most recent files to bound disk usage. + let file_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + (writer, guard) + }); + + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .into_diagnostic()?; + + let exit_code = runtime.block_on(async move { + // Install rustls crypto provider before any TLS connections (including log push). + let _ = rustls::crypto::ring::default_provider().install_default(); + + // Set up optional log push layer (gRPC mode only). + let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = + (&args.sandbox_id, &args.openshell_endpoint) + { + let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( + endpoint.clone(), + sandbox_id.clone(), + ); + let layer = + openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); + Some((layer, handle)) + } else { + None + }; + let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); + let _log_push_handle = log_push_state.map(|(_, handle)| handle); + + // Shared flag: the sandbox poll loop toggles this when the + // `ocsf_json_enabled` setting changes. The JSONL layer checks it + // on each event and short-circuits when false. + let ocsf_enabled = Arc::new(AtomicBool::new(false)); + + // Keep guards alive for the entire process. When a guard is dropped the + // non-blocking writer flushes remaining logs. + let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { + let file_filter = EnvFilter::new("info"); + + // OCSF JSONL file: rolling appender matching the main log file + // (daily rotation, 3 files max). Created eagerly but gated by the + // enabled flag — no JSONL is written until ocsf_json_enabled is set. + let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell-ocsf") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + (layer, guard) + }); + let (jsonl_layer, jsonl_guard) = match jsonl_logging { + Some((layer, guard)) => (Some(layer), Some(guard)), + None => (None, None), + }; + + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with( + OcsfShorthandLayer::new(file_writer) + .with_non_ocsf(true) + .with_filter(file_filter), + ) + .with(jsonl_layer.with_filter(LevelFilter::INFO)) + .with(push_layer.clone()) + .init(); + (Some(file_guard), jsonl_guard) + } else { + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with(push_layer) + .init(); + // Log the warning after the subscriber is initialized + warn!("Could not open /var/log for log rotation; using stderr-only logging"); + (None, None) + }; + + // Get command - either from CLI args, environment variable, or default to /bin/bash + let command = if !args.command.is_empty() { + args.command + } else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) { + // Simple shell-like splitting on whitespace + c.split_whitespace().map(String::from).collect() + } else { + vec!["/bin/bash".to_string()] + }; + + info!(command = ?command, "Starting sandbox"); + // Note: "Starting sandbox" stays as plain info!() since the OCSF context + // is not yet initialized at this point (run_sandbox hasn't been called). + // The shorthand layer will render it in fallback format. + + let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { + https_proxy: args.upstream_proxy, + no_proxy: args.upstream_no_proxy, + proxy_auth_file: args.upstream_proxy_auth_file, + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, + }; + + run_sandbox( + command, + args.workdir, + args.timeout, + args.interactive, + args.sandbox_id, + args.sandbox, + args.openshell_endpoint, + args.policy_rules, + args.policy_data, + args.ssh_socket_path, + args.health_check, + args.health_port, + args.inference_routes, + ocsf_enabled, + args.mode.network, + args.mode.process, + upstream_proxy_args, + ) + .await + })?; + + std::process::exit(exit_code); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + /// Drives `copy_self`'s file-copy logic against an arbitrary source path + /// so tests don't depend on `current_exe()`. + fn copy_executable(src: &Path, dest: &Path) -> Result<()> { + let final_path = if dest.is_dir() { + dest.join(src.file_name().unwrap()) + } else { + dest.to_path_buf() + }; + if let Some(parent) = final_path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + std::fs::copy(src, &final_path).into_diagnostic()?; + let mut perms = std::fs::metadata(&final_path) + .into_diagnostic()? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&final_path, perms).into_diagnostic()?; + Ok(()) + } + + #[test] + fn copy_self_writes_executable_at_target_path() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("source-bin"); + std::fs::write(&src, b"#!/bin/false\n").unwrap(); + + let dest = tmp.path().join("subdir/openshell-sandbox"); + copy_executable(&src, &dest).unwrap(); + + assert!(dest.exists(), "destination file should exist"); + let mode = std::fs::metadata(&dest).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o755, "destination must be 0755"); + let copied = std::fs::read(&dest).unwrap(); + assert_eq!(copied, b"#!/bin/false\n"); + } + + #[test] + fn copy_self_into_existing_directory_uses_source_filename() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("openshell-sandbox"); + std::fs::write(&src, b"binary").unwrap(); + + let dest_dir = tmp.path().join("bin"); + std::fs::create_dir_all(&dest_dir).unwrap(); + + copy_executable(&src, &dest_dir).unwrap(); + + let final_path = dest_dir.join("openshell-sandbox"); + assert!(final_path.exists(), "binary should land inside dest dir"); + } + + #[test] + fn mode_parses_network_init_standalone() { + let mode = "network-init".parse::().unwrap(); + assert!(mode.network_init); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_combined_network_init() { + let err = "network-init,network".parse::().unwrap_err(); + assert!(err.contains("cannot be combined")); + } + + #[test] + fn mode_rejects_empty_value() { + let err = "".parse::().unwrap_err(); + assert!(err.contains("at least one")); + } + + #[cfg(target_os = "linux")] + #[test] + fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { + assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); + assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); + assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); + assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { + validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_still_rejects_low_non_root_proxy_ids() { + let uid_err = + validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); + assert!(uid_err.to_string().contains("--proxy-uid")); + + let gid_err = validate_network_init_ids(0, 999).unwrap_err(); + assert!(gid_err.to_string().contains("--proxy-gid")); + } +} diff --git a/crates/openshell-sandbox/src/main_entry.rs b/crates/openshell-sandbox/src/main_entry.rs new file mode 100644 index 0000000000..432402121a --- /dev/null +++ b/crates/openshell-sandbox/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-sandbox/src/main_unix.rs b/crates/openshell-sandbox/src/main_unix.rs deleted file mode 100644 index 62ae37b5a1..0000000000 --- a/crates/openshell-sandbox/src/main_unix.rs +++ /dev/null @@ -1,749 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! `OpenShell` Sandbox - process sandbox and monitor. - -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; - -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; -use tracing::{info, warn}; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::LevelFilter; -use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; - -use openshell_sandbox::run_sandbox; - -/// Subcommand name used to self-copy the supervisor binary into a shared volume. -/// -/// Init containers invoke the binary directly instead of relying on `sh`/`cp` -/// to copy the binary out. Invoking the binary itself with this argument -/// performs the copy in pure Rust. -const COPY_SELF_SUBCOMMAND: &str = "copy-self"; - -/// Subcommand for one-shot debug RPCs from inside a sandbox container. -/// -/// Reads the same token sources as the supervisor (env, file, K8s SA -/// bootstrap) and issues a single gRPC call against the gateway. Useful -/// for end-to-end verification: e.g. `docker exec` into a sandbox, then -/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` -/// to confirm the cross-sandbox IDOR guard fires. -const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; - -/// Default `--mode` value: run both supervisor leaves in a single binary. -const DEFAULT_MODE: &str = "network,process"; -const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; -const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; -#[cfg(target_os = "linux")] -const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; -#[cfg(target_os = "linux")] -const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; -#[cfg(target_os = "linux")] -const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; -#[cfg(target_os = "linux")] -const SIDECAR_TLS_DIR_MODE: u32 = 0o755; -#[cfg(target_os = "linux")] -const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; - -/// Which supervisor leaves are enabled in this process. -/// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. -#[derive(Clone, Copy, Debug)] -struct Mode { - network: bool, - process: bool, - network_init: bool, -} - -impl std::str::FromStr for Mode { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut mode = Self { - network: false, - process: false, - network_init: false, - }; - for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { - match part { - "network" => mode.network = true, - "process" => mode.process = true, - "network-init" => mode.network_init = true, - other => { - return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" - )); - } - } - } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); - } - if !mode.network && !mode.process && !mode.network_init { - return Err( - "--mode must enable at least one of: network, process, network-init".into(), - ); - } - Ok(mode) - } -} - -/// `OpenShell` Sandbox - process isolation and monitoring. -// CLI flags are naturally boolean switches; grouping them into structs would -// only obscure the clap definition. -#[allow(clippy::struct_excessive_bools)] -#[derive(Parser, Debug)] -#[command(name = "openshell-sandbox")] -#[command(version = openshell_core::VERSION)] -#[command(about = "Process sandbox and monitor", long_about = None)] -struct Args { - /// Command to execute in the sandbox. - /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. - /// Defaults to `/bin/bash` if neither is provided. - #[arg(trailing_var_arg = true)] - command: Vec, - - /// Working directory for the sandboxed process. - #[arg(long, short)] - workdir: Option, - - /// Timeout in seconds (0 = no timeout). - #[arg(long, short, default_value = "0")] - timeout: u64, - - /// Run in interactive mode (inherit process group for terminal control). - #[arg(long, short = 'i')] - interactive: bool, - - /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. - /// Requires --openshell-endpoint to be set. - #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] - sandbox_id: Option, - - /// Sandbox (used for policy sync when the sandbox discovers policy - /// from disk or falls back to the restrictive default). - #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] - sandbox: Option, - - /// `OpenShell` server gRPC endpoint for fetching policy. - /// Required when using --sandbox-id. - #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] - openshell_endpoint: Option, - - /// Path to Rego policy file for OPA-based network access control. - /// Requires --policy-data to also be set. - #[arg(long, env = "OPENSHELL_POLICY_RULES")] - policy_rules: Option, - - /// Path to YAML data file containing network policies and sandbox config. - /// Requires --policy-rules to also be set. - #[arg(long, env = "OPENSHELL_POLICY_DATA")] - policy_data: Option, - - /// Log level (trace, debug, info, warn, error). - #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] - log_level: String, - - /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning - /// with `@` selects an abstract socket in the network namespace. - /// The supervisor bridges `RelayStream` traffic from the gateway onto - /// this socket; nothing else should connect to it. - #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] - ssh_socket_path: Option, - - /// Path to YAML inference routes for standalone routing. - /// When set, inference routes are loaded from this file instead of - /// fetching a bundle from the gateway. - #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] - inference_routes: Option, - - /// Enable health check endpoint. - #[arg(long)] - health_check: bool, - - /// Port for health check endpoint. - #[arg(long, default_value = "8080")] - health_port: u16, - - /// Which supervisor components to run. Comma-separated list of - /// "network" and/or "process". Defaults to both (single-binary - /// topology). Use --mode=network for a network-only sidecar, or - /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. Use --mode=network-init only in - /// the Kubernetes init container that prepares sidecar nftables. - #[arg(long, default_value = DEFAULT_MODE)] - mode: Mode, - - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] - proxy_uid: u32, - - /// GID assigned to shared sidecar state directories. Defaults to - /// `--proxy-uid` when omitted. - #[arg(long, env = "OPENSHELL_PROXY_GID")] - proxy_gid: Option, - - /// Shared state directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] - sidecar_state_dir: String, - - /// Shared TLS work directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] - sidecar_tls_dir: String, - - // Corporate upstream proxy. Operator-owned egress boundary: accepted - // only as command-line arguments (no `env =`), because the driver - // controls the supervisor's argv while a sandbox image could bake - // matching `ENV` values. - /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. - #[arg(long)] - upstream_proxy: Option, - - /// Comma-separated `NO_PROXY` list for the corporate proxy. - #[arg(long)] - upstream_no_proxy: Option, - - /// Path to the root-only file holding corporate proxy credentials (`user:pass`). - #[arg(long)] - upstream_proxy_auth_file: Option, - - /// Acknowledge that proxy credentials travel as cleartext Basic auth over - /// the plain-TCP connection to the `http://` proxy. - #[arg(long)] - upstream_proxy_auth_allow_insecure: bool, - - /// Send the destination hostname in CONNECT instead of a validated IP - /// (for proxies whose ACLs filter on hostnames). - #[arg(long)] - upstream_proxy_connect_by_hostname: bool, -} - -/// Copy the running executable to `dest`, creating parent directories as -/// needed and ensuring the result is executable (mode `0755`). -/// -/// If `dest` already exists as a directory, the binary is placed inside it -/// using the source executable's file name. This mirrors `cp` semantics so -/// callers can pass either a full target path or a directory. -fn copy_self(dest: &str) -> Result<()> { - let exe = std::env::current_exe().into_diagnostic()?; - - let dest_path = Path::new(dest); - let final_path = if dest_path.is_dir() { - let file_name = exe - .file_name() - .ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?; - dest_path.join(file_name) - } else { - dest_path.to_path_buf() - }; - - if let Some(parent) = final_path.parent() - && !parent.as_os_str().is_empty() - { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - - std::fs::copy(&exe, &final_path).into_diagnostic()?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut perms = std::fs::metadata(&final_path) - .into_diagnostic()? - .permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&final_path, perms).into_diagnostic()?; - } - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {uid}:{gid}", - path.display() - ) - })?; - Ok(()) -} - -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - let uid = Uid::current(); - let gid = Gid::current(); - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - chown(path, Some(uid), Some(gid)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {}:{}", - path.display(), - uid.as_raw(), - gid.as_raw() - ) - })?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - Ok(()) -} - -#[cfg(target_os = "linux")] -fn copy_sidecar_client_tls_if_present( - source_dir: &Path, - sidecar_tls_dir: &Path, - uid: u32, - gid: u32, -) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - if !source_dir.exists() { - return Ok(()); - } - - let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); - prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; - for file_name in CLIENT_TLS_FILES { - let source = source_dir.join(file_name); - if !source.exists() { - return Err(miette::miette!( - "client TLS source file is missing: {}", - source.display() - )); - } - let dest = dest_dir.join(file_name); - if dest.exists() { - std::fs::remove_file(&dest) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to remove stale client TLS file {}", dest.display()) - })?; - } - std::fs::copy(&source, &dest) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to copy client TLS file {} to {}", - source.display(), - dest.display() - ) - })?; - let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); - perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); - std::fs::set_permissions(&dest, perms) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to chmod copied client TLS file {}", dest.display()) - })?; - chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown copied client TLS file {} to {uid}:{gid}", - dest.display() - ) - })?; - } - - prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn run_network_init( - proxy_user_id: u32, - proxy_primary_group_id: u32, - sidecar_state_dir: &str, - sidecar_tls_dir: &str, -) -> Result<()> { - validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; - - let sidecar_state_dir = Path::new(sidecar_state_dir); - let sidecar_tls_dir = Path::new(sidecar_tls_dir); - prepare_sidecar_directory( - sidecar_state_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_STATE_DIR_MODE, - )?; - // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the - // TLS work directory owned by the init user until the client cert copy is - // complete, then hand it to the long-running proxy UID. - prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; - copy_sidecar_client_tls_if_present( - Path::new(CLIENT_TLS_DIR), - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - )?; - prepare_sidecar_directory( - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_TLS_DIR_MODE, - )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) -} - -#[cfg(target_os = "linux")] -fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { - if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-uid must be 0 or at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-gid must be at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - Ok(()) -} - -#[cfg(not(target_os = "linux"))] -fn run_network_init( - _proxy_uid: u32, - _proxy_gid: u32, - _sidecar_state_dir: &str, - _sidecar_tls_dir: &str, -) -> Result<()> { - Err(miette::miette!( - "--mode=network-init is only supported on Linux" - )) -} - -fn main() -> Result<()> { - // Handle `copy-self ` before clap so it works without any of the - // sandbox flags. Kubernetes init containers invoke this path to seed an - // emptyDir volume that the agent container then executes from. - let raw_args: Vec = std::env::args().collect(); - if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { - let dest = raw_args.get(2).ok_or_else(|| { - miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") - })?; - return copy_self(dest); - } - - // Handle `debug-rpc [args]` before clap. Uses a small - // dedicated runtime so we don't pay the supervisor's full startup cost. - if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .into_diagnostic()?; - return runtime.block_on(async move { - let _ = rustls::crypto::ring::default_provider().install_default(); - let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; - std::process::exit(exit); - }); - } - - let args = Args::parse(); - - if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); - return run_network_init( - args.proxy_uid, - proxy_gid, - &args.sidecar_state_dir, - &args.sidecar_tls_dir, - ); - } - - // Try to open a rolling log file; fall back to stderr-only logging if it fails - // (e.g., /var/log is not writable in custom workload images). - // Rotates daily, keeps the 3 most recent files to bound disk usage. - let file_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - (writer, guard) - }); - - let console_filter = - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .into_diagnostic()?; - - let exit_code = runtime.block_on(async move { - // Install rustls crypto provider before any TLS connections (including log push). - let _ = rustls::crypto::ring::default_provider().install_default(); - - // Set up optional log push layer (gRPC mode only). - let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = - (&args.sandbox_id, &args.openshell_endpoint) - { - let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), - sandbox_id.clone(), - ); - let layer = - openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); - Some((layer, handle)) - } else { - None - }; - let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); - let _log_push_handle = log_push_state.map(|(_, handle)| handle); - - // Shared flag: the sandbox poll loop toggles this when the - // `ocsf_json_enabled` setting changes. The JSONL layer checks it - // on each event and short-circuits when false. - let ocsf_enabled = Arc::new(AtomicBool::new(false)); - - // Keep guards alive for the entire process. When a guard is dropped the - // non-blocking writer flushes remaining logs. - let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { - let file_filter = EnvFilter::new("info"); - - // OCSF JSONL file: rolling appender matching the main log file - // (daily rotation, 3 files max). Created eagerly but gated by the - // enabled flag — no JSONL is written until ocsf_json_enabled is set. - let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell-ocsf") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); - (layer, guard) - }); - let (jsonl_layer, jsonl_guard) = match jsonl_logging { - Some((layer, guard)) => (Some(layer), Some(guard)), - None => (None, None), - }; - - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with( - OcsfShorthandLayer::new(file_writer) - .with_non_ocsf(true) - .with_filter(file_filter), - ) - .with(jsonl_layer.with_filter(LevelFilter::INFO)) - .with(push_layer.clone()) - .init(); - (Some(file_guard), jsonl_guard) - } else { - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with(push_layer) - .init(); - // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stderr-only logging"); - (None, None) - }; - - // Get command - either from CLI args, environment variable, or default to /bin/bash - let command = if !args.command.is_empty() { - args.command - } else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) { - // Simple shell-like splitting on whitespace - c.split_whitespace().map(String::from).collect() - } else { - vec!["/bin/bash".to_string()] - }; - - info!(command = ?command, "Starting sandbox"); - // Note: "Starting sandbox" stays as plain info!() since the OCSF context - // is not yet initialized at this point (run_sandbox hasn't been called). - // The shorthand layer will render it in fallback format. - - let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { - https_proxy: args.upstream_proxy, - no_proxy: args.upstream_no_proxy, - proxy_auth_file: args.upstream_proxy_auth_file, - proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, - }; - - run_sandbox( - command, - args.workdir, - args.timeout, - args.interactive, - args.sandbox_id, - args.sandbox, - args.openshell_endpoint, - args.policy_rules, - args.policy_data, - args.ssh_socket_path, - args.health_check, - args.health_port, - args.inference_routes, - ocsf_enabled, - args.mode.network, - args.mode.process, - upstream_proxy_args, - ) - .await - })?; - - std::process::exit(exit_code); -} - -#[cfg(test)] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - /// Drives `copy_self`'s file-copy logic against an arbitrary source path - /// so tests don't depend on `current_exe()`. - fn copy_executable(src: &Path, dest: &Path) -> Result<()> { - let final_path = if dest.is_dir() { - dest.join(src.file_name().unwrap()) - } else { - dest.to_path_buf() - }; - if let Some(parent) = final_path.parent() - && !parent.as_os_str().is_empty() - { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - std::fs::copy(src, &final_path).into_diagnostic()?; - let mut perms = std::fs::metadata(&final_path) - .into_diagnostic()? - .permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&final_path, perms).into_diagnostic()?; - Ok(()) - } - - #[test] - fn copy_self_writes_executable_at_target_path() { - let tmp = tempfile::tempdir().unwrap(); - let src = tmp.path().join("source-bin"); - std::fs::write(&src, b"#!/bin/false\n").unwrap(); - - let dest = tmp.path().join("subdir/openshell-sandbox"); - copy_executable(&src, &dest).unwrap(); - - assert!(dest.exists(), "destination file should exist"); - let mode = std::fs::metadata(&dest).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o755, "destination must be 0755"); - let copied = std::fs::read(&dest).unwrap(); - assert_eq!(copied, b"#!/bin/false\n"); - } - - #[test] - fn copy_self_into_existing_directory_uses_source_filename() { - let tmp = tempfile::tempdir().unwrap(); - let src = tmp.path().join("openshell-sandbox"); - std::fs::write(&src, b"binary").unwrap(); - - let dest_dir = tmp.path().join("bin"); - std::fs::create_dir_all(&dest_dir).unwrap(); - - copy_executable(&src, &dest_dir).unwrap(); - - let final_path = dest_dir.join("openshell-sandbox"); - assert!(final_path.exists(), "binary should land inside dest dir"); - } - - #[test] - fn mode_parses_network_init_standalone() { - let mode = "network-init".parse::().unwrap(); - assert!(mode.network_init); - assert!(!mode.network); - assert!(!mode.process); - } - - #[test] - fn mode_rejects_combined_network_init() { - let err = "network-init,network".parse::().unwrap_err(); - assert!(err.contains("cannot be combined")); - } - - #[test] - fn mode_rejects_empty_value() { - let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); - } - - #[cfg(target_os = "linux")] - #[test] - fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { - assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); - assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); - assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); - assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { - validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_still_rejects_low_non_root_proxy_ids() { - let uid_err = - validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); - assert!(uid_err.to_string().contains("--proxy-uid")); - - let gid_err = validate_network_init_ids(0, 999).unwrap_err(); - assert!(gid_err.to_string().contains("--proxy-gid")); - } -} diff --git a/crates/openshell-sandbox/src/main_win.rs b/crates/openshell-sandbox/src/main_win.rs new file mode 100644 index 0000000000..460b379c4c --- /dev/null +++ b/crates/openshell-sandbox/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-sandbox supervisor is not available on Windows"); + std::process::exit(1); +} From 9e5da6ca04ac8f4b16fa26f486b2778f21730c71 Mon Sep 17 00:00:00 2001 From: Akber Raza Date: Wed, 29 Jul 2026 16:15:28 -0500 Subject: [PATCH 30/30] fix(windows): restore proto include cfg gating Signed-off-by: Akber Raza --- crates/openshell-core/build.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index ba0a05c7be..86fce73ef2 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -35,7 +35,10 @@ fn main() -> Result<(), Box> { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); + #[cfg(target_os = "windows")] let proto_includes = proto_include_dirs(&proto_root)?; + #[cfg(not(target_os = "windows"))] + let proto_includes = vec![proto_root.clone()]; let mut proto_files = Vec::new(); collect_proto_files(&proto_root, &mut proto_files)?; @@ -61,12 +64,10 @@ fn main() -> Result<(), Box> { Ok(()) } +#[cfg(target_os = "windows")] fn proto_include_dirs(proto_root: &Path) -> Result, Box> { let mut includes = vec![proto_root.to_path_buf()]; - #[cfg(target_os = "windows")] - { - includes.push(protoc_bin_vendored::include_path()?); - } + includes.push(protoc_bin_vendored::include_path()?); Ok(includes) }