From ec4815f324bccc210bd7f764f0740b437eda7ca6 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 13:51:41 -0700 Subject: [PATCH 01/16] feat(sandbox): honor OCI image working directories Closes #2526 Signed-off-by: Matthew Grossman --- .../skills/generate-sandbox-policy/SKILL.md | 1 - .../generate-sandbox-policy/examples.md | 1 - .agents/skills/openshell-cli/SKILL.md | 8 +- .agents/skills/openshell-cli/cli-reference.md | 4 +- architecture/compute-runtimes.md | 13 +- crates/openshell-cli/src/ssh.rs | 222 +++++++++------ crates/openshell-core/src/driver_mounts.rs | 90 +++++- crates/openshell-driver-docker/README.md | 29 +- crates/openshell-driver-docker/src/lib.rs | 35 ++- crates/openshell-driver-docker/src/tests.rs | 70 ++++- .../openshell-driver-kubernetes/src/driver.rs | 28 +- crates/openshell-driver-podman/README.md | 32 ++- crates/openshell-driver-podman/src/client.rs | 13 +- .../openshell-driver-podman/src/container.rs | 116 +++++++- crates/openshell-driver-podman/src/driver.rs | 5 + crates/openshell-policy/src/lib.rs | 6 +- crates/openshell-sandbox/src/lib.rs | 28 +- .../src/process.rs | 265 +++++++++++++++++- .../openshell-supervisor-process/src/run.rs | 11 +- .../openshell-supervisor-process/src/ssh.rs | 96 +++++-- .../tutorials/first-network-policy.mdx | 2 +- docs/get-started/tutorials/github-sandbox.mdx | 1 - docs/reference/policy-schema.mdx | 3 +- docs/reference/sandbox-compute-drivers.mdx | 15 +- docs/sandboxes/manage-sandboxes.mdx | 13 +- docs/sandboxes/policies.mdx | 5 +- docs/security/best-practices.mdx | 2 +- e2e/rust/src/harness/sandbox.rs | 33 +++ e2e/rust/tests/custom_image.rs | 67 ++++- e2e/rust/tests/driver_config_volume.rs | 101 ++++++- 30 files changed, 1120 insertions(+), 195 deletions(-) diff --git a/.agents/skills/generate-sandbox-policy/SKILL.md b/.agents/skills/generate-sandbox-policy/SKILL.md index 3b261b5ded..5ce736d3f0 100644 --- a/.agents/skills/generate-sandbox-policy/SKILL.md +++ b/.agents/skills/generate-sandbox-policy/SKILL.md @@ -466,7 +466,6 @@ filesystem_policy: - /etc - /var/log read_write: - - /sandbox - /tmp - /dev/null diff --git a/.agents/skills/generate-sandbox-policy/examples.md b/.agents/skills/generate-sandbox-policy/examples.md index 179e46c00f..9f7832c1f7 100644 --- a/.agents/skills/generate-sandbox-policy/examples.md +++ b/.agents/skills/generate-sandbox-policy/examples.md @@ -823,7 +823,6 @@ filesystem_policy: - /etc - /var/log read_write: - - /sandbox - /tmp - /dev/null diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 06b06e4470..1f635c5b7a 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -220,8 +220,8 @@ openshell sandbox ssh-config my-sandbox >> ~/.ssh/config ### Upload and download files ```bash -# Upload local files to sandbox -openshell sandbox upload my-sandbox ./src /sandbox/src +# Upload local files to the sandbox working directory +openshell sandbox upload my-sandbox ./src # Download files from sandbox openshell sandbox download my-sandbox /sandbox/output ./local-output @@ -231,6 +231,10 @@ Uploads honor `.gitignore` by default. Add `--no-git-ignore` only when ignored f Uploads preserve symlinks, including dangling symlinks, instead of dereferencing their targets. A symlink source bypasses Git-aware filtering so the link itself is archived. +When the upload destination is omitted, the CLI discovers the remote working +directory. Downloads must use an absolute sandbox path within that same +canonical working directory. + ### Execute a non-interactive command ```bash diff --git a/.agents/skills/openshell-cli/cli-reference.md b/.agents/skills/openshell-cli/cli-reference.md index 8095660817..1673d80aeb 100644 --- a/.agents/skills/openshell-cli/cli-reference.md +++ b/.agents/skills/openshell-cli/cli-reference.md @@ -258,11 +258,11 @@ Open an interactive SSH shell. The name defaults to the last-used sandbox. `--ed ### `openshell sandbox upload [dest]` -Upload files using tar-over-SSH. The destination defaults to the container working directory. `.gitignore` filtering is enabled unless `--no-git-ignore` is passed. +Upload files using tar-over-SSH. The CLI discovers the canonical remote working directory when the destination is omitted. `.gitignore` filtering is enabled unless `--no-git-ignore` is passed. ### `openshell sandbox download [dest]` -Download files using tar-over-SSH. The local destination defaults to `.`. +Download files using tar-over-SSH. The sandbox source must be an absolute path within the canonical remote working directory. The local destination defaults to `.`. ### `openshell sandbox ssh-config [name]` diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 1629c50de4..8a90c40bb3 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -131,7 +131,8 @@ The gateway preserves whether each policy process field was omitted. The active driver then supplies one authoritative identity input to the supervisor: - Docker and Podman inspect the final sandbox image, pin container creation to - its immutable image ID, and pass its raw OCI `Config.User`. + its immutable image ID, and pass its raw OCI `Config.User`. They also resolve + the workspace from OCI `Config.WorkingDir` during that inspection. - Kubernetes passes its platform-resolved numeric UID/GID, including OpenShift SCC-derived values. - VM keeps its existing guest identity behavior. @@ -144,6 +145,16 @@ and uses the same privilege-drop path for direct and SSH children. When a declaration omits the group, the supervisor fills it with the user's numeric primary GID. It does not rewrite the account files. +Docker and Podman use an absolute OCI working directory as the workspace. An +empty or root (`/`) declaration falls back to `/sandbox`, which OpenShell +creates inside the container when needed. Before direct or SSH children start, +the supervisor transfers ownership of only the workspace directory to the +completed UID/GID. Image-provided contents and nested mounts retain their +ownership. The resolved workspace is also the child cwd and `HOME`, the +automatic writable policy path, and Podman's managed workspace-volume mount +target. Kubernetes/OpenShift keep their `/sandbox` PVC and `fsGroup` behavior, +and VM keeps its `/sandbox` guest initialization path. + Sandbox creation fails before the workload becomes ready when a required image identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. The supervisor itself remains root so it can establish isolation before diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index f8de99a692..1f21dfd698 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -308,22 +308,12 @@ pub async fn sandbox_connect_editor( tls: &TlsOptions, workspace: &str, ) -> Result<()> { - // Verify the sandbox exists before writing SSH config / launching the editor. - let mut client = grpc_client(server, tls).await?; - client - .get_sandbox(GetSandboxRequest { - name: name.to_string(), - workspace: workspace.to_string(), - }) - .await - .into_diagnostic()? - .into_inner() - .sandbox - .ok_or_else(|| miette::miette!("sandbox not found: {name}"))?; + let session = ssh_session_config(server, name, tls, workspace).await?; + let workspace_root = discover_workspace_root(&session).await?; let host_alias = host_alias(name, workspace); install_ssh_config(gateway, name, workspace)?; - launch_editor(editor, &host_alias)?; + launch_editor(editor, &host_alias, &workspace_root)?; eprintln!( "{} Opened {} for sandbox {}", "✓".green().bold(), @@ -776,9 +766,8 @@ fn local_upload_path_is_file_like(path: &Path) -> bool { /// sandbox. Callers are responsible for splitting the destination path so /// that `dest_dir` is always a directory. /// -/// When `dest_dir` is `None`, the sandbox user's home directory (`$HOME`) is -/// used as the extraction target. This avoids hard-coding any particular -/// path and works for custom container images with non-default `WORKDIR`. +/// When `dest_dir` is `None`, the canonical remote workspace is discovered +/// from the SSH session and used as the extraction target. async fn ssh_tar_upload( server: &str, name: &str, @@ -789,9 +778,14 @@ async fn ssh_tar_upload( ) -> Result<()> { let session = ssh_session_config(server, name, tls, workspace).await?; - // When no explicit destination is given, use the unescaped `$HOME` shell - // variable so the remote shell resolves it at runtime. - let escaped_dest = dest_dir.map_or_else(|| "$HOME".to_string(), shell_escape); + let discovered_workspace; + let dest_dir = if let Some(dest_dir) = dest_dir { + dest_dir + } else { + discovered_workspace = discover_workspace_root(&session).await?; + &discovered_workspace + }; + let escaped_dest = shell_escape(dest_dir); let mut ssh = ssh_base_command(&session.proxy_command); ssh.arg("-T") @@ -844,10 +838,6 @@ fn split_sandbox_path(path: &str) -> (&str, &str) { } } -/// Writable root inside every sandbox. Used as the boundary for path-traversal -/// checks on sandbox-side source paths in download flows. -const SANDBOX_WORKSPACE_ROOT: &str = "/sandbox"; - /// Lexically clean a POSIX-style absolute path by resolving `.` and `..` /// components, collapsing repeated separators, and stripping any trailing /// slash. Returns `None` if the input is empty or relative — the caller is @@ -887,44 +877,45 @@ fn lexical_clean_absolute_path(path: &str) -> Option { /// resolves under the sandbox writable root. /// /// Returns the cleaned, traversal-resolved path on success. Refuses any -/// path that lexically escapes `/sandbox` (e.g. `/etc/passwd`, -/// `/sandbox/../etc/passwd`) with a user-facing error. +/// path that lexically escapes the discovered workspace root with a +/// user-facing error. /// /// This is a lexical guard only — it does not follow symlinks. Call /// `resolve_sandbox_source_path` after this on any path that will be passed -/// to a subsequent SSH I/O operation, so a symlink such as -/// `/sandbox/etc-link -> /etc` cannot leak files outside the workspace. -fn validate_sandbox_source_path(path: &str) -> Result { +/// to a subsequent SSH I/O operation, so a workspace symlink to `/etc` cannot +/// leak files outside the workspace. +fn validate_sandbox_source_path(path: &str, workspace_root: &str) -> Result { if path.is_empty() { return Err(miette::miette!("sandbox source path is empty")); } let cleaned = lexical_clean_absolute_path(path) .ok_or_else(|| miette::miette!("sandbox source path must be absolute (got '{path}')"))?; - if !is_under_sandbox_workspace(&cleaned) { + if !is_under_workspace(&cleaned, workspace_root) { return Err(miette::miette!( - "sandbox source path '{path}' is outside the sandbox workspace ({SANDBOX_WORKSPACE_ROOT})" + "sandbox source path '{path}' is outside the sandbox workspace ({workspace_root})" )); } Ok(cleaned) } -/// Pure helper: is `path` equal to `/sandbox` or a descendant of it? -fn is_under_sandbox_workspace(path: &str) -> bool { - path == SANDBOX_WORKSPACE_ROOT || path.starts_with(&format!("{SANDBOX_WORKSPACE_ROOT}/")) +/// Pure helper: is `path` equal to the workspace root or a descendant of it? +fn is_under_workspace(path: &str, workspace_root: &str) -> bool { + path == workspace_root || path.starts_with(&format!("{workspace_root}/")) } /// Resolve every symlink in `sandbox_path` on the sandbox side and refuse the -/// result if it lands outside `/sandbox`. +/// result if it lands outside the discovered workspace root. /// /// The lexical guard in `validate_sandbox_source_path` cannot see symlinks; a -/// path such as `/sandbox/etc-link/passwd` (where `etc-link -> /etc`) clears -/// the lexical check but would still leak `/etc/passwd` once `tar -C` follows -/// the link. Resolving symlinks on the remote side and re-validating closes -/// that gap. The returned fully-resolved path is what the caller should hand -/// to probe and tar invocations. +/// workspace path through `etc-link -> /etc` clears the lexical check but +/// would still leak `/etc/passwd` once `tar -C` follows the link. Resolving +/// symlinks on the remote side and re-validating closes that gap. The returned +/// fully-resolved path is what the caller should hand to probe and tar +/// invocations. async fn resolve_sandbox_source_path( session: &SshSessionConfig, sandbox_path: &str, + workspace_root: &str, ) -> Result { let resolve_cmd = format!("realpath -e -- {path}", path = shell_escape(sandbox_path)); let resolved = ssh_run_capture_stdout(session, &resolve_cmd) @@ -935,9 +926,9 @@ async fn resolve_sandbox_source_path( "sandbox source path '{sandbox_path}' does not exist" )); } - if !is_under_sandbox_workspace(&resolved) { + if !is_under_workspace(&resolved, workspace_root) { return Err(miette::miette!( - "sandbox source path '{sandbox_path}' resolves to '{resolved}', outside the sandbox workspace ({SANDBOX_WORKSPACE_ROOT})" + "sandbox source path '{sandbox_path}' resolves to '{resolved}', outside the sandbox workspace ({workspace_root})" )); } Ok(resolved) @@ -971,7 +962,7 @@ fn resolve_file_download_target( /// /// Files are streamed as a tar archive to `ssh ... tar xf - -C ` on /// the sandbox side. When `dest` is `None`, files are uploaded to the -/// sandbox user's home directory. +/// discovered sandbox working directory. #[allow(clippy::too_many_arguments)] pub async fn sandbox_sync_up_files( server: &str, @@ -1003,11 +994,11 @@ pub async fn sandbox_sync_up_files( /// Push a local path (file or directory) into a sandbox using tar-over-SSH. /// -/// When `sandbox_path` is `None`, files are uploaded to the sandbox user's -/// home directory. When uploading a single file to an explicit destination -/// that does not end with `/`, the destination is treated as a file path: -/// the parent directory is created and the file is written with the -/// destination's basename. This matches `cp` / `scp` semantics. +/// When `sandbox_path` is `None`, files are uploaded to the discovered sandbox +/// working directory. When uploading a single file to an explicit destination +/// that does not end with `/`, the destination is treated as a file path: the +/// parent directory is created and the file is written with the destination's +/// basename. This matches `cp` / `scp` semantics. pub async fn sandbox_sync_up( server: &str, name: &str, @@ -1021,10 +1012,10 @@ pub async fn sandbox_sync_up( // `mkdir -p` creates the parent and tar extracts the file with the right // name. // - // Exception: if splitting would yield "/" as the parent (e.g. the user - // passed "/sandbox"), fall through to directory semantics instead. The - // sandbox user cannot write to "/" and the intent is almost certainly - // "put the file inside /sandbox", not "create a file named sandbox in /". + // Exception: if splitting would yield "/" as the parent, fall through to + // directory semantics instead. The sandbox user cannot write to "/" and + // the intent is almost certainly to place the file inside the named + // top-level directory. let local_path_is_file_like = local_upload_path_is_file_like(local_path); if let Some(path) = sandbox_path && local_path_is_file_like @@ -1127,6 +1118,29 @@ async fn ssh_run_capture_stdout(session: &SshSessionConfig, command: &str) -> Re Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } +fn validate_discovered_workspace_root(root: &str) -> Result { + let cleaned = lexical_clean_absolute_path(root) + .ok_or_else(|| miette::miette!("remote workspace must be an absolute path"))?; + if cleaned == "/" { + return Err(miette::miette!( + "remote workspace resolved to the container root" + )); + } + if cleaned != root { + return Err(miette::miette!( + "remote workspace '{root}' is not a canonical absolute path" + )); + } + Ok(cleaned) +} + +async fn discover_workspace_root(session: &SshSessionConfig) -> Result { + let root = ssh_run_capture_stdout(session, "pwd -P") + .await + .wrap_err("failed to discover remote workspace")?; + validate_discovered_workspace_root(&root) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum SandboxSourceKind { File, @@ -1170,7 +1184,7 @@ async fn probe_sandbox_source_kind( /// /// The sandbox source path is also subjected to a workspace-boundary check /// before any SSH command is issued; paths that lexically resolve outside -/// `/sandbox` are refused. +/// the discovered workspace root are refused. pub async fn sandbox_sync_down( server: &str, name: &str, @@ -1179,9 +1193,11 @@ pub async fn sandbox_sync_down( tls: &TlsOptions, workspace: &str, ) -> Result<()> { - let sandbox_path = validate_sandbox_source_path(sandbox_path)?; let session = ssh_session_config(server, name, tls, workspace).await?; - let sandbox_path = resolve_sandbox_source_path(&session, &sandbox_path).await?; + let workspace_root = discover_workspace_root(&session).await?; + let sandbox_path = validate_sandbox_source_path(sandbox_path, &workspace_root)?; + let sandbox_path = + resolve_sandbox_source_path(&session, &sandbox_path, &workspace_root).await?; let kind = probe_sandbox_source_kind(&session, &sandbox_path).await?; match kind { @@ -1631,19 +1647,25 @@ pub fn install_ssh_config(gateway: &str, name: &str, workspace: &str) -> Result< Ok(managed_config) } -fn launch_editor(editor: Editor, host_alias: &str) -> Result<()> { +fn launch_editor(editor: Editor, host_alias: &str, workspace_root: &str) -> Result<()> { launch_editor_command( editor.binary(), editor.label(), &Editor::remote_target(host_alias), + workspace_root, ) } -fn launch_editor_command(binary: &str, label: &str, remote_target: &str) -> Result<()> { +fn launch_editor_command( + binary: &str, + label: &str, + remote_target: &str, + workspace_root: &str, +) -> Result<()> { let status = Command::new(binary) .arg("--remote") .arg(remote_target) - .arg("/sandbox") + .arg(workspace_root) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -1776,6 +1798,7 @@ mod tests { "openshell-test-missing-binary", "Test Editor", "ssh-remote+openshell-demo", + "/workspace/project", ) .unwrap_err(); let text = format!("{err}"); @@ -1968,68 +1991,105 @@ mod tests { #[test] fn validate_sandbox_source_path_accepts_workspace_paths() { + let workspace_root = "/workspace/project"; assert_eq!( - validate_sandbox_source_path("/sandbox/file.txt").unwrap(), - "/sandbox/file.txt" + validate_sandbox_source_path("/workspace/project/file.txt", workspace_root).unwrap(), + "/workspace/project/file.txt" ); assert_eq!( - validate_sandbox_source_path("/sandbox/.agent/workspace/hello.txt").unwrap(), - "/sandbox/.agent/workspace/hello.txt" + validate_sandbox_source_path( + "/workspace/project/.agent/workspace/hello.txt", + workspace_root + ) + .unwrap(), + "/workspace/project/.agent/workspace/hello.txt" ); assert_eq!( - validate_sandbox_source_path("/sandbox").unwrap(), - "/sandbox" + validate_sandbox_source_path("/workspace/project", workspace_root).unwrap(), + "/workspace/project" ); assert_eq!( - validate_sandbox_source_path("/sandbox/").unwrap(), - "/sandbox" + validate_sandbox_source_path("/workspace/project/", workspace_root).unwrap(), + "/workspace/project" ); assert_eq!( - validate_sandbox_source_path("/sandbox/sub/../file").unwrap(), - "/sandbox/file" + validate_sandbox_source_path("/workspace/project/sub/../file", workspace_root).unwrap(), + "/workspace/project/file" ); } #[test] fn validate_sandbox_source_path_rejects_traversal_and_escapes() { - let traversal = validate_sandbox_source_path("/etc/passwd").unwrap_err(); + let workspace_root = "/workspace/project"; + let traversal = validate_sandbox_source_path("/etc/passwd", workspace_root).unwrap_err(); assert!( format!("{traversal}").contains("outside the sandbox workspace"), "unexpected error: {traversal}" ); - let parent_escape = validate_sandbox_source_path("/sandbox/../etc/passwd").unwrap_err(); + let parent_escape = + validate_sandbox_source_path("/workspace/project/../../etc/passwd", workspace_root) + .unwrap_err(); assert!( format!("{parent_escape}").contains("outside the sandbox workspace"), "unexpected error: {parent_escape}" ); - let prefix_only = validate_sandbox_source_path("/sandboxed/secrets").unwrap_err(); + let prefix_only = + validate_sandbox_source_path("/workspace/projected/secrets", workspace_root) + .unwrap_err(); assert!( format!("{prefix_only}").contains("outside the sandbox workspace"), "unexpected error: {prefix_only}" ); - let empty = validate_sandbox_source_path("").unwrap_err(); + let empty = validate_sandbox_source_path("", workspace_root).unwrap_err(); assert!(format!("{empty}").contains("empty")); - let relative = validate_sandbox_source_path("sandbox/file").unwrap_err(); + let relative = + validate_sandbox_source_path("workspace/project/file", workspace_root).unwrap_err(); assert!(format!("{relative}").contains("must be absolute")); } #[test] - fn is_under_sandbox_workspace_accepts_root_and_descendants() { - assert!(is_under_sandbox_workspace("/sandbox")); - assert!(is_under_sandbox_workspace("/sandbox/file")); - assert!(is_under_sandbox_workspace("/sandbox/sub/nested")); + fn is_under_workspace_accepts_root_and_descendants() { + assert!(is_under_workspace( + "/workspace/project", + "/workspace/project" + )); + assert!(is_under_workspace( + "/workspace/project/file", + "/workspace/project" + )); + assert!(is_under_workspace( + "/workspace/project/sub/nested", + "/workspace/project" + )); + } + + #[test] + fn is_under_workspace_rejects_outside_paths_and_prefix_collisions() { + assert!(!is_under_workspace("/etc/passwd", "/workspace/project")); + assert!(!is_under_workspace( + "/workspace/projected/secrets", + "/workspace/project" + )); + assert!(!is_under_workspace("/", "/workspace/project")); + assert!(!is_under_workspace("", "/workspace/project")); } #[test] - fn is_under_sandbox_workspace_rejects_outside_paths_and_prefix_collisions() { - assert!(!is_under_sandbox_workspace("/etc/passwd")); - assert!(!is_under_sandbox_workspace("/sandboxed/secrets")); - assert!(!is_under_sandbox_workspace("/")); - assert!(!is_under_sandbox_workspace("")); + fn discovered_workspace_root_must_be_canonical_absolute_non_root() { + assert_eq!( + validate_discovered_workspace_root("/workspace/project").unwrap(), + "/workspace/project" + ); + for invalid in ["", "workspace", "/", "/workspace/../etc", "/workspace/"] { + assert!( + validate_discovered_workspace_root(invalid).is_err(), + "expected '{invalid}' to be rejected" + ); + } } #[test] diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 086d992c37..e1b94f11a1 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -32,6 +32,10 @@ const RESERVED_MOUNT_TARGETS: &[&str] = &[ "/run/netns", ]; +/// Compatibility workspace used when an OCI image has no usable working +/// directory and by drivers whose workspace remains fixed. +pub const DEFAULT_WORKSPACE_ROOT: &str = "/sandbox"; + /// Validate a non-empty driver mount source. pub fn validate_mount_source(source: &str, field: &str) -> Result<(), String> { if source.is_empty() { @@ -114,9 +118,6 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { { return Err("mount target must not contain '..'".to_string()); } - if path == Path::new("/sandbox") { - return Err("mount target '/sandbox' is reserved for the OpenShell workspace".to_string()); - } for reserved in RESERVED_MOUNT_TARGETS { if path_is_or_under(path, Path::new(reserved)) { return Err(format!( @@ -127,6 +128,51 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { Ok(()) } +/// Resolve an OCI image working directory to the internal workspace root used +/// by local container drivers. +/// +/// Empty declarations and `/` use the compatibility fallback. Non-empty +/// declarations must already be normalized absolute paths so the inspected +/// value and the path passed to the supervisor cannot be interpreted +/// differently. +pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { + if working_dir.is_empty() || working_dir == "/" { + return Ok(DEFAULT_WORKSPACE_ROOT.to_string()); + } + if working_dir.as_bytes().contains(&0) { + return Err("OCI WorkingDir must not contain NUL bytes".to_string()); + } + if !working_dir.starts_with('/') { + return Err(format!( + "OCI WorkingDir '{working_dir}' must be an absolute container path" + )); + } + + let segments = working_dir.split('/').skip(1).collect::>(); + let has_internal_empty_segment = segments + .iter() + .take(segments.len().saturating_sub(1)) + .any(|segment| segment.is_empty()); + if has_internal_empty_segment || segments.contains(&".") || segments.contains(&"..") { + return Err(format!( + "OCI WorkingDir '{working_dir}' must be normalized without empty, '.', or '..' path segments" + )); + } + + Ok(working_dir.trim_end_matches('/').to_string()) +} + +/// Reject a user-supplied mount that would replace the resolved workspace +/// root. Mounts below the workspace remain valid. +pub fn validate_workspace_mount_target(target: &str, workspace_root: &str) -> Result<(), String> { + if normalize_mount_target(target) == workspace_root { + return Err(format!( + "mount target '{target}' is reserved for the OpenShell workspace" + )); + } + Ok(()) +} + /// Normalize a validated container-side mount target for semantic comparison. pub fn normalize_mount_target(target: &str) -> String { if target == "/" { @@ -151,10 +197,42 @@ mod tests { } #[test] - fn container_target_rejects_workspace_root_only() { - let err = validate_container_mount_target("/sandbox/").unwrap_err(); + fn container_target_workspace_reservation_is_dynamic() { + validate_container_mount_target("/sandbox/").unwrap(); + validate_workspace_mount_target("/sandbox/", "/sandbox").unwrap_err(); + validate_workspace_mount_target("/workspace/", "/sandbox").unwrap(); + validate_workspace_mount_target("/workspace/cache", "/workspace").unwrap(); + } - assert!(err.contains("reserved for the OpenShell workspace")); + #[test] + fn oci_workspace_root_uses_fallback_and_accepts_normalized_absolute_paths() { + assert_eq!(resolve_oci_workspace_root("").unwrap(), "/sandbox"); + assert_eq!(resolve_oci_workspace_root("/").unwrap(), "/sandbox"); + assert_eq!( + resolve_oci_workspace_root("/workspace/project/").unwrap(), + "/workspace/project" + ); + assert_eq!( + resolve_oci_workspace_root("/workspace with spaces").unwrap(), + "/workspace with spaces" + ); + } + + #[test] + fn oci_workspace_root_rejects_relative_and_malformed_paths() { + for invalid in [ + "workspace", + "./workspace", + "/workspace/../etc", + "/workspace/./project", + "/workspace//project", + "/workspace\0project", + ] { + assert!( + resolve_oci_workspace_root(invalid).is_err(), + "expected '{invalid}' to be rejected" + ); + } } #[test] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index b2e74231ba..2457faf3f5 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -19,13 +19,22 @@ sandbox and starts the `openshell-sandbox` supervisor inside that container. The supervisor then creates the nested sandbox namespace for the agent process. Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID, preventing a mutable tag from changing between inspection -and launch. The supervisor runs as root, resolves omitted policy identity fields -from the image declaration, and drops only agent children to the resulting -identity. Named OCI components remain names after validation; a missing group -is filled with the user's numeric primary GID. Explicit `process.run_as_user` -and `process.run_as_group` values take precedence independently. +captures its immutable image ID, raw OCI `Config.User`, and OCI +`Config.WorkingDir`. Container creation uses that image ID, preventing a +mutable tag from changing between inspection and launch. The supervisor runs as +root, resolves omitted policy identity fields from the image declaration, and +drops only agent children to the resulting identity. Named OCI components +remain names after validation; a missing group is filled with the user's +numeric primary GID. Explicit `process.run_as_user` and +`process.run_as_group` values take precedence independently. + +An absolute OCI working directory becomes the agent workspace. An empty or +root (`/`) declaration falls back to `/sandbox`, which OpenShell creates when +necessary. The supervisor transfers ownership of only the workspace directory +to the completed identity before direct or SSH children start. Existing +contents and nested bind or volume mounts retain their ownership. The workspace +is the child cwd and `HOME`; an invalid, symlinked, or unpreparable workspace +fails sandbox startup. Docker containers join an OpenShell-managed bridge network. The driver injects `host.openshell.internal` and `host.docker.internal` so supervisors have stable @@ -77,9 +86,9 @@ optional `selinux_label` of `shared` (applies `:z`) or `private` (applies `subpath`. User-supplied bind and volume mounts are read-only by default; set `read_only: false` to make them writable. Mount `source`, `target`, and `subpath` values must not contain surrounding whitespace. Mount targets must be -absolute container paths and must not replace the workspace root (`/sandbox`) -or overlap OpenShell supervisor files, `/etc/openshell`, `/etc/openshell-tls`, -or `/run/netns`. +absolute container paths and must not replace the resolved workspace root. +Nested workspace mounts remain valid. Mounts also must not overlap OpenShell +supervisor files, `/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. Example named-volume usage: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1b3fd25ef1..9ebf618db1 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -231,6 +231,7 @@ struct DockerProvisioningFailure { struct DockerImageMetadata { id: String, user: String, + working_dir: String, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -1348,11 +1349,20 @@ impl DockerComputeDriver { "docker image '{image}' inspection did not return an immutable image ID" )) })?; - let user = inspect - .config - .and_then(|config| config.user) - .unwrap_or_default(); - Ok(DockerImageMetadata { id, user }) + let (user, working_dir) = inspect.config.map_or_else( + || (String::new(), String::new()), + |config| { + ( + config.user.unwrap_or_default(), + config.working_dir.unwrap_or_default(), + ) + }, + ); + Ok(DockerImageMetadata { + id, + user, + working_dir, + }) } async fn pull_image(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { @@ -2361,6 +2371,7 @@ fn build_container_create_body_with_gpu_devices( &DockerImageMetadata { id: template.image.clone(), user: String::new(), + working_dir: String::new(), }, ) } @@ -2381,6 +2392,18 @@ fn build_container_create_body_for_image( .as_ref() .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; let resource_limits = docker_resource_limits(template)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; + for mount in &driver_config.mounts { + let target = match mount { + DockerDriverMountConfig::Bind { target, .. } + | DockerDriverMountConfig::Volume { target, .. } + | DockerDriverMountConfig::Tmpfs { target, .. } + | DockerDriverMountConfig::Image { target, .. } => target, + }; + driver_mounts::validate_workspace_mount_target(target, &workspace_root) + .map_err(Status::failed_precondition)?; + } let user_mounts = docker_driver_mounts(driver_config)?; let user_bind_strings = docker_driver_bind_strings(driver_config)?; let device_requests = gpu_device_ids.map(|device_ids| { @@ -2417,7 +2440,7 @@ fn build_container_create_body_for_image( entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Clear the image CMD so Docker does not append inherited args to the // supervisor entrypoint. - cmd: Some(Vec::new()), + cmd: Some(vec!["--workdir".to_string(), workspace_root]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 7562e4a8fb..518c2d8b67 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -571,6 +571,7 @@ fn container_creation_uses_inspected_immutable_image() { let metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), + working_dir: "/workspace/project".to_string(), }; let body = build_container_create_body_for_image( &sandbox, @@ -583,12 +584,74 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!(body.image.as_deref(), Some("sha256:immutable")); assert_eq!(body.user.as_deref(), Some("0")); + assert_eq!( + body.cmd.as_deref(), + Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) + ); assert!(body.env.unwrap().contains(&format!( "{}=1234:1235", openshell_core::sandbox_env::OCI_IMAGE_USER ))); } +#[test] +fn container_creation_rejects_invalid_oci_working_dir() { + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "relative/workspace".to_string(), + }; + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + ) + .unwrap_err(); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("must be an absolute container path")); +} + +#[test] +fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/workspace".to_string(), + }; + let root_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace"}] + })) + .unwrap(); + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &root_mount, + None, + &metadata, + ) + .unwrap_err(); + assert!( + err.message() + .contains("reserved for the OpenShell workspace") + ); + + let nested_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace/cache"}] + })) + .unwrap(); + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &nested_mount, + None, + &metadata, + ) + .expect("nested workspace mounts remain supported"); +} + #[test] fn build_environment_keeps_path_driver_controlled() { let mut sandbox = test_sandbox(); @@ -1219,14 +1282,17 @@ fn managed_container_label_filters_include_gateway_namespace() { } #[test] -fn build_container_create_body_clears_inherited_cmd() { +fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); assert_eq!( create_body.entrypoint, Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]) ); - assert_eq!(create_body.cmd, Some(Vec::new())); + assert_eq!( + create_body.cmd, + Some(vec!["--workdir".to_string(), "/sandbox".to_string()]) + ); assert_eq!( create_body .labels diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 2d947b8a28..ccb71290b7 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -312,6 +312,10 @@ fn validate_kubernetes_driver_volume_mounts( } driver_mounts::validate_container_mount_target(&mount.mount_path)?; + driver_mounts::validate_workspace_mount_target( + &mount.mount_path, + driver_mounts::DEFAULT_WORKSPACE_ROOT, + )?; let normalized_mount_path = driver_mounts::normalize_mount_target(&mount.mount_path); if !mount_paths.insert(normalized_mount_path.clone()) { return Err(format!( @@ -1601,7 +1605,11 @@ fn apply_supervisor_sideload( // Override command to use the side-loaded supervisor binary container.insert( "command".to_string(), - serde_json::json!([format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH)]), + serde_json::json!([ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]), ); // Force the supervisor to run as root (UID 0). Sandbox images may set @@ -1917,7 +1925,9 @@ fn apply_supervisor_sidecar_topology( "command".to_string(), serde_json::json!([ format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--mode=process" + "--mode=process", + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT ]), ); @@ -4064,6 +4074,16 @@ mod tests { command[0].as_str().unwrap(), format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") ); + assert_eq!( + command, + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]) + .as_array() + .unwrap() + ); // Agent volume mount should be read-only let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] @@ -4211,7 +4231,9 @@ mod tests { agent["command"], serde_json::json!([ format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--mode=process" + "--mode=process", + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT ]) ); assert_eq!(agent["securityContext"]["runAsUser"], 1500); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 965a295d19..b5161eae27 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -8,14 +8,23 @@ isolation enforcement to the `openshell-sandbox` supervisor binary, which is sideloaded into each container via an OCI image volume mount. Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID with pulling disabled, preventing a mutable tag from changing -between inspection and launch. The supervisor runs as root, resolves omitted -policy identity fields from the image declaration, and drops only agent -children to the completed identity. Named OCI components remain names after -validation; a missing group is filled with the user's numeric primary GID. Explicit -`process.run_as_user` and `process.run_as_group` values take precedence -independently. +captures its immutable image ID, raw OCI `Config.User`, and OCI +`Config.WorkingDir`. Container creation uses that image ID with pulling +disabled, preventing a mutable tag from changing between inspection and launch. +The supervisor runs as root, resolves omitted policy identity fields from the +image declaration, and drops only agent children to the completed identity. +Named OCI components remain names after validation; a missing group is filled +with the user's numeric primary GID. Explicit `process.run_as_user` and +`process.run_as_group` values take precedence independently. + +An absolute OCI working directory becomes the agent workspace. An empty or +root (`/`) declaration falls back to `/sandbox`, which OpenShell creates when +necessary. The supervisor transfers ownership of only the workspace directory +to the completed identity before direct or SSH children start. Existing +contents and nested bind or volume mounts retain their ownership. The workspace +is the child cwd and `HOME`, and the Podman-managed workspace volume is mounted +there so normal volume copy-up preserves image content. An invalid, symlinked, +or unpreparable workspace fails sandbox startup. For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). @@ -87,9 +96,10 @@ optional `selinux_label` of `shared` (applies `:z`) or `private` (applies read-only by default; set `read_only: false` to make them writable. Podman image and volume mounts do not support `subpath` in OpenShell driver config. Mount `source` and `target` values must not contain surrounding whitespace. -Mount targets must be absolute container paths and must not replace -the workspace root (`/sandbox`) or overlap OpenShell supervisor files, -`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. +Mount targets must be absolute container paths and must not replace the +resolved workspace root. Nested workspace mounts remain valid. Mounts also +must not overlap OpenShell supervisor files, `/etc/openshell`, +`/etc/openshell-tls`, or `/run/netns`. Example named-volume usage: diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 952938d47a..51dadced1d 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -177,6 +177,8 @@ pub struct ImageInspect { pub struct ImageConfig { #[serde(default)] pub user: String, + #[serde(default)] + pub working_dir: String, } /// A container summary returned by the list API. @@ -932,12 +934,12 @@ mod tests { } #[tokio::test] - async fn inspect_image_reads_immutable_id_and_oci_user() { + async fn inspect_image_reads_immutable_id_and_oci_config() { let (socket_path, request_log, handle) = spawn_podman_stub( "inspect-image", vec![StubResponse::new( StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"User":"app:staff"}}"#, + r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","WorkingDir":"/workspace/project"}}"#, )], ); let client = PodmanClient::new(socket_path.clone()); @@ -952,6 +954,13 @@ mod tests { image.config.as_ref().map(|config| config.user.as_str()), Some("app:staff") ); + assert_eq!( + image + .config + .as_ref() + .map(|config| config.working_dir.as_str()), + Some("/workspace/project") + ); handle.await.expect("stub task should finish"); assert_eq!( request_log diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 90ef0fec21..ba6f48ca48 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -612,6 +612,7 @@ pub fn podman_driver_image_mount_sources( fn podman_user_mounts( sandbox: &DriverSandbox, enable_bind_mounts: bool, + workspace_root: &str, ) -> Result { let template = sandbox .spec @@ -623,6 +624,13 @@ fn podman_user_mounts( let config = podman_driver_config(template, enable_bind_mounts)?; let mut result = PodmanUserMounts::default(); for mount in config.mounts { + let target = match &mount { + PodmanDriverMountConfig::Bind { target, .. } + | PodmanDriverMountConfig::Volume { target, .. } + | PodmanDriverMountConfig::Tmpfs { target, .. } + | PodmanDriverMountConfig::Image { target, .. } => target, + }; + driver_mounts::validate_workspace_mount_target(target, workspace_root)?; match mount { PodmanDriverMountConfig::Bind { source, @@ -905,6 +913,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( image, image, "", + "", ) } @@ -916,14 +925,17 @@ pub fn build_container_spec_for_image( requested_image: &str, image_id: &str, oci_user: &str, + oci_working_dir: &str, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); + let workspace_root = driver_mounts::resolve_oci_workspace_root(oci_working_dir) + .map_err(ComputeDriverError::Precondition)?; let env = build_env(sandbox, config, requested_image, oci_user); let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); - let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) + let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts, &workspace_root) .map_err(ComputeDriverError::InvalidArgument)?; if sandbox .spec @@ -952,7 +964,7 @@ pub fn build_container_spec_for_image( let mut volumes = vec![NamedVolume { name: vol, - dest: "/sandbox".into(), + dest: workspace_root.clone(), options: vec!["rw".into()], }]; volumes.extend(user_mounts.volumes); @@ -964,6 +976,9 @@ pub fn build_container_spec_for_image( }]; image_volumes.extend(user_mounts.image_volumes); + let mut command = vec!["--workdir".to_string(), workspace_root]; + command.extend(upstream_proxy_cli_args(config)); + let container_spec = ContainerSpec { name, image: image_id.to_string(), @@ -987,7 +1002,7 @@ pub fn build_container_spec_for_image( // Operator-owned corporate proxy flags. The workload command is not // part of argv (the supervisor takes it from the reserved command // env var), so these flags are the whole command list. - command: upstream_proxy_cli_args(config), + command, // Force the supervisor to run as root (UID 0). Sandbox images may // set a non-root USER directive (e.g. `USER sandbox`), but the // supervisor needs root to create network namespaces, set up the @@ -1379,6 +1394,7 @@ mod tests { "registry.example/app:latest", "sha256:immutable", "app:staff", + "/workspace/project", ) .unwrap(); @@ -1389,6 +1405,17 @@ mod tests { ); assert_eq!(container["user"].as_str(), Some("0:0")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); + assert_eq!( + container["command"], + serde_json::json!(["--workdir", "/workspace/project"]) + ); + assert!(container["volumes"].as_array().is_some_and(|volumes| { + volumes.iter().any(|volume| { + volume["name"].as_str() == Some("openshell-sandbox-test-id-workspace") + && volume["dest"].as_str() == Some("/workspace/project") + && volume["options"] == serde_json::json!(["rw"]) + }) + })); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") @@ -1403,6 +1430,89 @@ mod tests { ); } + #[test] + fn container_spec_rejects_invalid_oci_working_dir() { + let err = build_container_spec_for_image( + &test_sandbox("test-id", "test-name"), + &test_config(), + None, + None, + "registry.example/app:latest", + "sha256:immutable", + "app:staff", + "relative/workspace", + ) + .unwrap_err(); + + assert!( + err.to_string() + .contains("must be an absolute container path") + ); + } + + #[test] + fn container_spec_reserves_resolved_workspace_root_but_allows_nested_mounts() { + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec = Some(openshell_core::proto::compute::v1::DriverSandboxSpec { + template: Some(DriverSandboxTemplate::default()), + ..Default::default() + }); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{ + "type": "tmpfs", + "target": "/workspace" + }] + }))); + + let err = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "registry.example/app:latest", + "sha256:immutable", + "app:staff", + "/workspace", + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("reserved for the OpenShell workspace") + ); + + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{ + "type": "tmpfs", + "target": "/workspace/cache" + }] + }))); + build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "registry.example/app:latest", + "sha256:immutable", + "app:staff", + "/workspace", + ) + .expect("nested workspace mounts remain supported"); + } + #[test] fn volume_name_uses_id() { assert_eq!( diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index aa3df9cbd7..43c7c68794 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -585,6 +585,10 @@ impl PodmanComputeDriver { .config .as_ref() .map_or("", |config| config.user.as_str()); + let image_working_dir = inspected_image + .config + .as_ref() + .map_or("", |config| config.working_dir.as_str()); for image in container::podman_driver_image_mount_sources(sandbox, self.config.enable_bind_mounts) @@ -652,6 +656,7 @@ impl PodmanComputeDriver { image, &inspected_image.id, image_user, + image_working_dir, ) { Ok(spec) => spec, Err(e) => { diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 7937a3757e..1e81451c7b 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -1070,7 +1070,7 @@ pub fn restrictive_default_policy() -> SandboxPolicy { "/etc".into(), "/var/log".into(), ], - read_write: vec!["/sandbox".into(), "/tmp".into(), "/dev/null".into()], + read_write: vec!["/tmp".into(), "/dev/null".into()], }), landlock: Some(LandlockPolicy { compatibility: "best_effort".into(), @@ -1650,8 +1650,8 @@ network_policies: "read_only should contain /usr" ); assert!( - fs.read_write.iter().any(|p| p == "/sandbox"), - "read_write should contain /sandbox" + !fs.read_write.iter().any(|p| p == "/sandbox"), + "the workspace should be granted through include_workdir, not a literal /sandbox path" ); assert!( fs.read_write.iter().any(|p| p == "/tmp"), diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 5ba41b9d7e..1585a3010f 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -189,16 +189,23 @@ pub async fn run_sandbox( // OpenShift retain their authoritative numeric pair; Docker and Podman // fill only omitted policy fields from OCI Config.User. #[cfg(unix)] - let resolved_process_identity = { + let (resolved_process_identity, workspace_home) = { let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; - openshell_supervisor_process::identity::resolve_process_identity( + let workspace_home = matches!( + &driver_identity, + openshell_supervisor_process::identity::DriverIdentity::OciUser { .. } + ); + let resolved = openshell_supervisor_process::identity::resolve_process_identity( &mut policy, &driver_identity, - )? + )?; + (resolved, workspace_home) }; #[cfg(not(unix))] - let resolved_process_identity = - openshell_supervisor_process::process::ResolvedProcessIdentity::default(); + let (resolved_process_identity, workspace_home) = ( + openshell_supervisor_process::process::ResolvedProcessIdentity::default(), + false, + ); #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] let (provider_credentials, mut provider_env) = @@ -684,6 +691,7 @@ pub async fn run_sandbox( program, args, workdir.as_deref(), + workspace_home, timeout_secs, interactive, sandbox_id.as_deref(), @@ -1151,9 +1159,9 @@ const PROXY_BASELINE_READ_ONLY: &[&str] = &[ "/dev/urandom", ]; -/// Minimum read-write paths required for a proxy-mode sandbox child process: -/// user working directory and temporary files. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; +/// Minimum read-write paths required for a proxy-mode sandbox child process. +/// The active workspace is granted separately through `include_workdir`. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp"]; /// GPU read-only paths. /// @@ -1502,10 +1510,10 @@ mod baseline_tests { } #[test] - fn baseline_read_write_always_includes_sandbox_and_tmp() { + fn baseline_read_write_uses_dynamic_workdir_instead_of_sandbox() { let (_ro, rw) = baseline_enrichment_paths(); - assert!(rw.contains(&"/sandbox".to_string())); assert!(rw.contains(&"/tmp".to_string())); + assert!(!rw.contains(&"/sandbox".to_string())); } #[test] diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 93ab4787b3..99e82505da 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -528,6 +528,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, + workspace_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -540,6 +541,7 @@ impl ProcessHandle { program, args, workdir, + workspace_home, interactive, policy, resolved_identity, @@ -561,6 +563,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, + workspace_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -572,6 +575,7 @@ impl ProcessHandle { program, args, workdir, + workspace_home, interactive, policy, resolved_identity, @@ -587,6 +591,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, + workspace_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -613,6 +618,9 @@ impl ProcessHandle { if let Some(dir) = workdir { cmd.current_dir(dir); + if workspace_home { + cmd.env("HOME", dir); + } } if matches!(policy.network.mode, NetworkMode::Proxy) { @@ -742,6 +750,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, + workspace_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -765,6 +774,9 @@ impl ProcessHandle { if let Some(dir) = workdir { cmd.current_dir(dir); + if workspace_home { + cmd.env("HOME", dir); + } } if matches!(policy.network.mode, NetworkMode::Proxy) { @@ -1252,6 +1264,68 @@ fn chown_sandbox_home(root: &Path, uid: Option, gid: Option) -> Result Ok(()) } +#[cfg(unix)] +fn prepare_oci_workspace(root: &Path, uid: Option, gid: Option) -> Result<()> { + prepare_oci_workspace_with(root, uid, gid, &nix::unistd::chown) +} + +/// Prepare only the resolved `OpenShell` workspace directory itself. +/// +/// Image-provided children retain their declared ownership. This avoids +/// crossing symlinks or user-provided nested mounts. +#[cfg(unix)] +fn prepare_oci_workspace_with( + root: &Path, + uid: Option, + gid: Option, + do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, +) -> Result<()> { + if !root.is_absolute() || root == Path::new("/") { + return Err(miette::miette!( + "workspace path '{}' must be an absolute non-root path", + root.display() + )); + } + + let components = root + .components() + .skip(1) + .map(|component| match component { + std::path::Component::Normal(component) => Ok(component), + _ => Err(miette::miette!( + "workspace path '{}' must be normalized", + root.display() + )), + }) + .collect::>>()?; + + let mut current = PathBuf::from("/"); + for component in components { + current.push(component); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + current.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + current.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(¤t).into_diagnostic()?; + } + Err(error) => return Err(error).into_diagnostic(), + } + } + + do_chown(root, uid, gid).into_diagnostic() +} + #[cfg(unix)] fn chown_children( dir: &Path, @@ -1315,13 +1389,15 @@ fn chown_recursive( /// UIDs/GIDs (passed directly to `chown` without a passwd lookup). #[cfg(unix)] pub fn prepare_filesystem(policy: &SandboxPolicy) -> Result<()> { - prepare_filesystem_with_identity(policy, ResolvedProcessIdentity::default()) + prepare_filesystem_with_identity(policy, ResolvedProcessIdentity::default(), None, false) } #[cfg(unix)] pub fn prepare_filesystem_with_identity( policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, + workdir: Option<&str>, + prepare_workspace: bool, ) -> Result<()> { use nix::unistd::chown; use nix::unistd::{Gid, Uid}; @@ -1364,6 +1440,20 @@ pub fn prepare_filesystem_with_identity( }, }; + // Docker and Podman own workspace resolution and must make the selected + // root usable by the final effective identity, including when both policy + // identity fields were explicit. Validate it before processing any + // user-authored read-write paths so an unsafe image path fails first. + // Other drivers retain their preparation. + if prepare_workspace { + let workspace = workdir.ok_or_else(|| { + miette::miette!("local container driver did not supply a workspace workdir") + })?; + let workspace = Path::new(workspace); + info!(path = %workspace.display(), ?uid, ?gid, "Preparing resolved workspace"); + prepare_oci_workspace(workspace, uid, gid)?; + } + // Create missing read_write paths and only chown the ones we created. for path in &policy.filesystem.read_write { if prepare_read_write_path(path)? { @@ -2437,6 +2527,179 @@ mod tests { assert!(result.is_err(), "non-EROFS errors should propagate"); } + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_chowns_only_root() { + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + let child = root.join("image-content.txt"); + std::fs::write(&child, "image-owned").unwrap(); + + let chowned = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&chowned); + let fake_chown = + move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + observed.lock().unwrap().push(path.to_path_buf()); + Ok(()) + }; + + prepare_oci_workspace_with( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &fake_chown, + ) + .expect("workspace root should be prepared"); + + assert_eq!(*chowned.lock().unwrap(), vec![root]); + assert!(child.exists(), "image-provided child should be untouched"); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_symlink_root() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let target = base.join("real"); + let link = base.join("link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let err = prepare_oci_workspace( + &link, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + ) + .unwrap_err(); + assert!( + err.to_string().contains("symlink"), + "expected symlink rejection: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_symlink_parent() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let target = base.join("real"); + let parent_link = base.join("parent-link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &parent_link).unwrap(); + + let err = prepare_oci_workspace( + &parent_link.join("workspace"), + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + ) + .unwrap_err(); + assert!( + err.to_string().contains("symlink"), + "expected parent symlink rejection: {err}" + ); + assert!( + !target.join("workspace").exists(), + "workspace must not be created through a symlink parent" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_parent_traversal() { + let err = prepare_oci_workspace( + Path::new("/tmp/workspace/../escape"), + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + ) + .unwrap_err(); + assert!( + err.to_string().contains("must be normalized"), + "expected traversal rejection: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_non_directory_root() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::write(&root, "not a directory").unwrap(); + + let error = prepare_oci_workspace( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + ) + .unwrap_err(); + assert!( + error.to_string().contains("is not a directory"), + "unexpected error: {error}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_propagates_root_chown_error() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + let fake_chown = |_path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + Err(nix::errno::Errno::EROFS) + }; + + let error = prepare_oci_workspace_with( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &fake_chown, + ) + .unwrap_err(); + + assert!( + error.to_string().contains("Read-only file system"), + "unexpected error: {error}" + ); + } + + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_creates_missing_root() { + use std::sync::{Arc, Mutex}; + + let dir = tempfile::tempdir().unwrap(); + let missing = dir + .path() + .canonicalize() + .unwrap() + .join("missing") + .join("sandbox"); + let chowned = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&chowned); + let fake_chown = + move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { + observed.lock().unwrap().push(path.to_path_buf()); + Ok(()) + }; + + prepare_oci_workspace_with( + &missing, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &fake_chown, + ) + .expect("missing OCI workspace should be created"); + + assert!(missing.is_dir()); + assert_eq!(*chowned.lock().unwrap(), vec![missing]); + } + #[cfg(unix)] #[test] fn rewrite_passwd_modifies_existing_sandbox_entry() { diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index a5ff0456c9..8d95eef344 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -54,6 +54,7 @@ pub async fn run_process( program: &str, args: &[String], workdir: Option<&str>, + workspace_home: bool, timeout_secs: u64, interactive: bool, sandbox_id: Option<&str>, @@ -95,7 +96,12 @@ pub async fn run_process( // is forked so the workload sees writable paths it owns. #[cfg(unix)] if enforcement_mode.uses_privileged_process_setup() { - crate::process::prepare_filesystem_with_identity(policy, resolved_process_identity)?; + crate::process::prepare_filesystem_with_identity( + policy, + resolved_process_identity, + workdir, + workspace_home, + )?; } // Eagerly fetch initial settings and install the agent skill if the @@ -244,6 +250,7 @@ pub async fn run_process( ssh_ready_tx, policy_clone, workdir_clone, + workspace_home, netns_fd, proxy_url, ca_paths, @@ -320,6 +327,7 @@ pub async fn run_process( program, args, workdir, + workspace_home, interactive, policy, resolved_process_identity, @@ -334,6 +342,7 @@ pub async fn run_process( program, args, workdir, + workspace_home, interactive, policy, resolved_process_identity, diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 7fa6482a99..73a583a1ff 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -112,6 +112,7 @@ pub async fn run_ssh_server( ready_tx: tokio::sync::oneshot::Sender>, policy: SandboxPolicy, workdir: Option, + workspace_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, @@ -157,6 +158,7 @@ pub async fn run_ssh_server( config, policy, workdir, + workspace_home, netns_fd, proxy_url, ca_paths, @@ -186,6 +188,7 @@ async fn handle_connection( config: Arc, policy: SandboxPolicy, workdir: Option, + workspace_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, @@ -211,6 +214,7 @@ async fn handle_connection( let handler = SshHandler::new( policy, workdir, + workspace_home, netns_fd, proxy_url, ca_file_paths, @@ -241,6 +245,7 @@ struct ChannelState { struct SshHandler { policy: SandboxPolicy, workdir: Option, + workspace_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, @@ -256,6 +261,7 @@ impl SshHandler { fn new( policy: SandboxPolicy, workdir: Option, + workspace_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, @@ -267,6 +273,7 @@ impl SshHandler { Self { policy, workdir, + workspace_home, netns_fd, proxy_url, ca_file_paths, @@ -489,6 +496,7 @@ impl russh::server::Handler for SshHandler { let input_sender = spawn_pipe_exec( &self.policy, self.workdir.clone(), + self.workspace_home, Some("/usr/lib/openssh/sftp-server".to_string()), session.handle(), channel, @@ -586,6 +594,7 @@ impl SshHandler { let (pty_master, input_sender) = spawn_pty_shell( &self.policy, self.workdir.clone(), + self.workspace_home, command, &pty, handle, @@ -607,6 +616,7 @@ impl SshHandler { let input_sender = spawn_pipe_exec( &self.policy, self.workdir.clone(), + self.workspace_home, command, handle, channel, @@ -699,28 +709,31 @@ impl Default for PtyRequest { /// For name-based identities, looks up the home directory via `/etc/passwd` /// (or defaults to `/home/{user}`). /// -/// For numeric UIDs, there is no passwd entry — falls back to -/// `("{uid}", "/sandbox")` so the agent session still has a meaningful -/// USER identifier. -fn session_user_and_home(policy: &SandboxPolicy) -> (String, String) { - match policy.process.run_as_user.as_deref() { +/// For numeric UIDs, there is no passwd entry, so the default remains +/// `("{uid}", "/sandbox")`. Docker and Podman replace that default with their +/// resolved image workspace. +fn session_user_and_home(policy: &SandboxPolicy, workspace_home: Option<&str>) -> (String, String) { + let (user, default_home) = match policy.process.run_as_user.as_deref() { Some(user) if !user.is_empty() => { // Numeric UID — no passwd entry expected; use default HOME. if user.parse::().is_ok() { - return (user.to_string(), "/sandbox".to_string()); + (user.to_string(), "/sandbox".to_string()) + } else { + // Name-based identity — look up home from /etc/passwd. + let home = nix::unistd::User::from_name(user) + .ok() + .flatten() + .map_or_else( + || format!("/home/{user}"), + |u| u.dir.to_string_lossy().into_owned(), + ); + (user.to_string(), home) } - // Name-based identity — look up home from /etc/passwd. - let home = nix::unistd::User::from_name(user) - .ok() - .flatten() - .map_or_else( - || format!("/home/{user}"), - |u| u.dir.to_string_lossy().into_owned(), - ); - (user.to_string(), home) } _ => ("sandbox".to_string(), "/sandbox".to_string()), - } + }; + let home = workspace_home.map_or(default_home, str::to_string); + (user, home) } #[allow(clippy::too_many_arguments)] @@ -774,6 +787,7 @@ fn apply_child_env( fn spawn_pty_shell( policy: &SandboxPolicy, workdir: Option, + workspace_home: bool, command: Option, pty: &PtyRequest, handle: Handle, @@ -824,7 +838,14 @@ fn spawn_pty_shell( // Derive USER and HOME from the policy's run_as_user when available, // falling back to "sandbox" / "/sandbox" for backward compatibility. - let (session_user, session_home) = session_user_and_home(policy); + let (session_user, session_home) = session_user_and_home( + policy, + if workspace_home { + workdir.as_deref() + } else { + None + }, + ); apply_child_env( &mut cmd, &session_home, @@ -947,6 +968,7 @@ fn spawn_pty_shell( fn spawn_pipe_exec( policy: &SandboxPolicy, workdir: Option, + workspace_home: bool, command: Option, handle: Handle, channel: ChannelId, @@ -978,7 +1000,14 @@ fn spawn_pipe_exec( }, ); - let (session_user, session_home) = session_user_and_home(policy); + let (session_user, session_home) = session_user_and_home( + policy, + if workspace_home { + workdir.as_deref() + } else { + None + }, + ); apply_child_env( &mut cmd, &session_home, @@ -1693,12 +1722,33 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy); + let (user, home) = session_user_and_home(&policy, None); assert_eq!(user, "1000"); // Numeric UID has no passwd entry — defaults to /sandbox. assert_eq!(home, "/sandbox"); } + #[test] + fn session_user_and_home_uses_driver_workspace_when_supplied() { + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, + }; + let policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user: Some("1234".into()), + run_as_group: Some("1235".into()), + }, + }; + + let (user, home) = session_user_and_home(&policy, Some("/workspace/project")); + assert_eq!(user, "1234"); + assert_eq!(home, "/workspace/project"); + } + #[test] fn session_user_and_home_returns_name_from_passwd() { use openshell_core::policy::{ @@ -1714,7 +1764,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy); + let (user, home) = session_user_and_home(&policy, None); assert_eq!(user, "sandbox"); // Name-based — should resolve via passwd (or /home/{user}). assert!(!home.is_empty()); @@ -1735,7 +1785,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy); + let (user, home) = session_user_and_home(&policy, None); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -1755,7 +1805,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy); + let (user, home) = session_user_and_home(&policy, None); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -1775,7 +1825,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy); + let (user, home) = session_user_and_home(&policy, None); assert_eq!(user, "1000660000"); assert_eq!(home, "/sandbox"); } diff --git a/docs/get-started/tutorials/first-network-policy.mdx b/docs/get-started/tutorials/first-network-policy.mdx index 178bd2f095..05888278ba 100644 --- a/docs/get-started/tutorials/first-network-policy.mdx +++ b/docs/get-started/tutorials/first-network-policy.mdx @@ -97,7 +97,7 @@ version: 1 filesystem_policy: include_workdir: true read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] + read_write: [/tmp, /dev/null] landlock: compatibility: best_effort diff --git a/docs/get-started/tutorials/github-sandbox.mdx b/docs/get-started/tutorials/github-sandbox.mdx index 0b11b39345..89b6c0885e 100644 --- a/docs/get-started/tutorials/github-sandbox.mdx +++ b/docs/get-started/tutorials/github-sandbox.mdx @@ -180,7 +180,6 @@ filesystem_policy: - /etc - /var/log read_write: - - /sandbox - /tmp - /dev/null diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 9c4e877eaf..8e50d004b2 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -52,7 +52,7 @@ Controls filesystem access inside the sandbox. Paths not listed in either `read_ |---|---|---|---| | `include_workdir` | bool | No | When `true`, automatically adds the agent's working directory to `read_write`. | | `read_only` | list of strings | No | Paths the agent can read but not modify. Typically system directories like `/usr`, `/lib`, `/etc`. | -| `read_write` | list of strings | No | Paths the agent can read and write. Typically `/sandbox` (working directory) and `/tmp`. | +| `read_write` | list of strings | No | Paths the agent can read and write. Typically `/tmp`; set `include_workdir: true` to add the driver-resolved working directory. | **Validation constraints:** @@ -76,7 +76,6 @@ filesystem_policy: - /dev/urandom - /etc read_write: - - /sandbox - /tmp - /dev/null ``` diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 6700a22c9f..353feb5cd1 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -433,6 +433,14 @@ declared name or numeric components for both direct and SSH children. When `USER` omits the group, the supervisor uses the user's numeric primary GID. It does not modify `/etc/passwd` or `/etc/group`. +Docker and Podman also inspect OCI `WorkingDir`. An absolute value becomes the +agent workspace; an empty or root (`/`) value falls back to `/sandbox`. +OpenShell creates the resolved directory when needed and transfers ownership of +only that directory to the completed UID/GID. Existing image contents and +nested bind or volume mounts retain their ownership. The resolved workspace is +the cwd and `HOME` for direct and SSH children. Sandbox startup fails before +readiness when the declaration is invalid or the workspace cannot be prepared. + Sandbox creation fails before readiness if a required `USER` component is missing, malformed, unknown, ambiguous, or resolves to UID/GID 0. An image without `USER` therefore works only when policy explicitly provides both @@ -462,7 +470,6 @@ The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/e Docker and Podman custom images do not need a baked-in `"sandbox"` user. Declare a non-root OCI `USER`, or set both process identity fields explicitly in policy. Named image users require matching account entries; a numeric `UID:GID` pair -does not. OpenShell continues to use `/sandbox` as the workspace; it does not -adopt the image's OCI working directory. Until OCI working-directory support is -added, custom images must create `/sandbox` and make it writable by the selected -identity. +does not. Declare an absolute OCI `WORKDIR` to select the workspace. Images with +no working directory, or `WORKDIR /`, use OpenShell's `/sandbox` fallback. +Kubernetes/OpenShift and VM sandboxes continue to use `/sandbox`. diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index f3f36ecc9e..2b07b02886 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -392,10 +392,12 @@ Append the output to `~/.ssh/config` or use `--editor` on `sandbox create`/`sand Upload files from your host into the sandbox: ```shell -openshell sandbox upload my-sandbox ./src /sandbox/src +openshell sandbox upload my-sandbox ./src ``` -When the local path is a named directory, OpenShell preserves that basename at the destination, matching `scp -r` and `cp -r`. The example above creates `/sandbox/src/src`. +When you omit the destination, OpenShell discovers the sandbox's working +directory and uploads there. For a named local directory, OpenShell preserves +the basename, matching `scp -r` and `cp -r`. OpenShell preserves symlinks during upload. A symlink arrives in the sandbox as a symlink with the same target path instead of an expanded copy of the target file or directory. Dangling symlinks are also preserved. @@ -407,7 +409,12 @@ openshell sandbox download my-sandbox /sandbox/output ./local When the sandbox-side source is a single file, the destination follows `cp`-style placement: if the destination already exists as a directory or ends with `/`, the file lands inside it as `/`; otherwise the file is written at the exact destination path. -The CLI only allows sandbox-side sources that resolve inside the writable workspace (`/sandbox`). Paths that escape lexically (`/etc/passwd`, `/sandbox/../etc/passwd`) and paths that escape through a symlink (`/sandbox/etc-link` pointing at `/etc`) are both refused before any data is transferred. +The CLI discovers the sandbox's canonical working directory and only allows +sandbox-side sources that resolve inside it. Paths that escape lexically, such +as `/etc/passwd` or `/sandbox/../etc/passwd`, and paths that escape through a +symlink are refused before any data is transferred. Use the absolute workspace +path reported by `pwd -P` for images that declare a different OCI working +directory. You can also upload files at creation time with the `--upload` flag on diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 8eee13bee5..8fcb6080c8 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -19,8 +19,9 @@ version: 1 # Static: locked at sandbox creation. Paths the agent can read vs read/write. filesystem_policy: + include_workdir: true read_only: [/usr, /lib, /etc] - read_write: [/sandbox, /tmp] + read_write: [/tmp] # Static: Landlock LSM kernel enforcement. best_effort uses highest ABI the host supports. landlock: @@ -100,7 +101,7 @@ See [Supervisor Middleware](/extensibility/supervisor-middleware) for registrati ## Baseline Filesystem Paths -When a sandbox runs in proxy mode (the default), OpenShell automatically adds baseline filesystem paths required for the sandbox child process to function: `/usr`, `/lib`, `/etc`, `/var/log` (read-only) and `/sandbox`, `/tmp` (read-write). Paths like `/app` are included in the baseline set but are only added if they exist in the container image. +When a sandbox runs in proxy mode (the default), OpenShell automatically adds baseline filesystem paths required for the sandbox child process to function: `/usr`, `/lib`, `/etc`, and `/var/log` (read-only), plus `/tmp` and the resolved working directory (read-write). Paths like `/app` are included in the baseline set but are only added if they exist in the container image. For GPU sandboxes, OpenShell also adds existing GPU device nodes as read-write paths. CUDA workloads require write access to procfs for thread metadata, so GPU baseline enrichment moves `/proc` from read-only to read-write when GPU devices are present. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 0ac0d5528f..8bbcc604d4 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -174,7 +174,7 @@ The policy separates filesystem paths into read-only and read-write groups. | Aspect | Detail | |---|---| -| Default | System paths (`/usr`, `/lib`, `/etc`, `/var/log`) are read-only. Working paths (`/sandbox`, `/tmp`) are read-write. `/app` is conditionally included if it exists. | +| Default | System paths (`/usr`, `/lib`, `/etc`, `/var/log`) are read-only. The resolved working directory and `/tmp` are read-write. `/app` is conditionally included if it exists. | | What you can change | Add or remove paths in `filesystem_policy.read_only` and `filesystem_policy.read_write`. | | Risk if relaxed | Making system paths writable lets the agent replace binaries, modify TLS trust stores, or change DNS resolution. Validation rejects broad read-write paths (like `/`). | | Recommendation | Keep system paths read-only. If the agent needs additional writable space, add a specific subdirectory. | diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index 0aeb25038c..3475353041 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -341,6 +341,39 @@ impl SandboxGuard { Ok(combined) } + /// Upload local files to the sandbox's discovered working directory. + /// + /// # Errors + /// + /// Returns an error if the upload command fails. + pub async fn upload_to_workdir(&self, local_path: &str) -> Result { + let mut cmd = openshell_cmd(); + cmd.arg("sandbox") + .arg("upload") + .arg(&self.name) + .arg(local_path) + .arg("--no-git-ignore"); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + + let output = cmd + .output() + .await + .map_err(|e| format!("failed to spawn openshell upload: {e}"))?; + + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let combined = format!("{stdout}{stderr}"); + + if !output.status.success() { + return Err(format!( + "sandbox upload failed (exit {:?}):\n{combined}", + output.status.code() + )); + } + + Ok(combined) + } + /// Upload local files with `.gitignore` filtering (default behavior). /// /// Unlike [`upload`], this does NOT pass `--no-git-ignore`, so the CLI diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index 4cda100dfe..8b814a8ea3 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -10,7 +10,7 @@ //! - The matching container runtime running (for image builds) //! - The `openshell` binary (built automatically from the workspace) -use std::io::Write; +use std::{fs, io::Write}; use openshell_e2e::harness::output::strip_ansi; use openshell_e2e::harness::sandbox::SandboxGuard; @@ -24,6 +24,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ RUN groupadd -g 1235 appstaff && \ useradd -m -u 1234 -g appstaff app +# A custom image may declare a root-owned OCI working directory with existing +# root-owned content. OpenShell prepares only the working directory itself. +WORKDIR /workspace/project +RUN printf root-owned > root-owned.txt + # Write a marker file so we can verify this is our custom image. # Place under /etc (Landlock baseline read-only path) so the sandbox # can read it when filesystem restrictions are properly enforced. @@ -44,7 +49,8 @@ CMD ["sleep", "infinity"] const MARKER: &str = "custom-image-e2e-marker"; -/// Direct and SSH children use the same named OCI identity. +/// A named OCI user can write through direct and SSH children when the image +/// declares a root-owned WorkingDir; existing content retains its ownership. #[tokio::test] async fn sandbox_from_custom_dockerfile() { // Step 1: Write a temporary Dockerfile. @@ -63,7 +69,11 @@ async fn sandbox_from_custom_dockerfile() { &[ "sh", "-c", - "set -eu; id -u; id -g; cat /etc/marker.txt; echo Ready; sleep infinity", + "set -eu; id -u; id -g; test \"$(pwd -P)\" = /workspace/project; \ + test \"$HOME\" = /workspace/project; test \"$(cat root-owned.txt)\" = root-owned; \ + test \"$(stat -c %u:%g .)\" = 1234:1235; \ + test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ + touch direct-oci-user-write; cat /etc/marker.txt; echo Ready; sleep infinity", ], "Ready", ) @@ -85,13 +95,40 @@ async fn sandbox_from_custom_dockerfile() { .exec(&[ "sh", "-c", - "set -eu; test \"$(id -u):$(id -g)\" = 1234:1235; echo ssh-identity-ok", + "set -eu; test \"$(id -u):$(id -g)\" = 1234:1235; \ + test \"$(pwd -P)\" = /workspace/project; test \"$HOME\" = /workspace/project; \ + touch ssh-oci-user-write; echo ssh-write-ok", ]) .await - .expect("SSH child should use OCI identity"); + .expect("SSH child should write to prepared workspace"); assert!( - ssh_output.contains("ssh-identity-ok"), - "expected SSH identity marker:\n{ssh_output}" + ssh_output.contains("ssh-write-ok"), + "expected SSH write marker:\n{ssh_output}" + ); + + let transfer_source = tmpdir.path().join("workspace-transfer.txt"); + fs::write(&transfer_source, "workspace-transfer-ok").expect("write transfer fixture"); + guard + .upload_to_workdir( + transfer_source + .to_str() + .expect("transfer fixture path is UTF-8"), + ) + .await + .expect("upload should default to the OCI workspace"); + let transfer_download = tmpdir.path().join("workspace-transfer-downloaded.txt"); + guard + .download( + "/workspace/project/workspace-transfer.txt", + transfer_download + .to_str() + .expect("download destination path is UTF-8"), + ) + .await + .expect("download should accept the OCI workspace"); + assert_eq!( + fs::read_to_string(transfer_download).expect("read downloaded transfer fixture"), + "workspace-transfer-ok" ); // Explicit cleanup (also happens in Drop, but explicit is clearer in tests). @@ -99,6 +136,7 @@ async fn sandbox_from_custom_dockerfile() { } /// A numeric OCI user/group pair works without passwd or group entries. +/// The image intentionally has no pre-existing `/sandbox`. #[tokio::test] async fn sandbox_from_passwd_less_numeric_oci_user() { let tmpdir = tempfile::tempdir().expect("create tmpdir"); @@ -110,10 +148,17 @@ async fn sandbox_from_passwd_less_numeric_oci_user() { } let dockerfile_str = dockerfile_path.to_str().expect("Dockerfile path is UTF-8"); - let mut guard = - SandboxGuard::create(&["--from", dockerfile_str, "--", "sh", "-c", "id -u; id -g"]) - .await - .expect("sandbox create from numeric OCI Dockerfile"); + let mut guard = SandboxGuard::create(&[ + "--from", + dockerfile_str, + "--", + "sh", + "-c", + "set -eu; id -u; id -g; test \"$(pwd -P)\" = /sandbox; \ + test \"$HOME\" = /sandbox; touch numeric-oci-user-write", + ]) + .await + .expect("sandbox create from numeric OCI Dockerfile"); let clean_output = strip_ansi(&guard.create_output); assert!( diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index ad8cffc2f9..a480997aaa 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -7,6 +7,7 @@ use std::fs; use std::io::Write; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use bollard::Docker; @@ -23,6 +24,20 @@ use serde_json::{Map, Value}; const TEST_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest"; const VOLUME_TARGET: &str = "/sandbox/e2e-volume"; const BIND_TARGET: &str = "/sandbox/e2e-bind"; +#[cfg(feature = "e2e-docker")] +const OCI_VOLUME_TARGET: &str = "/workspace/project/e2e-volume"; +#[cfg(feature = "e2e-docker")] +const OCI_USER_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim + +RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace/project +USER 2234:2235 +CMD ["sleep", "infinity"] +"#; + +static NEXT_VOLUME_ID: AtomicU64 = AtomicU64::new(0); struct VolumeGuard { docker: Docker, @@ -101,6 +116,72 @@ async fn sandbox_mounts_existing_driver_config_volume() { .expect("verify sandbox wrote to named test volume"); } +#[tokio::test] +#[cfg(feature = "e2e-docker")] +async fn oci_workspace_preparation_skips_nested_volume_ownership() { + let driver = e2e_driver().expect("OPENSHELL_E2E_DRIVER must be set by the e2e wrapper"); + assert!( + matches!(driver.as_str(), "docker" | "podman"), + "OCI workspace mount e2e requires docker or podman, got {driver}" + ); + + let volume = VolumeGuard::create(&driver) + .await + .expect("create named test volume"); + seed_volume(&volume).await.expect("seed named test volume"); + + let image_context = tempfile::tempdir().expect("create OCI image context"); + let dockerfile = image_context.path().join("Dockerfile"); + fs::write(&dockerfile, OCI_USER_DOCKERFILE).expect("write OCI image Dockerfile"); + let dockerfile = dockerfile.to_str().expect("Dockerfile path must be UTF-8"); + + let driver_config = format!( + r#"{{"{driver}":{{"mounts":[{{"type":"volume","source":"{}","target":"{OCI_VOLUME_TARGET}","read_only":false}}]}}}}"#, + volume.name + ); + let mut sandbox = SandboxGuard::create_keep_with_args( + &[ + "--from", + dockerfile, + "--driver-config-json", + &driver_config, + "--no-tty", + ], + &[ + "sh", + "-lc", + "set -eu; test \"$(id -u):$(id -g)\" = 2234:2235; \ + test \"$(pwd -P)\" = /workspace/project; test \"$HOME\" = /workspace/project; \ + test \"$(stat -c %u:%g /workspace/project/e2e-volume/input.txt)\" = 0:0; \ + touch direct-write; echo Ready; sleep infinity", + ], + "Ready", + ) + .await + .expect("create OCI-user sandbox with nested volume"); + + let ssh_output = sandbox + .exec(&[ + "sh", + "-lc", + "set -eu; test \"$(pwd -P)\" = /workspace/project; \ + test \"$HOME\" = /workspace/project; \ + test \"$(stat -c %u:%g /workspace/project/e2e-volume/input.txt)\" = 0:0; \ + touch ssh-write; echo nested-mount-owner-ok", + ]) + .await + .expect("SSH child should preserve nested volume ownership"); + assert!( + ssh_output.contains("nested-mount-owner-ok"), + "expected nested mount ownership marker:\n{ssh_output}" + ); + + sandbox.cleanup().await; + verify_volume_ownership(&volume) + .await + .expect("nested volume ownership should remain unchanged"); +} + #[tokio::test] async fn sandbox_mounts_enabled_driver_config_bind() { let driver = e2e_driver().expect("OPENSHELL_E2E_DRIVER must be set by the e2e wrapper"); @@ -208,6 +289,23 @@ async fn verify_volume(volume: &VolumeGuard) -> Result<(), String> { Ok(()) } +#[cfg(feature = "e2e-docker")] +async fn verify_volume_ownership(volume: &VolumeGuard) -> Result<(), String> { + let output = run_volume_container( + volume, + "verify-owner", + true, + "set -eu; test \"$(stat -c %u:%g /vol/input.txt)\" = 0:0; echo owner-ok", + ) + .await?; + if !output.contains("owner-ok") { + return Err(format!( + "volume ownership verification did not print expected marker:\n{output}" + )); + } + Ok(()) +} + async fn run_volume_container( volume: &VolumeGuard, purpose: &str, @@ -413,8 +511,9 @@ fn unique_volume_name(driver: &str) -> String { .duration_since(UNIX_EPOCH) .expect("system clock should be after Unix epoch") .as_nanos(); + let sequence = NEXT_VOLUME_ID.fetch_add(1, Ordering::Relaxed); format!( - "openshell-e2e-driver-config-volume-{driver}-{}-{nanos}", + "openshell-e2e-driver-config-volume-{driver}-{}-{nanos}-{sequence}", std::process::id() ) } From 12a79bcb0d9c01da0ad1b44964d457adaa277a75 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 16:20:12 -0700 Subject: [PATCH 02/16] fix(sandbox): harden OCI workspace handling Signed-off-by: Matthew Grossman --- .agents/skills/openshell-cli/SKILL.md | 4 +- architecture/compute-runtimes.md | 16 +-- crates/openshell-cli/src/ssh.rs | 23 +++- crates/openshell-core/src/driver_mounts.rs | 111 +++++++++++++++++- crates/openshell-driver-docker/README.md | 12 +- crates/openshell-driver-docker/src/lib.rs | 3 + crates/openshell-driver-docker/src/tests.rs | 42 +++++++ crates/openshell-driver-podman/README.md | 10 +- .../openshell-driver-podman/src/container.rs | 96 ++++++++++++--- crates/openshell-driver-podman/src/driver.rs | 13 +- .../src/process.rs | 9 +- docs/reference/sandbox-compute-drivers.mdx | 13 +- e2e/rust/tests/podman_oci_identity.rs | 40 +++++-- 13 files changed, 319 insertions(+), 73 deletions(-) diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 1f635c5b7a..02535521e7 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -223,8 +223,8 @@ openshell sandbox ssh-config my-sandbox >> ~/.ssh/config # Upload local files to the sandbox working directory openshell sandbox upload my-sandbox ./src -# Download files from sandbox -openshell sandbox download my-sandbox /sandbox/output ./local-output +# Download files from sandbox (replace /workspace with the path from `pwd -P`) +openshell sandbox download my-sandbox /workspace/output ./local-output ``` Uploads honor `.gitignore` by default. Add `--no-git-ignore` only when ignored files are intentionally in scope. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 8a90c40bb3..626dbda4a2 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -147,13 +147,15 @@ primary GID. It does not rewrite the account files. Docker and Podman use an absolute OCI working directory as the workspace. An empty or root (`/`) declaration falls back to `/sandbox`, which OpenShell -creates inside the container when needed. Before direct or SSH children start, -the supervisor transfers ownership of only the workspace directory to the -completed UID/GID. Image-provided contents and nested mounts retain their -ownership. The resolved workspace is also the child cwd and `HOME`, the -automatic writable policy path, and Podman's managed workspace-volume mount -target. Kubernetes/OpenShift keep their `/sandbox` PVC and `fsGroup` behavior, -and VM keeps its `/sandbox` guest initialization path. +creates inside the container when needed. Protected system trees and +OpenShell-reserved paths cannot become workspaces. The container runtime starts +the supervisor from `/`; before direct or SSH children start, the supervisor +validates the resolved path and transfers ownership of only the workspace +directory to the completed UID/GID. Image-provided contents and nested mounts +retain their ownership. The resolved workspace is also the child cwd and +`HOME`, the automatic writable policy path, and Podman's managed +workspace-volume mount target. Kubernetes/OpenShift keep their `/sandbox` PVC +and `fsGroup` behavior, and VM keeps its `/sandbox` guest initialization path. Sandbox creation fails before the workload becomes ready when a required image identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index 1f21dfd698..312bb966f1 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -1115,7 +1115,15 @@ async fn ssh_run_capture_stdout(session: &SshSessionConfig, command: &str) -> Re output.status )); } - Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + decode_ssh_probe_stdout(output.stdout) +} + +fn decode_ssh_probe_stdout(stdout: Vec) -> Result { + let stdout = String::from_utf8(stdout) + .map_err(|error| miette::miette!("ssh probe returned non-UTF-8 output: {error}"))?; + let stdout = stdout.strip_suffix('\n').unwrap_or(&stdout); + let stdout = stdout.strip_suffix('\r').unwrap_or(stdout); + Ok(stdout.to_string()) } fn validate_discovered_workspace_root(root: &str) -> Result { @@ -2092,6 +2100,19 @@ mod tests { } } + #[test] + fn ssh_probe_output_only_removes_the_protocol_line_ending() { + assert_eq!( + decode_ssh_probe_stdout(b"/workspace/project \n".to_vec()).unwrap(), + "/workspace/project " + ); + assert_eq!( + decode_ssh_probe_stdout(b"/workspace/project\r\n".to_vec()).unwrap(), + "/workspace/project" + ); + assert!(decode_ssh_probe_stdout(vec![0xff]).is_err()); + } + #[test] fn build_single_file_tar_cmd_inserts_double_dash_before_basename() { // Without `--`, a basename such as `--checkpoint-action=...` would be diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index e1b94f11a1..1957e545a2 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -32,6 +32,46 @@ const RESERVED_MOUNT_TARGETS: &[&str] = &[ "/run/netns", ]; +/// Container filesystem trees that image metadata must never turn into an +/// agent-owned writable workspace. +const PROTECTED_WORKSPACE_TREES: &[&str] = &[ + "/bin", + "/boot", + "/dev", + "/etc", + "/lib", + "/lib64", + "/proc", + "/root", + "/run", + "/sbin", + "/sys", + "/usr/bin", + "/usr/lib", + "/usr/lib64", + "/usr/local/bin", + "/usr/local/lib", + "/usr/local/lib64", + "/usr/local/sbin", + "/usr/local/share", + "/usr/sbin", + "/usr/share", + "/var/log", +]; + +/// Broad container roots that are unsafe as workspaces themselves, while +/// application-specific descendants remain valid. +const PROTECTED_WORKSPACE_ROOTS: &[&str] = &[ + "/home", + "/mnt", + "/opt", + "/srv", + "/usr", + "/usr/local", + "/var", + "/var/lib", +]; + /// Compatibility workspace used when an OCI image has no usable working /// directory and by drivers whose workspace remains fixed. pub const DEFAULT_WORKSPACE_ROOT: &str = "/sandbox"; @@ -139,8 +179,11 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { if working_dir.is_empty() || working_dir == "/" { return Ok(DEFAULT_WORKSPACE_ROOT.to_string()); } - if working_dir.as_bytes().contains(&0) { - return Err("OCI WorkingDir must not contain NUL bytes".to_string()); + if working_dir != working_dir.trim() { + return Err("OCI WorkingDir must not contain surrounding whitespace".to_string()); + } + if working_dir.chars().any(char::is_control) { + return Err("OCI WorkingDir must not contain control characters".to_string()); } if !working_dir.starts_with('/') { return Err(format!( @@ -159,13 +202,32 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { )); } - Ok(working_dir.trim_end_matches('/').to_string()) + let workspace_root = working_dir.trim_end_matches('/').to_string(); + let workspace_path = Path::new(&workspace_root); + if PROTECTED_WORKSPACE_ROOTS + .iter() + .any(|protected| workspace_path == Path::new(protected)) + || PROTECTED_WORKSPACE_TREES + .iter() + .any(|protected| path_is_or_under(workspace_path, Path::new(protected))) + || RESERVED_MOUNT_TARGETS.iter().any(|reserved| { + let reserved = Path::new(reserved); + path_is_or_under(workspace_path, reserved) || path_is_or_under(reserved, workspace_path) + }) + { + return Err(format!( + "OCI WorkingDir '{working_dir}' conflicts with a protected container path" + )); + } + + Ok(workspace_root) } -/// Reject a user-supplied mount that would replace the resolved workspace -/// root. Mounts below the workspace remain valid. +/// Reject a user-supplied mount that would replace or contain the resolved +/// workspace root. Mounts below the workspace remain valid. pub fn validate_workspace_mount_target(target: &str, workspace_root: &str) -> Result<(), String> { - if normalize_mount_target(target) == workspace_root { + let normalized_target = normalize_mount_target(target); + if path_is_or_under(Path::new(workspace_root), Path::new(&normalized_target)) { return Err(format!( "mount target '{target}' is reserved for the OpenShell workspace" )); @@ -202,6 +264,8 @@ mod tests { validate_workspace_mount_target("/sandbox/", "/sandbox").unwrap_err(); validate_workspace_mount_target("/workspace/", "/sandbox").unwrap(); validate_workspace_mount_target("/workspace/cache", "/workspace").unwrap(); + validate_workspace_mount_target("/workspace", "/workspace/project").unwrap_err(); + validate_workspace_mount_target("/workspace-other", "/workspace/project").unwrap(); } #[test] @@ -227,6 +291,8 @@ mod tests { "/workspace/./project", "/workspace//project", "/workspace\0project", + "/workspace ", + "/workspace\nproject", ] { assert!( resolve_oci_workspace_root(invalid).is_err(), @@ -235,6 +301,39 @@ mod tests { } } + #[test] + fn oci_workspace_root_rejects_protected_container_paths() { + for invalid in [ + "/etc", + "/etc/project", + "/usr", + "/usr/bin/project", + "/var", + "/var/log/project", + "/opt", + "/opt/openshell/project", + ] { + assert!( + resolve_oci_workspace_root(invalid).is_err(), + "expected protected workspace '{invalid}' to be rejected" + ); + } + + for valid in [ + "/app", + "/home/app", + "/opt/app", + "/usr/src/app", + "/var/lib/app", + ] { + assert_eq!( + resolve_oci_workspace_root(valid).unwrap(), + valid, + "expected application workspace '{valid}' to remain valid" + ); + } + } + #[test] fn container_target_rejects_reserved_openshell_tls_legacy_path() { let err = validate_container_mount_target("/etc/openshell-tls/client").unwrap_err(); diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 2457faf3f5..d6057710be 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -33,8 +33,9 @@ root (`/`) declaration falls back to `/sandbox`, which OpenShell creates when necessary. The supervisor transfers ownership of only the workspace directory to the completed identity before direct or SSH children start. Existing contents and nested bind or volume mounts retain their ownership. The workspace -is the child cwd and `HOME`; an invalid, symlinked, or unpreparable workspace -fails sandbox startup. +is the child cwd and `HOME`. Protected system trees and OpenShell-reserved +paths cannot become workspaces. The supervisor starts from `/`, then reports an +invalid, symlinked, or unpreparable workspace as a readiness failure. Docker containers join an OpenShell-managed bridge network. The driver injects `host.openshell.internal` and `host.docker.internal` so supervisors have stable @@ -86,9 +87,10 @@ optional `selinux_label` of `shared` (applies `:z`) or `private` (applies `subpath`. User-supplied bind and volume mounts are read-only by default; set `read_only: false` to make them writable. Mount `source`, `target`, and `subpath` values must not contain surrounding whitespace. Mount targets must be -absolute container paths and must not replace the resolved workspace root. -Nested workspace mounts remain valid. Mounts also must not overlap OpenShell -supervisor files, `/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. +absolute container paths and must not replace or contain the resolved workspace +root. Nested workspace mounts remain valid. Mounts also must not overlap +OpenShell supervisor files, `/etc/openshell`, `/etc/openshell-tls`, or +`/run/netns`. Example named-volume usage: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 9ebf618db1..1108786467 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2436,6 +2436,9 @@ fn build_container_create_body_for_image( Ok(ContainerCreateBody { image: Some(image.id.clone()), user: Some("0".to_string()), + // The image workspace may need to be created or rejected by the + // supervisor, so do not let the OCI runtime chdir there first. + working_dir: Some("/".to_string()), env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Clear the image CMD so Docker does not append inherited args to the diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 518c2d8b67..4531c76daf 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -584,6 +584,7 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!(body.image.as_deref(), Some("sha256:immutable")); assert_eq!(body.user.as_deref(), Some("0")); + assert_eq!(body.working_dir.as_deref(), Some("/")); assert_eq!( body.cmd.as_deref(), Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) @@ -614,6 +615,26 @@ fn container_creation_rejects_invalid_oci_working_dir() { assert!(err.message().contains("must be an absolute container path")); } +#[test] +fn container_creation_rejects_protected_oci_working_dir() { + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/etc".to_string(), + }; + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + ) + .unwrap_err(); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("protected container path")); +} + #[test] fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { let metadata = DockerImageMetadata { @@ -638,6 +659,27 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( .contains("reserved for the OpenShell workspace") ); + let ancestor_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/workspace"}] + })) + .unwrap(); + let nested_metadata = DockerImageMetadata { + working_dir: "/workspace/project".to_string(), + ..metadata.clone() + }; + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &ancestor_mount, + None, + &nested_metadata, + ) + .unwrap_err(); + assert!( + err.message() + .contains("reserved for the OpenShell workspace") + ); + let nested_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ "mounts": [{"type": "tmpfs", "target": "/workspace/cache"}] })) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index b5161eae27..a5b7fb2c8c 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -23,8 +23,10 @@ necessary. The supervisor transfers ownership of only the workspace directory to the completed identity before direct or SSH children start. Existing contents and nested bind or volume mounts retain their ownership. The workspace is the child cwd and `HOME`, and the Podman-managed workspace volume is mounted -there so normal volume copy-up preserves image content. An invalid, symlinked, -or unpreparable workspace fails sandbox startup. +there so normal volume copy-up preserves image content. Protected system trees +and OpenShell-reserved paths cannot become workspaces. The supervisor starts +from `/`, then reports an invalid, symlinked, or unpreparable workspace as a +readiness failure. For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). @@ -97,8 +99,8 @@ read-only by default; set `read_only: false` to make them writable. Podman image and volume mounts do not support `subpath` in OpenShell driver config. Mount `source` and `target` values must not contain surrounding whitespace. Mount targets must be absolute container paths and must not replace the -resolved workspace root. Nested workspace mounts remain valid. Mounts also -must not overlap OpenShell supervisor files, `/etc/openshell`, +resolved workspace root or any of its parents. Nested workspace mounts remain +valid. Mounts also must not overlap OpenShell supervisor files, `/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. Example named-volume usage: diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index ba6f48ca48..bcf6cb577b 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -3,6 +3,7 @@ //! Container spec construction for the Podman driver. +use crate::client::ImageInspect; use crate::config::PodmanComputeConfig; use openshell_core::ComputeDriverError; use openshell_core::driver_mounts::SelinuxLabel; @@ -190,6 +191,9 @@ struct ContainerSpec { volumes: Vec, image_volumes: Vec, hostname: String, + /// Start the supervisor independently of the image workspace. The + /// supervisor validates and prepares that workspace before child launch. + work_dir: String, /// Overrides the image's ENTRYPOINT. In Podman's libpod API, `command` /// only overrides CMD (appended as args to the entrypoint). We must set /// `entrypoint` explicitly so the supervisor binary runs directly, @@ -911,9 +915,10 @@ pub fn build_container_spec_with_token_and_gpu_devices( token_secret_name, gpu_device_ids, image, - image, - "", - "", + &ImageInspect { + id: image.to_string(), + config: None, + }, ) } @@ -923,12 +928,18 @@ pub fn build_container_spec_for_image( token_secret_name: Option<&str>, gpu_device_ids: Option<&[String]>, requested_image: &str, - image_id: &str, - oci_user: &str, - oci_working_dir: &str, + inspected_image: &ImageInspect, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); + let oci_user = inspected_image + .config + .as_ref() + .map_or("", |config| config.user.as_str()); + let oci_working_dir = inspected_image + .config + .as_ref() + .map_or("", |config| config.working_dir.as_str()); let workspace_root = driver_mounts::resolve_oci_workspace_root(oci_working_dir) .map_err(ComputeDriverError::Precondition)?; @@ -981,7 +992,7 @@ pub fn build_container_spec_for_image( let container_spec = ContainerSpec { name, - image: image_id.to_string(), + image: inspected_image.id.clone(), labels, env, volumes, @@ -992,6 +1003,7 @@ pub fn build_container_spec_for_image( // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), + work_dir: "/".to_string(), // Override the image's ENTRYPOINT so the supervisor binary runs // directly. Sandbox images (e.g. the community base image) set // ENTRYPOINT ["/bin/bash"], and Podman's `command` field only @@ -1269,6 +1281,7 @@ fn parse_memory_to_bytes(quantity: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::client::ImageConfig; use openshell_core::proto::compute::v1::{GpuResourceRequirements, ResourceRequirements}; static ENV_LOCK: std::sync::LazyLock> = @@ -1288,6 +1301,16 @@ mod tests { } } + fn inspected_image(id: &str, user: &str, working_dir: &str) -> ImageInspect { + ImageInspect { + id: id.to_string(), + config: Some(ImageConfig { + user: user.to_string(), + working_dir: working_dir.to_string(), + }), + } + } + #[test] fn parse_cpu_millicore() { assert_eq!(parse_cpu_to_microseconds("500m"), Some(50_000)); @@ -1392,9 +1415,7 @@ mod tests { None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", - "/workspace/project", + &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), ) .unwrap(); @@ -1409,6 +1430,7 @@ mod tests { container["command"], serde_json::json!(["--workdir", "/workspace/project"]) ); + assert_eq!(container["work_dir"].as_str(), Some("/")); assert!(container["volumes"].as_array().is_some_and(|volumes| { volumes.iter().any(|volume| { volume["name"].as_str() == Some("openshell-sandbox-test-id-workspace") @@ -1438,9 +1460,7 @@ mod tests { None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", - "relative/workspace", + &inspected_image("sha256:immutable", "app:staff", "relative/workspace"), ) .unwrap_err(); @@ -1450,6 +1470,21 @@ mod tests { ); } + #[test] + fn container_spec_rejects_protected_oci_working_dir() { + let err = build_container_spec_for_image( + &test_sandbox("test-id", "test-name"), + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", "/etc"), + ) + .unwrap_err(); + + assert!(err.to_string().contains("protected container path")); + } + #[test] fn container_spec_reserves_resolved_workspace_root_but_allows_nested_mounts() { let mut sandbox = test_sandbox("test-id", "test-name"); @@ -1477,9 +1512,34 @@ mod tests { None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", - "/workspace", + &inspected_image("sha256:immutable", "app:staff", "/workspace"), + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("reserved for the OpenShell workspace") + ); + + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{ + "type": "tmpfs", + "target": "/workspace" + }] + }))); + let err = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), ) .unwrap_err(); assert!( @@ -1506,9 +1566,7 @@ mod tests { None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", - "/workspace", + &inspected_image("sha256:immutable", "app:staff", "/workspace"), ) .expect("nested workspace mounts remain supported"); } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 43c7c68794..b08fe9140a 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -581,15 +581,6 @@ impl PodmanComputeDriver { "podman image '{image}' inspection did not return an immutable image ID" ))); } - let image_user = inspected_image - .config - .as_ref() - .map_or("", |config| config.user.as_str()); - let image_working_dir = inspected_image - .config - .as_ref() - .map_or("", |config| config.working_dir.as_str()); - for image in container::podman_driver_image_mount_sources(sandbox, self.config.enable_bind_mounts) .map_err(ComputeDriverError::Precondition)? @@ -654,9 +645,7 @@ impl PodmanComputeDriver { token_secret_name.as_deref(), gpu_devices.as_deref(), image, - &inspected_image.id, - image_user, - image_working_dir, + &inspected_image, ) { Ok(spec) => spec, Err(e) => { diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 99e82505da..fe5803d124 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -1280,9 +1280,14 @@ fn prepare_oci_workspace_with( gid: Option, do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, ) -> Result<()> { - if !root.is_absolute() || root == Path::new("/") { + let root_str = root + .to_str() + .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; + let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) + .map_err(|error| miette::miette!(error))?; + if Path::new(&validated_root) != root { return Err(miette::miette!( - "workspace path '{}' must be an absolute non-root path", + "workspace path '{}' must be a normalized absolute non-root path", root.display() )); } diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 353feb5cd1..520df0615b 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -240,9 +240,10 @@ Podman mount schema: Podman `volume` and `image` mounts do not support `subpath` in OpenShell driver config, and OpenShell rejects `subpath` for those mount types. OpenShell rejects mount `source` and `target` values with surrounding whitespace. OpenShell also -rejects mount targets that replace the workspace root, container root, supervisor -files, `/etc/openshell`, `/etc/openshell-tls`, authentication material, or -network namespace paths. These checks do not make host bind mounts safe. +rejects mount targets that replace or contain the workspace root, target the +container root or supervisor files, or overlap `/etc/openshell`, +`/etc/openshell-tls`, authentication material, or network namespace paths. +These checks do not make host bind mounts safe. ## MicroVM Driver @@ -438,8 +439,10 @@ agent workspace; an empty or root (`/`) value falls back to `/sandbox`. OpenShell creates the resolved directory when needed and transfers ownership of only that directory to the completed UID/GID. Existing image contents and nested bind or volume mounts retain their ownership. The resolved workspace is -the cwd and `HOME` for direct and SSH children. Sandbox startup fails before -readiness when the declaration is invalid or the workspace cannot be prepared. +the cwd and `HOME` for direct and SSH children. Protected system trees and +OpenShell-reserved paths cannot become workspaces. The supervisor itself starts +from `/`, so a missing or invalid workspace is handled during readiness instead +of preventing the container from starting. Sandbox creation fails before readiness if a required `USER` component is missing, malformed, unknown, ambiguous, or resolves to UID/GID 0. An image diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index e30516bf09..27755addfc 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -3,14 +3,14 @@ #![cfg(feature = "e2e-podman")] -//! Podman-specific E2E coverage for OCI identity inspection and immutable-image -//! launch. +//! Podman-specific E2E coverage for OCI identity/workspace inspection, +//! workspace-volume copy-up, and immutable-image launch. //! //! The test builds an image through the selected Podman engine, creates a -//! sandbox from its mutable tag, and verifies both the child identity and the -//! image ID recorded on the real sandbox container. This exercises the Podman -//! API inspect → protected metadata → create path rather than only its unit -//! serialization boundaries. +//! sandbox from its mutable tag, and verifies the child identity, workspace, +//! copied image content, and image ID recorded on the real sandbox container. +//! This exercises the Podman API inspect → protected metadata → create path +//! rather than only its unit serialization boundaries. use std::process::Stdio; @@ -54,7 +54,13 @@ impl ImageGuard { let containerfile = context.path().join("Containerfile"); std::fs::write( &containerfile, - format!("FROM {BASE_IMAGE}\nUSER {OCI_UID}:{OCI_GID}\n"), + format!( + "FROM {BASE_IMAGE}\n\ + USER 0:0\n\ + WORKDIR /workspace/project\n\ + RUN printf root-owned > root-owned.txt\n\ + USER {OCI_UID}:{OCI_GID}\n" + ), ) .map_err(|err| format!("write Containerfile: {err}"))?; @@ -164,7 +170,7 @@ fn normalized_image_id(image_id: &str) -> &str { } #[tokio::test] -async fn podman_uses_oci_identity_and_inspected_image_id() { +async fn podman_uses_oci_identity_workspace_and_inspected_image_id() { if !is_e2e_driver("podman") { eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); return; @@ -188,7 +194,15 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { &[ "sh", "-c", - "set -eu; printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; echo podman-oci-identity-ready; sleep infinity", + "set -eu; \ + test \"$(pwd -P)\" = /workspace/project; \ + test \"$HOME\" = /workspace/project; \ + test \"$(cat root-owned.txt)\" = root-owned; \ + test \"$(stat -c %u:%g .)\" = 2345:2346; \ + test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ + touch direct-workspace-write; \ + printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ + echo podman-oci-identity-ready; sleep infinity", ], READY_MARKER, ) @@ -205,7 +219,13 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { .exec(&[ "sh", "-c", - "test \"$(id -u):$(id -g)\" = 2345:2346; echo podman-ssh-identity-ok", + "set -eu; \ + test \"$(id -u):$(id -g)\" = 2345:2346; \ + test \"$(pwd -P)\" = /workspace/project; \ + test \"$HOME\" = /workspace/project; \ + test -f direct-workspace-write; \ + touch ssh-workspace-write; \ + echo podman-ssh-identity-ok", ]) .await .expect("SSH child should use Podman OCI identity"); From 8d909493849351c69969fbb50713cd3013a83a05 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 20:35:56 -0700 Subject: [PATCH 03/16] fix(ci): format Podman OCI identity test Signed-off-by: Matthew Grossman --- e2e/rust/tests/podman_oci_identity.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 27755addfc..aff8d44dfe 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -184,13 +184,7 @@ async fn podman_uses_oci_identity_workspace_and_inspected_image_id() { std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); let policy_path = policy.path().to_str().expect("policy path is UTF-8"); let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - "--from", - &image.tag, - "--policy", - policy_path, - "--no-tty", - ], + &["--from", &image.tag, "--policy", policy_path, "--no-tty"], &[ "sh", "-c", From 263a9f472cf1ea85d4630f7e2c69c344bea89e20 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 21:07:04 -0700 Subject: [PATCH 04/16] fix(cli): stay within SSH connection limit Signed-off-by: Matthew Grossman --- crates/openshell-cli/src/ssh.rs | 51 ++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index 312bb966f1..ff48059f71 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -766,8 +766,8 @@ fn local_upload_path_is_file_like(path: &Path) -> bool { /// sandbox. Callers are responsible for splitting the destination path so /// that `dest_dir` is always a directory. /// -/// When `dest_dir` is `None`, the canonical remote workspace is discovered -/// from the SSH session and used as the extraction target. +/// When `dest_dir` is `None`, tar extracts relative to the SSH session's +/// working directory. async fn ssh_tar_upload( server: &str, name: &str, @@ -778,13 +778,7 @@ async fn ssh_tar_upload( ) -> Result<()> { let session = ssh_session_config(server, name, tls, workspace).await?; - let discovered_workspace; - let dest_dir = if let Some(dest_dir) = dest_dir { - dest_dir - } else { - discovered_workspace = discover_workspace_root(&session).await?; - &discovered_workspace - }; + let dest_dir = dest_dir.unwrap_or("."); let escaped_dest = shell_escape(dest_dir); let mut ssh = ssh_base_command(&session.proxy_command); @@ -903,35 +897,49 @@ fn is_under_workspace(path: &str, workspace_root: &str) -> bool { path == workspace_root || path.starts_with(&format!("{workspace_root}/")) } -/// Resolve every symlink in `sandbox_path` on the sandbox side and refuse the -/// result if it lands outside the discovered workspace root. +/// Discover the workspace root and resolve every symlink in `sandbox_path` in +/// one SSH probe, then refuse the result if it lands outside the workspace. /// /// The lexical guard in `validate_sandbox_source_path` cannot see symlinks; a /// workspace path through `etc-link -> /etc` clears the lexical check but /// would still leak `/etc/passwd` once `tar -C` follows the link. Resolving /// symlinks on the remote side and re-validating closes that gap. The returned /// fully-resolved path is what the caller should hand to probe and tar -/// invocations. +/// invocations. Combining discovery and resolution also keeps downloads within +/// the gateway's three-connection limit: this probe, the type probe, and tar. async fn resolve_sandbox_source_path( session: &SshSessionConfig, sandbox_path: &str, - workspace_root: &str, ) -> Result { - let resolve_cmd = format!("realpath -e -- {path}", path = shell_escape(sandbox_path)); - let resolved = ssh_run_capture_stdout(session, &resolve_cmd) + let resolve_cmd = format!( + "pwd -P && realpath -e -- {path}", + path = shell_escape(sandbox_path) + ); + let output = ssh_run_capture_stdout(session, &resolve_cmd) .await .wrap_err_with(|| format!("failed to resolve sandbox source path '{sandbox_path}'"))?; + let (workspace_root, resolved) = output.split_once('\n').ok_or_else(|| { + miette::miette!("unexpected response while resolving sandbox source path '{sandbox_path}'") + })?; + if resolved.contains('\n') { + return Err(miette::miette!( + "unexpected response while resolving sandbox source path '{sandbox_path}'" + )); + } + + let workspace_root = validate_discovered_workspace_root(workspace_root)?; + validate_sandbox_source_path(sandbox_path, &workspace_root)?; if resolved.is_empty() { return Err(miette::miette!( "sandbox source path '{sandbox_path}' does not exist" )); } - if !is_under_workspace(&resolved, workspace_root) { + if !is_under_workspace(resolved, &workspace_root) { return Err(miette::miette!( "sandbox source path '{sandbox_path}' resolves to '{resolved}', outside the sandbox workspace ({workspace_root})" )); } - Ok(resolved) + Ok(resolved.to_string()) } /// Resolve the host-side target path for a downloaded *file*, following @@ -962,7 +970,7 @@ fn resolve_file_download_target( /// /// Files are streamed as a tar archive to `ssh ... tar xf - -C ` on /// the sandbox side. When `dest` is `None`, files are uploaded to the -/// discovered sandbox working directory. +/// SSH session's working directory. #[allow(clippy::too_many_arguments)] pub async fn sandbox_sync_up_files( server: &str, @@ -994,7 +1002,7 @@ pub async fn sandbox_sync_up_files( /// Push a local path (file or directory) into a sandbox using tar-over-SSH. /// -/// When `sandbox_path` is `None`, files are uploaded to the discovered sandbox +/// When `sandbox_path` is `None`, files are uploaded to the SSH session's /// working directory. When uploading a single file to an explicit destination /// that does not end with `/`, the destination is treated as a file path: the /// parent directory is created and the file is written with the destination's @@ -1202,10 +1210,7 @@ pub async fn sandbox_sync_down( workspace: &str, ) -> Result<()> { let session = ssh_session_config(server, name, tls, workspace).await?; - let workspace_root = discover_workspace_root(&session).await?; - let sandbox_path = validate_sandbox_source_path(sandbox_path, &workspace_root)?; - let sandbox_path = - resolve_sandbox_source_path(&session, &sandbox_path, &workspace_root).await?; + let sandbox_path = resolve_sandbox_source_path(&session, sandbox_path).await?; let kind = probe_sandbox_source_kind(&session, &sandbox_path).await?; match kind { From 4005df51874c5e949f02d07f955f882b7092b028 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 21:29:45 -0700 Subject: [PATCH 05/16] fix(sandbox): ensure OCI workspace is writable Signed-off-by: Matthew Grossman --- .../src/process.rs | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index fe5803d124..1d710efe77 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -19,6 +19,8 @@ use std::ffi::CString; use std::os::fd::{AsRawFd, OwnedFd, RawFd}; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; #[cfg(any(test, unix))] use std::path::Path; use std::path::PathBuf; @@ -1328,7 +1330,15 @@ fn prepare_oci_workspace_with( } } - do_chown(root, uid, gid).into_diagnostic() + do_chown(root, uid, gid).into_diagnostic()?; + + let metadata = std::fs::symlink_metadata(root).into_diagnostic()?; + let mode = metadata.permissions().mode() & 0o7777; + if mode & 0o300 != 0o300 { + std::fs::set_permissions(root, std::fs::Permissions::from_mode(mode | 0o300)) + .into_diagnostic()?; + } + Ok(()) } #[cfg(unix)] @@ -2563,6 +2573,24 @@ mod tests { assert!(child.exists(), "image-provided child should be untouched"); } + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_makes_existing_root_owner_writable() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("sandbox"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o555)).unwrap(); + + prepare_oci_workspace_with(&root, None, None, &|_, _, _| Ok(())) + .expect("read-only workspace root should be prepared"); + + let mode = std::fs::symlink_metadata(&root) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o755); + } + #[cfg(unix)] #[test] fn prepare_oci_workspace_rejects_symlink_root() { From bf05429cc0b01012f9df15020ea8d7d72b20445e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 22:19:40 -0700 Subject: [PATCH 06/16] fix(sandbox): harden OCI workspace resolution Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 15 ++-- crates/openshell-core/src/driver_mounts.rs | 52 +++++--------- crates/openshell-driver-docker/README.md | 8 ++- crates/openshell-driver-docker/src/lib.rs | 9 +-- crates/openshell-driver-docker/src/tests.rs | 14 ++++ crates/openshell-driver-podman/README.md | 5 ++ .../src/process.rs | 72 ++++++++++++++++++- docs/reference/sandbox-compute-drivers.mdx | 9 ++- 8 files changed, 129 insertions(+), 55 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 626dbda4a2..a14b6b6a8e 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -148,12 +148,15 @@ primary GID. It does not rewrite the account files. Docker and Podman use an absolute OCI working directory as the workspace. An empty or root (`/`) declaration falls back to `/sandbox`, which OpenShell creates inside the container when needed. Protected system trees and -OpenShell-reserved paths cannot become workspaces. The container runtime starts -the supervisor from `/`; before direct or SSH children start, the supervisor -validates the resolved path and transfers ownership of only the workspace -directory to the completed UID/GID. Image-provided contents and nested mounts -retain their ownership. The resolved workspace is also the child cwd and -`HOME`, the automatic writable policy path, and Podman's managed +OpenShell-reserved paths cannot become workspaces. `/usr` and `/var` are +protected except for the reviewed application conventions `/usr/src/app`, +`/var/app`, `/var/task`, and `/var/www`. The container runtime starts the +supervisor from `/`; before direct or SSH children start, the supervisor +validates that existing parent directories are traversable by the resolved +identity, creates missing parents with mode `0755`, and transfers ownership of +only the workspace directory to the completed UID/GID. Image-provided contents +and nested mounts retain their ownership. The resolved workspace is also the +child cwd and `HOME`, the automatic writable policy path, and Podman's managed workspace-volume mount target. Kubernetes/OpenShift keep their `/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` guest initialization path. diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 1957e545a2..3e320e4470 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -35,42 +35,17 @@ const RESERVED_MOUNT_TARGETS: &[&str] = &[ /// Container filesystem trees that image metadata must never turn into an /// agent-owned writable workspace. const PROTECTED_WORKSPACE_TREES: &[&str] = &[ - "/bin", - "/boot", - "/dev", - "/etc", - "/lib", - "/lib64", - "/proc", - "/root", - "/run", - "/sbin", - "/sys", - "/usr/bin", - "/usr/lib", - "/usr/lib64", - "/usr/local/bin", - "/usr/local/lib", - "/usr/local/lib64", - "/usr/local/sbin", - "/usr/local/share", - "/usr/sbin", - "/usr/share", - "/var/log", + "/bin", "/boot", "/dev", "/etc", "/lib", "/lib32", "/lib64", "/libexec", "/proc", "/root", + "/run", "/sbin", "/sys", "/usr", "/var", ]; +/// Narrow application workspace conventions allowed inside otherwise +/// protected system trees. +const ALLOWED_WORKSPACE_TREES: &[&str] = &["/usr/src/app", "/var/app", "/var/task", "/var/www"]; + /// Broad container roots that are unsafe as workspaces themselves, while /// application-specific descendants remain valid. -const PROTECTED_WORKSPACE_ROOTS: &[&str] = &[ - "/home", - "/mnt", - "/opt", - "/srv", - "/usr", - "/usr/local", - "/var", - "/var/lib", -]; +const PROTECTED_WORKSPACE_ROOTS: &[&str] = &["/home", "/mnt", "/opt", "/srv"]; /// Compatibility workspace used when an OCI image has no usable working /// directory and by drivers whose workspace remains fixed. @@ -207,9 +182,12 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { if PROTECTED_WORKSPACE_ROOTS .iter() .any(|protected| workspace_path == Path::new(protected)) - || PROTECTED_WORKSPACE_TREES + || (PROTECTED_WORKSPACE_TREES .iter() .any(|protected| path_is_or_under(workspace_path, Path::new(protected))) + && !ALLOWED_WORKSPACE_TREES + .iter() + .any(|allowed| path_is_or_under(workspace_path, Path::new(allowed)))) || RESERVED_MOUNT_TARGETS.iter().any(|reserved| { let reserved = Path::new(reserved); path_is_or_under(workspace_path, reserved) || path_is_or_under(reserved, workspace_path) @@ -308,10 +286,14 @@ mod tests { "/etc/project", "/usr", "/usr/bin/project", + "/usr/libexec/project", "/var", "/var/log/project", + "/var/lib/app", + "/var/lib/dpkg", "/opt", "/opt/openshell/project", + "/lib32/project", ] { assert!( resolve_oci_workspace_root(invalid).is_err(), @@ -324,7 +306,9 @@ mod tests { "/home/app", "/opt/app", "/usr/src/app", - "/var/lib/app", + "/var/app/current", + "/var/task", + "/var/www/app", ] { assert_eq!( resolve_oci_workspace_root(valid).unwrap(), diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index d6057710be..840ba84dde 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -34,8 +34,12 @@ necessary. The supervisor transfers ownership of only the workspace directory to the completed identity before direct or SSH children start. Existing contents and nested bind or volume mounts retain their ownership. The workspace is the child cwd and `HOME`. Protected system trees and OpenShell-reserved -paths cannot become workspaces. The supervisor starts from `/`, then reports an -invalid, symlinked, or unpreparable workspace as a readiness failure. +paths cannot become workspaces. OpenShell protects `/usr` and `/var` except for +the application conventions `/usr/src/app`, `/var/app`, `/var/task`, and +`/var/www`. Existing workspace parents must be traversable by the resolved +identity; OpenShell creates missing parents with mode `0755`. The supervisor +starts from `/`, then reports an invalid, symlinked, or unpreparable workspace +as a readiness failure. Docker containers join an OpenShell-managed bridge network. The driver injects `host.openshell.internal` and `host.docker.internal` so supervisors have stable diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1108786467..8f92a8f6bd 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -644,19 +644,13 @@ impl DockerComputeDriver { Self::validate_sandbox_auth(sandbox)?; self.validate_user_volume_mounts_available(&validated.driver_config) .await?; - let gpu_devices = self + let _ = self .resolve_gpu_cdi_devices( validated.gpu_requirements, &validated.driver_config, CdiGpuDefaultSelector::peek_device_ids, ) .await?; - let _ = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - )?; if self .find_managed_container_summary(&sandbox.id, &sandbox.name) @@ -2352,6 +2346,7 @@ fn build_container_create_body( build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) } +#[cfg(test)] fn build_container_create_body_with_gpu_devices( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 4531c76daf..0782cd1898 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -692,6 +692,20 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &metadata, ) .expect("nested workspace mounts remain supported"); + + let compatibility_path_mount: DockerSandboxDriverConfig = + serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/sandbox"}] + })) + .unwrap(); + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &compatibility_path_mount, + None, + &metadata, + ) + .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); } #[test] diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index a5b7fb2c8c..413b435c6a 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -103,6 +103,11 @@ resolved workspace root or any of its parents. Nested workspace mounts remain valid. Mounts also must not overlap OpenShell supervisor files, `/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. +OpenShell protects `/usr` and `/var` as OCI workspaces except for the +application conventions `/usr/src/app`, `/var/app`, `/var/task`, and +`/var/www`. Existing workspace parents must be traversable by the resolved +identity; OpenShell creates missing parents with mode `0755`. + Example named-volume usage: ```shell diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 1d710efe77..1cf1318c58 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -20,7 +20,7 @@ use std::os::fd::{AsRawFd, OwnedFd, RawFd}; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; #[cfg(unix)] -use std::os::unix::fs::PermissionsExt; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; #[cfg(any(test, unix))] use std::path::Path; use std::path::PathBuf; @@ -1306,8 +1306,9 @@ fn prepare_oci_workspace_with( }) .collect::>>()?; + let last_component = components.len().saturating_sub(1); let mut current = PathBuf::from("/"); - for component in components { + for (index, component) in components.into_iter().enumerate() { current.push(component); match std::fs::symlink_metadata(¤t) { Ok(metadata) if metadata.file_type().is_symlink() => { @@ -1322,9 +1323,18 @@ fn prepare_oci_workspace_with( current.display() )); } - Ok(_) => {} + Ok(metadata) => { + if index != last_component && !identity_can_traverse(&metadata, uid, gid) { + return Err(miette::miette!( + "workspace parent '{}' is not traversable by the sandbox identity", + current.display() + )); + } + } Err(error) if error.kind() == std::io::ErrorKind::NotFound => { std::fs::create_dir(¤t).into_diagnostic()?; + std::fs::set_permissions(¤t, std::fs::Permissions::from_mode(0o755)) + .into_diagnostic()?; } Err(error) => return Err(error).into_diagnostic(), } @@ -1341,6 +1351,24 @@ fn prepare_oci_workspace_with( Ok(()) } +#[cfg(unix)] +fn identity_can_traverse(metadata: &std::fs::Metadata, uid: Option, gid: Option) -> bool { + let user_id = uid.unwrap_or_else(nix::unistd::geteuid).as_raw(); + if user_id == 0 { + return true; + } + + let group_id = gid.unwrap_or_else(nix::unistd::getegid).as_raw(); + let mode = metadata.permissions().mode(); + if metadata.uid() == user_id { + mode & 0o100 != 0 + } else if metadata.gid() == group_id { + mode & 0o010 != 0 + } else { + mode & 0o001 != 0 + } +} + #[cfg(unix)] fn chown_children( dir: &Path, @@ -2658,6 +2686,36 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_rejects_inaccessible_existing_parent() { + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); + let metadata = std::fs::symlink_metadata(&parent).unwrap(); + let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); + let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); + let root = parent.join("project"); + + let error = prepare_oci_workspace_with( + &root, + Some(different_user), + Some(different_group), + &|_, _, _| Ok(()), + ) + .unwrap_err(); + + assert!( + error.to_string().contains("is not traversable"), + "unexpected error: {error}" + ); + assert!( + !root.exists(), + "workspace must not be created below an inaccessible parent" + ); + } + #[cfg(unix)] #[test] fn prepare_oci_workspace_rejects_non_directory_root() { @@ -2730,6 +2788,14 @@ mod tests { .expect("missing OCI workspace should be created"); assert!(missing.is_dir()); + assert_eq!( + std::fs::symlink_metadata(missing.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o755 + ); assert_eq!(*chowned.lock().unwrap(), vec![missing]); } diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 520df0615b..5edf888d8f 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -440,9 +440,12 @@ OpenShell creates the resolved directory when needed and transfers ownership of only that directory to the completed UID/GID. Existing image contents and nested bind or volume mounts retain their ownership. The resolved workspace is the cwd and `HOME` for direct and SSH children. Protected system trees and -OpenShell-reserved paths cannot become workspaces. The supervisor itself starts -from `/`, so a missing or invalid workspace is handled during readiness instead -of preventing the container from starting. +OpenShell-reserved paths cannot become workspaces. OpenShell protects `/usr` +and `/var` except for `/usr/src/app`, `/var/app`, `/var/task`, `/var/www`, and +their descendants. Existing workspace parents must be traversable by the +resolved identity; OpenShell creates missing parents with mode `0755`. The +supervisor itself starts from `/`, so a missing or invalid workspace is handled +during readiness instead of preventing the container from starting. Sandbox creation fails before readiness if a required `USER` component is missing, malformed, unknown, ambiguous, or resolves to UID/GID 0. An image From 7cd2bbc692e76363d6359e5cd9986dc98463272a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 28 Jul 2026 23:11:16 -0700 Subject: [PATCH 07/16] fix(sandbox): honor supplementary workspace groups Signed-off-by: Matthew Grossman --- architecture/compute-runtimes.md | 3 +- .../src/process.rs | 132 +++++++++++++++++- docs/sandboxes/policies.mdx | 2 +- 3 files changed, 128 insertions(+), 9 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index a14b6b6a8e..4aa1d0e854 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -156,7 +156,8 @@ validates that existing parent directories are traversable by the resolved identity, creates missing parents with mode `0755`, and transfers ownership of only the workspace directory to the completed UID/GID. Image-provided contents and nested mounts retain their ownership. The resolved workspace is also the -child cwd and `HOME`, the automatic writable policy path, and Podman's managed +child cwd and `HOME`; when `filesystem.include_workdir` is enabled, it becomes +the automatic writable policy path. Podman also uses it as the managed workspace-volume mount target. Kubernetes/OpenShift keep their `/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` guest initialization path. diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 1cf1318c58..ea67b13c3b 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -1267,8 +1267,13 @@ fn chown_sandbox_home(root: &Path, uid: Option, gid: Option) -> Result } #[cfg(unix)] -fn prepare_oci_workspace(root: &Path, uid: Option, gid: Option) -> Result<()> { - prepare_oci_workspace_with(root, uid, gid, &nix::unistd::chown) +fn prepare_oci_workspace( + root: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> Result<()> { + prepare_oci_workspace_with(root, uid, gid, supplementary_gids, &nix::unistd::chown) } /// Prepare only the resolved `OpenShell` workspace directory itself. @@ -1280,6 +1285,7 @@ fn prepare_oci_workspace_with( root: &Path, uid: Option, gid: Option, + supplementary_gids: &[Gid], do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, ) -> Result<()> { let root_str = root @@ -1324,7 +1330,9 @@ fn prepare_oci_workspace_with( )); } Ok(metadata) => { - if index != last_component && !identity_can_traverse(&metadata, uid, gid) { + if index != last_component + && !identity_can_traverse(&metadata, uid, gid, supplementary_gids) + { return Err(miette::miette!( "workspace parent '{}' is not traversable by the sandbox identity", current.display() @@ -1352,7 +1360,12 @@ fn prepare_oci_workspace_with( } #[cfg(unix)] -fn identity_can_traverse(metadata: &std::fs::Metadata, uid: Option, gid: Option) -> bool { +fn identity_can_traverse( + metadata: &std::fs::Metadata, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> bool { let user_id = uid.unwrap_or_else(nix::unistd::geteuid).as_raw(); if user_id == 0 { return true; @@ -1362,13 +1375,46 @@ fn identity_can_traverse(metadata: &std::fs::Metadata, uid: Option, gid: Op let mode = metadata.permissions().mode(); if metadata.uid() == user_id { mode & 0o100 != 0 - } else if metadata.gid() == group_id { + } else if metadata.gid() == group_id + || supplementary_gids + .iter() + .any(|supplementary_gid| supplementary_gid.as_raw() == metadata.gid()) + { mode & 0o010 != 0 } else { mode & 0o001 != 0 } } +#[cfg(not(any( + target_os = "aix", + target_os = "haiku", + target_os = "illumos", + target_os = "ios", + target_os = "macos", + target_os = "redox", + target_os = "solaris" +)))] +fn named_user_supplementary_groups(user_name: &str, primary_gid: Gid) -> Result> { + let user_name = CString::new(user_name).map_err(|_| miette::miette!("Invalid user name"))?; + nix::unistd::getgrouplist(user_name.as_c_str(), primary_gid).into_diagnostic() +} + +#[cfg(any( + target_os = "aix", + target_os = "haiku", + target_os = "illumos", + target_os = "ios", + target_os = "macos", + target_os = "redox", + target_os = "solaris" +))] +#[allow(clippy::unnecessary_wraps)] +fn named_user_supplementary_groups(_user_name: &str, _primary_gid: Gid) -> Result> { + // Privilege dropping does not call initgroups on these targets. + Ok(Vec::new()) +} + #[cfg(unix)] fn chown_children( dir: &Path, @@ -1483,6 +1529,23 @@ pub fn prepare_filesystem_with_identity( }, }; + let supplementary_gids = match user_name { + Some(name) if name.parse::().is_err() && resolved_identity.uid().is_none() => { + let primary_gid = if let Some(gid) = gid { + gid + } else { + let uid = + uid.ok_or_else(|| miette::miette!("Failed to resolve sandbox user '{name}'"))?; + User::from_uid(uid) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("Failed to resolve user from UID {uid}"))? + .gid + }; + named_user_supplementary_groups(name, primary_gid)? + } + _ => Vec::new(), + }; + // Docker and Podman own workspace resolution and must make the selected // root usable by the final effective identity, including when both policy // identity fields were explicit. Validate it before processing any @@ -1494,7 +1557,7 @@ pub fn prepare_filesystem_with_identity( })?; let workspace = Path::new(workspace); info!(path = %workspace.display(), ?uid, ?gid, "Preparing resolved workspace"); - prepare_oci_workspace(workspace, uid, gid)?; + prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; } // Create missing read_write paths and only chown the ones we created. @@ -2593,6 +2656,7 @@ mod tests { &root, Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], &fake_chown, ) .expect("workspace root should be prepared"); @@ -2609,7 +2673,7 @@ mod tests { std::fs::create_dir(&root).unwrap(); std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o555)).unwrap(); - prepare_oci_workspace_with(&root, None, None, &|_, _, _| Ok(())) + prepare_oci_workspace_with(&root, None, None, &[], &|_, _, _| Ok(())) .expect("read-only workspace root should be prepared"); let mode = std::fs::symlink_metadata(&root) @@ -2635,6 +2699,7 @@ mod tests { &link, Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], ) .unwrap_err(); assert!( @@ -2659,6 +2724,7 @@ mod tests { &parent_link.join("workspace"), Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], ) .unwrap_err(); assert!( @@ -2678,6 +2744,7 @@ mod tests { Path::new("/tmp/workspace/../escape"), Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], ) .unwrap_err(); assert!( @@ -2702,6 +2769,7 @@ mod tests { &root, Some(different_user), Some(different_group), + &[], &|_, _, _| Ok(()), ) .unwrap_err(); @@ -2716,6 +2784,53 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn prepare_oci_workspace_accepts_supplementary_group_parent() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o710)).unwrap(); + let parent = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o710)).unwrap(); + let metadata = std::fs::symlink_metadata(&parent).unwrap(); + let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); + let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); + let supplementary_group = Gid::from_raw(metadata.gid()); + let root = parent.join("project"); + + prepare_oci_workspace_with( + &root, + Some(different_user), + Some(different_group), + &[supplementary_group], + &|_, _, _| Ok(()), + ) + .expect("supplementary group execute permission should allow traversal"); + + assert!(root.is_dir()); + } + + #[cfg(not(any( + target_os = "aix", + target_os = "haiku", + target_os = "illumos", + target_os = "ios", + target_os = "macos", + target_os = "redox", + target_os = "solaris" + )))] + #[test] + fn named_user_supplementary_groups_include_primary_group() { + let user = User::from_uid(nix::unistd::geteuid()) + .expect("resolve current UID") + .expect("current user exists"); + + let groups = named_user_supplementary_groups(&user.name, user.gid) + .expect("resolve named-user supplementary groups"); + + assert!(groups.contains(&user.gid)); + } + #[cfg(unix)] #[test] fn prepare_oci_workspace_rejects_non_directory_root() { @@ -2727,6 +2842,7 @@ mod tests { &root, Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], ) .unwrap_err(); assert!( @@ -2749,6 +2865,7 @@ mod tests { &root, Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], &fake_chown, ) .unwrap_err(); @@ -2783,6 +2900,7 @@ mod tests { &missing, Some(nix::unistd::geteuid()), Some(nix::unistd::getegid()), + &[], &fake_chown, ) .expect("missing OCI workspace should be created"); diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 8fcb6080c8..a735c7082d 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -101,7 +101,7 @@ See [Supervisor Middleware](/extensibility/supervisor-middleware) for registrati ## Baseline Filesystem Paths -When a sandbox runs in proxy mode (the default), OpenShell automatically adds baseline filesystem paths required for the sandbox child process to function: `/usr`, `/lib`, `/etc`, and `/var/log` (read-only), plus `/tmp` and the resolved working directory (read-write). Paths like `/app` are included in the baseline set but are only added if they exist in the container image. +When a sandbox runs in proxy mode (the default), OpenShell automatically adds baseline filesystem paths required for the sandbox child process to function: `/usr`, `/lib`, `/etc`, and `/var/log` (read-only), plus `/tmp` (read-write). When `filesystem.include_workdir` is `true`, OpenShell also adds the resolved working directory as read-write. Paths like `/app` are included in the baseline set but are only added if they exist in the container image. For GPU sandboxes, OpenShell also adds existing GPU device nodes as read-write paths. CUDA workloads require write access to procfs for thread metadata, so GPU baseline enrichment moves `/proc` from read-only to read-write when GPU devices are present. From 0af4badfd437830521dade6f36439b12da9bb514 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 09:48:30 -0700 Subject: [PATCH 08/16] fix(cli): address workspace review feedback Signed-off-by: Matthew Grossman --- .agents/skills/openshell-cli/SKILL.md | 10 +- .agents/skills/openshell-cli/cli-reference.md | 4 +- crates/openshell-cli/src/ssh.rs | 97 ++++++++----------- crates/openshell-core/src/driver_mounts.rs | 3 + crates/openshell-driver-docker/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/driver.rs | 3 +- crates/openshell-sandbox/src/lib.rs | 12 +-- .../src/process.rs | 16 +-- .../openshell-supervisor-process/src/run.rs | 10 +- .../openshell-supervisor-process/src/ssh.rs | 32 +++--- docs/sandboxes/manage-sandboxes.mdx | 12 ++- e2e/rust/tests/custom_image.rs | 4 +- 12 files changed, 98 insertions(+), 109 deletions(-) diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 02535521e7..a0eb2f5283 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -223,8 +223,8 @@ openshell sandbox ssh-config my-sandbox >> ~/.ssh/config # Upload local files to the sandbox working directory openshell sandbox upload my-sandbox ./src -# Download files from sandbox (replace /workspace with the path from `pwd -P`) -openshell sandbox download my-sandbox /workspace/output ./local-output +# Download a path relative to the sandbox working directory +openshell sandbox download my-sandbox output ./local-output ``` Uploads honor `.gitignore` by default. Add `--no-git-ignore` only when ignored files are intentionally in scope. @@ -232,8 +232,10 @@ Uploads honor `.gitignore` by default. Add `--no-git-ignore` only when ignored f Uploads preserve symlinks, including dangling symlinks, instead of dereferencing their targets. A symlink source bypasses Git-aware filtering so the link itself is archived. When the upload destination is omitted, the CLI discovers the remote working -directory. Downloads must use an absolute sandbox path within that same -canonical working directory. +directory. Uploading a named directory merges it into an existing directory of +the same name, overwriting matching entries without deleting unrelated entries. +Downloads accept paths relative to that working directory or absolute paths +within it. ### Execute a non-interactive command diff --git a/.agents/skills/openshell-cli/cli-reference.md b/.agents/skills/openshell-cli/cli-reference.md index 1673d80aeb..aee5b30f55 100644 --- a/.agents/skills/openshell-cli/cli-reference.md +++ b/.agents/skills/openshell-cli/cli-reference.md @@ -258,11 +258,11 @@ Open an interactive SSH shell. The name defaults to the last-used sandbox. `--ed ### `openshell sandbox upload [dest]` -Upload files using tar-over-SSH. The CLI discovers the canonical remote working directory when the destination is omitted. `.gitignore` filtering is enabled unless `--no-git-ignore` is passed. +Upload files using tar-over-SSH. The CLI discovers the canonical remote working directory when the destination is omitted. A named directory merges into an existing directory of the same name, overwriting matching entries without deleting unrelated entries. `.gitignore` filtering is enabled unless `--no-git-ignore` is passed. ### `openshell sandbox download [dest]` -Download files using tar-over-SSH. The sandbox source must be an absolute path within the canonical remote working directory. The local destination defaults to `.`. +Download files using tar-over-SSH. The sandbox source may be relative to the canonical remote working directory or an absolute path within it. The local destination defaults to `.`. ### `openshell sandbox ssh-config [name]` diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index ff48059f71..2b0a813d4d 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -7,7 +7,6 @@ use crate::tls::{TlsOptions, grpc_client}; use miette::{IntoDiagnostic, Result, WrapErr}; #[cfg(unix)] use nix::sys::signal::{SaFlags, SigAction, SigHandler, SigSet, Signal, sigaction}; -use openshell_core::ObjectId; use openshell_core::forward::{ ForwardSpec, build_proxy_command, format_gateway_url, resolve_ssh_gateway, shell_escape, validate_ssh_session_response, write_forward_pid, @@ -16,6 +15,7 @@ use openshell_core::proto::{ CreateSshSessionRequest, GetSandboxRequest, SshRelayTarget, TcpForwardFrame, TcpForwardInit, tcp_forward_init, }; +use openshell_core::{ObjectId, driver_mounts}; use owo_colors::OwoColorize; use std::fs; use std::future::Future; @@ -867,24 +867,29 @@ fn lexical_clean_absolute_path(path: &str) -> Option { Some(out) } -/// Validate that a sandbox-side source path passed to `sandbox download` -/// resolves under the sandbox writable root. +/// Resolve a sandbox-side source path passed to `sandbox download` under the +/// sandbox writable root. /// /// Returns the cleaned, traversal-resolved path on success. Refuses any -/// path that lexically escapes the discovered workspace root with a -/// user-facing error. +/// path that lexically escapes the discovered workspace root with a user-facing +/// error. Relative paths are interpreted from the workspace root. /// /// This is a lexical guard only — it does not follow symlinks. Call /// `resolve_sandbox_source_path` after this on any path that will be passed /// to a subsequent SSH I/O operation, so a workspace symlink to `/etc` cannot /// leak files outside the workspace. -fn validate_sandbox_source_path(path: &str, workspace_root: &str) -> Result { +fn validate_sandbox_source_path(workspace_root: &str, path: &str) -> Result { if path.is_empty() { return Err(miette::miette!("sandbox source path is empty")); } - let cleaned = lexical_clean_absolute_path(path) - .ok_or_else(|| miette::miette!("sandbox source path must be absolute (got '{path}')"))?; - if !is_under_workspace(&cleaned, workspace_root) { + let candidate = if path.starts_with('/') { + path.to_string() + } else { + format!("{workspace_root}/{path}") + }; + let cleaned = lexical_clean_absolute_path(&candidate) + .ok_or_else(|| miette::miette!("sandbox source path is invalid (got '{path}')"))?; + if !driver_mounts::path_is_or_under(Path::new(&cleaned), Path::new(workspace_root)) { return Err(miette::miette!( "sandbox source path '{path}' is outside the sandbox workspace ({workspace_root})" )); @@ -892,11 +897,6 @@ fn validate_sandbox_source_path(path: &str, workspace_root: &str) -> Result bool { - path == workspace_root || path.starts_with(&format!("{workspace_root}/")) -} - /// Discover the workspace root and resolve every symlink in `sandbox_path` in /// one SSH probe, then refuse the result if it lands outside the workspace. /// @@ -928,13 +928,13 @@ async fn resolve_sandbox_source_path( } let workspace_root = validate_discovered_workspace_root(workspace_root)?; - validate_sandbox_source_path(sandbox_path, &workspace_root)?; + validate_sandbox_source_path(&workspace_root, sandbox_path)?; if resolved.is_empty() { return Err(miette::miette!( "sandbox source path '{sandbox_path}' does not exist" )); } - if !is_under_workspace(resolved, &workspace_root) { + if !driver_mounts::path_is_or_under(Path::new(resolved), Path::new(&workspace_root)) { return Err(miette::miette!( "sandbox source path '{sandbox_path}' resolves to '{resolved}', outside the sandbox workspace ({workspace_root})" )); @@ -1199,8 +1199,8 @@ async fn probe_sandbox_source_kind( /// behaviour for the directory-source case. /// /// The sandbox source path is also subjected to a workspace-boundary check -/// before any SSH command is issued; paths that lexically resolve outside -/// the discovered workspace root are refused. +/// before any file probe or archive command is issued; paths that resolve +/// outside the discovered workspace root are refused. pub async fn sandbox_sync_down( server: &str, name: &str, @@ -2006,42 +2006,50 @@ mod tests { fn validate_sandbox_source_path_accepts_workspace_paths() { let workspace_root = "/workspace/project"; assert_eq!( - validate_sandbox_source_path("/workspace/project/file.txt", workspace_root).unwrap(), + validate_sandbox_source_path(workspace_root, "/workspace/project/file.txt").unwrap(), "/workspace/project/file.txt" ); assert_eq!( validate_sandbox_source_path( - "/workspace/project/.agent/workspace/hello.txt", - workspace_root + workspace_root, + "/workspace/project/.agent/workspace/hello.txt" ) .unwrap(), "/workspace/project/.agent/workspace/hello.txt" ); assert_eq!( - validate_sandbox_source_path("/workspace/project", workspace_root).unwrap(), + validate_sandbox_source_path(workspace_root, "/workspace/project").unwrap(), "/workspace/project" ); assert_eq!( - validate_sandbox_source_path("/workspace/project/", workspace_root).unwrap(), + validate_sandbox_source_path(workspace_root, "/workspace/project/").unwrap(), "/workspace/project" ); assert_eq!( - validate_sandbox_source_path("/workspace/project/sub/../file", workspace_root).unwrap(), + validate_sandbox_source_path(workspace_root, "/workspace/project/sub/../file").unwrap(), "/workspace/project/file" ); + assert_eq!( + validate_sandbox_source_path(workspace_root, "output/file.txt").unwrap(), + "/workspace/project/output/file.txt" + ); + assert_eq!( + validate_sandbox_source_path(workspace_root, "./output/../file.txt").unwrap(), + "/workspace/project/file.txt" + ); } #[test] fn validate_sandbox_source_path_rejects_traversal_and_escapes() { let workspace_root = "/workspace/project"; - let traversal = validate_sandbox_source_path("/etc/passwd", workspace_root).unwrap_err(); + let traversal = validate_sandbox_source_path(workspace_root, "/etc/passwd").unwrap_err(); assert!( format!("{traversal}").contains("outside the sandbox workspace"), "unexpected error: {traversal}" ); let parent_escape = - validate_sandbox_source_path("/workspace/project/../../etc/passwd", workspace_root) + validate_sandbox_source_path(workspace_root, "/workspace/project/../../etc/passwd") .unwrap_err(); assert!( format!("{parent_escape}").contains("outside the sandbox workspace"), @@ -2049,46 +2057,19 @@ mod tests { ); let prefix_only = - validate_sandbox_source_path("/workspace/projected/secrets", workspace_root) + validate_sandbox_source_path(workspace_root, "/workspace/projected/secrets") .unwrap_err(); assert!( format!("{prefix_only}").contains("outside the sandbox workspace"), "unexpected error: {prefix_only}" ); - let empty = validate_sandbox_source_path("", workspace_root).unwrap_err(); + let empty = validate_sandbox_source_path(workspace_root, "").unwrap_err(); assert!(format!("{empty}").contains("empty")); - let relative = - validate_sandbox_source_path("workspace/project/file", workspace_root).unwrap_err(); - assert!(format!("{relative}").contains("must be absolute")); - } - - #[test] - fn is_under_workspace_accepts_root_and_descendants() { - assert!(is_under_workspace( - "/workspace/project", - "/workspace/project" - )); - assert!(is_under_workspace( - "/workspace/project/file", - "/workspace/project" - )); - assert!(is_under_workspace( - "/workspace/project/sub/nested", - "/workspace/project" - )); - } - - #[test] - fn is_under_workspace_rejects_outside_paths_and_prefix_collisions() { - assert!(!is_under_workspace("/etc/passwd", "/workspace/project")); - assert!(!is_under_workspace( - "/workspace/projected/secrets", - "/workspace/project" - )); - assert!(!is_under_workspace("/", "/workspace/project")); - assert!(!is_under_workspace("", "/workspace/project")); + let relative_escape = + validate_sandbox_source_path(workspace_root, "../../etc/passwd").unwrap_err(); + assert!(format!("{relative_escape}").contains("outside the sandbox workspace")); } #[test] diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 3e320e4470..8756ba291d 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -97,6 +97,9 @@ pub fn validate_mount_subpath(subpath: &str) -> Result<(), String> { } /// Validate a container-side mount target for user-supplied driver mounts. +/// +/// Workspace collisions depend on the inspected image's resolved working +/// directory and are checked separately by `validate_workspace_mount_target`. pub fn validate_container_mount_target(target: &str) -> Result<(), String> { if target.is_empty() { return Err("mount target must not be empty".to_string()); diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 8f92a8f6bd..0f3bdee4c3 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2436,8 +2436,8 @@ fn build_container_create_body_for_image( working_dir: Some("/".to_string()), env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Clear the image CMD so Docker does not append inherited args to the - // supervisor entrypoint. + // Replace the image CMD with the supervisor's resolved workspace + // argument so Docker cannot append inherited image arguments. cmd: Some(vec!["--workdir".to_string(), workspace_root]), labels: Some(labels), host_config: Some(HostConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index ccb71290b7..56361ef59c 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -4066,7 +4066,8 @@ mod tests { "init container must not depend on a shell" ); - // Agent container command should be overridden to the emptyDir path + // `--workdir` is optional for standalone supervisor invocations and + // has no implicit default, so Kubernetes must pass its fixed workspace. let command = pod_template["spec"]["containers"][0]["command"] .as_array() .expect("command should be set"); diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 1585a3010f..250ea8c3f7 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -189,9 +189,9 @@ pub async fn run_sandbox( // OpenShift retain their authoritative numeric pair; Docker and Podman // fill only omitted policy fields from OCI Config.User. #[cfg(unix)] - let (resolved_process_identity, workspace_home) = { + let (resolved_process_identity, workdir_is_home) = { let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; - let workspace_home = matches!( + let workdir_is_home = matches!( &driver_identity, openshell_supervisor_process::identity::DriverIdentity::OciUser { .. } ); @@ -199,10 +199,10 @@ pub async fn run_sandbox( &mut policy, &driver_identity, )?; - (resolved, workspace_home) + (resolved, workdir_is_home) }; #[cfg(not(unix))] - let (resolved_process_identity, workspace_home) = ( + let (resolved_process_identity, workdir_is_home) = ( openshell_supervisor_process::process::ResolvedProcessIdentity::default(), false, ); @@ -691,7 +691,7 @@ pub async fn run_sandbox( program, args, workdir.as_deref(), - workspace_home, + workdir_is_home, timeout_secs, interactive, sandbox_id.as_deref(), @@ -1510,7 +1510,7 @@ mod baseline_tests { } #[test] - fn baseline_read_write_uses_dynamic_workdir_instead_of_sandbox() { + fn baseline_read_write_does_not_hardcode_sandbox() { let (_ro, rw) = baseline_enrichment_paths(); assert!(rw.contains(&"/tmp".to_string())); assert!(!rw.contains(&"/sandbox".to_string())); diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index ea67b13c3b..2e5713f186 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -530,7 +530,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, - workspace_home: bool, + workdir_is_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -543,7 +543,7 @@ impl ProcessHandle { program, args, workdir, - workspace_home, + workdir_is_home, interactive, policy, resolved_identity, @@ -565,7 +565,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, - workspace_home: bool, + workdir_is_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -577,7 +577,7 @@ impl ProcessHandle { program, args, workdir, - workspace_home, + workdir_is_home, interactive, policy, resolved_identity, @@ -593,7 +593,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, - workspace_home: bool, + workdir_is_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -620,7 +620,7 @@ impl ProcessHandle { if let Some(dir) = workdir { cmd.current_dir(dir); - if workspace_home { + if workdir_is_home { cmd.env("HOME", dir); } } @@ -752,7 +752,7 @@ impl ProcessHandle { program: &str, args: &[String], workdir: Option<&str>, - workspace_home: bool, + workdir_is_home: bool, interactive: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, @@ -776,7 +776,7 @@ impl ProcessHandle { if let Some(dir) = workdir { cmd.current_dir(dir); - if workspace_home { + if workdir_is_home { cmd.env("HOME", dir); } } diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index 8d95eef344..c3ee7ca254 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -54,7 +54,7 @@ pub async fn run_process( program: &str, args: &[String], workdir: Option<&str>, - workspace_home: bool, + workdir_is_home: bool, timeout_secs: u64, interactive: bool, sandbox_id: Option<&str>, @@ -100,7 +100,7 @@ pub async fn run_process( policy, resolved_process_identity, workdir, - workspace_home, + workdir_is_home, )?; } @@ -250,7 +250,7 @@ pub async fn run_process( ssh_ready_tx, policy_clone, workdir_clone, - workspace_home, + workdir_is_home, netns_fd, proxy_url, ca_paths, @@ -327,7 +327,7 @@ pub async fn run_process( program, args, workdir, - workspace_home, + workdir_is_home, interactive, policy, resolved_process_identity, @@ -342,7 +342,7 @@ pub async fn run_process( program, args, workdir, - workspace_home, + workdir_is_home, interactive, policy, resolved_process_identity, diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 73a583a1ff..9d890f2f20 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -112,7 +112,7 @@ pub async fn run_ssh_server( ready_tx: tokio::sync::oneshot::Sender>, policy: SandboxPolicy, workdir: Option, - workspace_home: bool, + workdir_is_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, @@ -158,7 +158,7 @@ pub async fn run_ssh_server( config, policy, workdir, - workspace_home, + workdir_is_home, netns_fd, proxy_url, ca_paths, @@ -188,7 +188,7 @@ async fn handle_connection( config: Arc, policy: SandboxPolicy, workdir: Option, - workspace_home: bool, + workdir_is_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, @@ -214,7 +214,7 @@ async fn handle_connection( let handler = SshHandler::new( policy, workdir, - workspace_home, + workdir_is_home, netns_fd, proxy_url, ca_file_paths, @@ -245,7 +245,7 @@ struct ChannelState { struct SshHandler { policy: SandboxPolicy, workdir: Option, - workspace_home: bool, + workdir_is_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, @@ -261,7 +261,7 @@ impl SshHandler { fn new( policy: SandboxPolicy, workdir: Option, - workspace_home: bool, + workdir_is_home: bool, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, @@ -273,7 +273,7 @@ impl SshHandler { Self { policy, workdir, - workspace_home, + workdir_is_home, netns_fd, proxy_url, ca_file_paths, @@ -496,7 +496,7 @@ impl russh::server::Handler for SshHandler { let input_sender = spawn_pipe_exec( &self.policy, self.workdir.clone(), - self.workspace_home, + self.workdir_is_home, Some("/usr/lib/openssh/sftp-server".to_string()), session.handle(), channel, @@ -594,7 +594,7 @@ impl SshHandler { let (pty_master, input_sender) = spawn_pty_shell( &self.policy, self.workdir.clone(), - self.workspace_home, + self.workdir_is_home, command, &pty, handle, @@ -616,7 +616,7 @@ impl SshHandler { let input_sender = spawn_pipe_exec( &self.policy, self.workdir.clone(), - self.workspace_home, + self.workdir_is_home, command, handle, channel, @@ -712,7 +712,7 @@ impl Default for PtyRequest { /// For numeric UIDs, there is no passwd entry, so the default remains /// `("{uid}", "/sandbox")`. Docker and Podman replace that default with their /// resolved image workspace. -fn session_user_and_home(policy: &SandboxPolicy, workspace_home: Option<&str>) -> (String, String) { +fn session_user_and_home(policy: &SandboxPolicy, workdir_home: Option<&str>) -> (String, String) { let (user, default_home) = match policy.process.run_as_user.as_deref() { Some(user) if !user.is_empty() => { // Numeric UID — no passwd entry expected; use default HOME. @@ -732,7 +732,7 @@ fn session_user_and_home(policy: &SandboxPolicy, workspace_home: Option<&str>) - } _ => ("sandbox".to_string(), "/sandbox".to_string()), }; - let home = workspace_home.map_or(default_home, str::to_string); + let home = workdir_home.map_or(default_home, str::to_string); (user, home) } @@ -787,7 +787,7 @@ fn apply_child_env( fn spawn_pty_shell( policy: &SandboxPolicy, workdir: Option, - workspace_home: bool, + workdir_is_home: bool, command: Option, pty: &PtyRequest, handle: Handle, @@ -840,7 +840,7 @@ fn spawn_pty_shell( // falling back to "sandbox" / "/sandbox" for backward compatibility. let (session_user, session_home) = session_user_and_home( policy, - if workspace_home { + if workdir_is_home { workdir.as_deref() } else { None @@ -968,7 +968,7 @@ fn spawn_pty_shell( fn spawn_pipe_exec( policy: &SandboxPolicy, workdir: Option, - workspace_home: bool, + workdir_is_home: bool, command: Option, handle: Handle, channel: ChannelId, @@ -1002,7 +1002,7 @@ fn spawn_pipe_exec( let (session_user, session_home) = session_user_and_home( policy, - if workspace_home { + if workdir_is_home { workdir.as_deref() } else { None diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index 2b07b02886..7fcdd2e92b 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -397,14 +397,16 @@ openshell sandbox upload my-sandbox ./src When you omit the destination, OpenShell discovers the sandbox's working directory and uploads there. For a named local directory, OpenShell preserves -the basename, matching `scp -r` and `cp -r`. +the basename, matching `scp -r` and `cp -r`. If that directory already exists, +the upload merges into it and overwrites matching entries without deleting +unrelated entries. OpenShell preserves symlinks during upload. A symlink arrives in the sandbox as a symlink with the same target path instead of an expanded copy of the target file or directory. Dangling symlinks are also preserved. Download files from the sandbox to your host: ```shell -openshell sandbox download my-sandbox /sandbox/output ./local +openshell sandbox download my-sandbox output ./local ``` When the sandbox-side source is a single file, the destination follows `cp`-style placement: if the destination already exists as a directory or ends with `/`, the file lands inside it as `/`; otherwise the file is written at the exact destination path. @@ -412,9 +414,9 @@ When the sandbox-side source is a single file, the destination follows `cp`-styl The CLI discovers the sandbox's canonical working directory and only allows sandbox-side sources that resolve inside it. Paths that escape lexically, such as `/etc/passwd` or `/sandbox/../etc/passwd`, and paths that escape through a -symlink are refused before any data is transferred. Use the absolute workspace -path reported by `pwd -P` for images that declare a different OCI working -directory. +symlink are refused before any data is transferred. Relative sources are +resolved from the working directory; absolute sources within the same canonical +directory are also accepted. You can also upload files at creation time with the `--upload` flag on diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index 8b814a8ea3..eed4e29de9 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -119,13 +119,13 @@ async fn sandbox_from_custom_dockerfile() { let transfer_download = tmpdir.path().join("workspace-transfer-downloaded.txt"); guard .download( - "/workspace/project/workspace-transfer.txt", + "workspace-transfer.txt", transfer_download .to_str() .expect("download destination path is UTF-8"), ) .await - .expect("download should accept the OCI workspace"); + .expect("download should resolve relative to the OCI workspace"); assert_eq!( fs::read_to_string(transfer_download).expect("read downloaded transfer fixture"), "workspace-transfer-ok" From ee853985c3821554c6dcf99f12592cdfa334ad52 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 11:45:27 -0700 Subject: [PATCH 09/16] fix(podman): serialize image volume lifecycle Signed-off-by: Matthew Grossman --- crates/openshell-driver-podman/src/driver.rs | 55 +++++++++++++------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index b08fe9140a..4154c85041 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -42,6 +42,10 @@ pub struct PodmanComputeDriver { /// The host's IP on the bridge network. Sandbox containers use this to /// reach the gateway server when no explicit gRPC endpoint is configured. network_gateway_ip: Option, + /// Serializes container image-volume attachment and removal. Rootless + /// Podman can otherwise detach a shared type=image mount while another + /// sandbox container is starting. + container_lifecycle_lock: Arc>, gpu_selector: Arc, gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, } @@ -368,6 +372,7 @@ impl PodmanComputeDriver { client, config, network_gateway_ip, + container_lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -653,8 +658,28 @@ impl PodmanComputeDriver { return Err(e); } }; - match self.client.create_container(&spec).await { - Ok(_) => {} + // Podman implements the supervisor as a shared type=image mount. + // Keep attach/start atomic with respect to another sandbox's removal. + let lifecycle_result = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + match self.client.create_container(&spec).await { + Ok(_) => match self.client.start_container(&name).await { + Ok(()) => Ok(()), + Err(e) => { + warn!( + sandbox_name = %sandbox.name, + error = %e, + "Failed to start container; cleaning up" + ); + let _ = self.client.remove_container(&name).await; + Err(e) + } + }, + Err(e) => Err(e), + } + }; + match lifecycle_result { + Ok(()) => {} Err(PodmanApiError::Conflict(_)) => { // Clean up the volume we just created. It is keyed by *this* // sandbox's ID, not the conflicting container's ID (which @@ -669,18 +694,6 @@ impl PodmanComputeDriver { } } - // 5. Start container. - if let Err(e) = self.client.start_container(&name).await { - warn!( - sandbox_name = %sandbox.name, - error = %e, - "Failed to start container; cleaning up" - ); - let _ = self.client.remove_container(&name).await; - cleanup_created().await; - return Err(ComputeDriverError::from(e)); - } - info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -748,10 +761,15 @@ impl PodmanComputeDriver { .stop_container(&container_id, self.config.stop_timeout_secs) .await; - let container_existed = match self.client.remove_container(&container_id).await { - Ok(()) => true, - Err(PodmanApiError::NotFound(_)) => false, - Err(e) => return Err(ComputeDriverError::from(e)), + // Removing a container detaches its type=image mounts. Do not overlap + // that operation with another sandbox container's create/start window. + let container_existed = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + match self.client.remove_container(&container_id).await { + Ok(()) => true, + Err(PodmanApiError::NotFound(_)) => false, + Err(e) => return Err(ComputeDriverError::from(e)), + } }; // Remove workspace volume. @@ -886,6 +904,7 @@ impl PodmanComputeDriver { client, config, network_gateway_ip: None, + container_lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, From d6ea9d52ab9a57f07a6f513a5c7dd95ce91adaa9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 11:47:48 -0700 Subject: [PATCH 10/16] test(e2e): cover OCI volumes with Podman Signed-off-by: Matthew Grossman --- e2e/rust/tests/driver_config_volume.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index a480997aaa..d762becdc7 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -24,9 +24,9 @@ use serde_json::{Map, Value}; const TEST_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest"; const VOLUME_TARGET: &str = "/sandbox/e2e-volume"; const BIND_TARGET: &str = "/sandbox/e2e-bind"; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] const OCI_VOLUME_TARGET: &str = "/workspace/project/e2e-volume"; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] const OCI_USER_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ @@ -117,7 +117,7 @@ async fn sandbox_mounts_existing_driver_config_volume() { } #[tokio::test] -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] async fn oci_workspace_preparation_skips_nested_volume_ownership() { let driver = e2e_driver().expect("OPENSHELL_E2E_DRIVER must be set by the e2e wrapper"); assert!( @@ -289,7 +289,7 @@ async fn verify_volume(volume: &VolumeGuard) -> Result<(), String> { Ok(()) } -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] async fn verify_volume_ownership(volume: &VolumeGuard) -> Result<(), String> { let output = run_volume_container( volume, From 12b0bd6d25b6246cbec861fc95ed36ac13a5082e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 12:10:13 -0700 Subject: [PATCH 11/16] test(e2e): build OCI fixture with active engine Signed-off-by: Matthew Grossman --- e2e/rust/tests/driver_config_volume.rs | 58 ++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index d762becdc7..fd63571a0a 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -7,6 +7,7 @@ use std::fs; use std::io::Write; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +use std::process::Stdio; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -17,7 +18,7 @@ use bollard::query_parameters::{ RemoveVolumeOptionsBuilder, StartContainerOptions, WaitContainerOptions, }; use futures_util::TryStreamExt; -use openshell_e2e::harness::container::e2e_driver; +use openshell_e2e::harness::container::{ContainerEngine, e2e_driver}; use openshell_e2e::harness::sandbox::SandboxGuard; use serde_json::{Map, Value}; @@ -44,6 +45,56 @@ struct VolumeGuard { name: String, } +struct ImageGuard { + engine: ContainerEngine, + tag: String, +} + +impl ImageGuard { + fn build(driver: &str, dockerfile: &Path, context: &Path) -> Result { + let engine = ContainerEngine::from_env()?; + let tag = format!("localhost/{}-oci-user:latest", unique_volume_name(driver)); + let output = engine + .command() + .args([ + "build", + "--file", + dockerfile + .to_str() + .ok_or_else(|| "Dockerfile path must be UTF-8".to_string())?, + "--tag", + &tag, + context + .to_str() + .ok_or_else(|| "image context path must be UTF-8".to_string())?, + ]) + .output() + .map_err(|err| format!("run {} build: {err}", engine.name()))?; + if !output.status.success() { + return Err(format!( + "{} build failed (exit {:?}):\n{}{}", + engine.name(), + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(Self { engine, tag }) + } +} + +impl Drop for ImageGuard { + fn drop(&mut self) { + let _ = self + .engine + .command() + .args(["image", "rm", "--force", &self.tag]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } +} + impl VolumeGuard { async fn create(driver: &str) -> Result { let name = unique_volume_name(driver); @@ -133,7 +184,8 @@ async fn oci_workspace_preparation_skips_nested_volume_ownership() { let image_context = tempfile::tempdir().expect("create OCI image context"); let dockerfile = image_context.path().join("Dockerfile"); fs::write(&dockerfile, OCI_USER_DOCKERFILE).expect("write OCI image Dockerfile"); - let dockerfile = dockerfile.to_str().expect("Dockerfile path must be UTF-8"); + let image = ImageGuard::build(&driver, &dockerfile, image_context.path()) + .expect("build OCI-user image with selected container engine"); let driver_config = format!( r#"{{"{driver}":{{"mounts":[{{"type":"volume","source":"{}","target":"{OCI_VOLUME_TARGET}","read_only":false}}]}}}}"#, @@ -142,7 +194,7 @@ async fn oci_workspace_preparation_skips_nested_volume_ownership() { let mut sandbox = SandboxGuard::create_keep_with_args( &[ "--from", - dockerfile, + &image.tag, "--driver-config-json", &driver_config, "--no-tty", From 6a1a875912e6ae62670b91a49e35255bff9872c5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 12:41:45 -0700 Subject: [PATCH 12/16] test(e2e): cover upload directory merge Signed-off-by: Matthew Grossman --- e2e/rust/tests/custom_image.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index eed4e29de9..5494d1078b 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -131,6 +131,38 @@ async fn sandbox_from_custom_dockerfile() { "workspace-transfer-ok" ); + guard + .exec(&[ + "sh", + "-c", + "set -eu; mkdir -p merge-upload; \ + printf remote-conflict > merge-upload/conflict.txt; \ + printf remote-preserved > merge-upload/unrelated.txt", + ]) + .await + .expect("seed existing remote upload directory"); + let merge_source = tmpdir.path().join("merge-upload"); + fs::create_dir(&merge_source).expect("create local upload directory"); + fs::write(merge_source.join("conflict.txt"), "local-conflict") + .expect("write conflicting local upload file"); + fs::write(merge_source.join("added.txt"), "local-added") + .expect("write added local upload file"); + guard + .upload_to_workdir(merge_source.to_str().expect("merge upload path is UTF-8")) + .await + .expect("upload should merge into the existing remote directory"); + guard + .exec(&[ + "sh", + "-c", + "set -eu; \ + test \"$(cat merge-upload/conflict.txt)\" = local-conflict; \ + test \"$(cat merge-upload/added.txt)\" = local-added; \ + test \"$(cat merge-upload/unrelated.txt)\" = remote-preserved", + ]) + .await + .expect("upload should overwrite conflicts and preserve unrelated remote files"); + // Explicit cleanup (also happens in Drop, but explicit is clearer in tests). guard.cleanup().await; } From a672d3c7c883341a183f0b97022771b6cf0a904d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 13:24:18 -0700 Subject: [PATCH 13/16] fix(sandbox): preserve OCI workspace authority Signed-off-by: Matthew Grossman --- .../skills/debug-openshell-cluster/SKILL.md | 4 + architecture/compute-runtimes.md | 29 +- crates/openshell-core/src/driver_mounts.rs | 114 ++--- crates/openshell-core/src/sandbox_env.rs | 9 + crates/openshell-driver-docker/README.md | 22 +- crates/openshell-driver-docker/src/lib.rs | 12 + crates/openshell-driver-docker/src/tests.rs | 8 +- crates/openshell-driver-podman/README.md | 32 +- .../openshell-driver-podman/src/container.rs | 174 ++++++- crates/openshell-driver-podman/src/driver.rs | 164 +++++++ crates/openshell-prover/src/lib.rs | 31 +- crates/openshell-prover/src/model.rs | 2 +- crates/openshell-prover/src/policy.rs | 16 +- crates/openshell-sandbox/src/lib.rs | 21 + crates/openshell-sandbox/src/main.rs | 80 ++++ crates/openshell-server/src/compute/mod.rs | 48 +- .../src/process.rs | 433 +++++++++++++++--- docs/reference/sandbox-compute-drivers.mdx | 44 +- e2e/rust/tests/custom_image.rs | 49 +- e2e/rust/tests/driver_config_volume.rs | 1 + e2e/rust/tests/podman_oci_identity.rs | 2 +- 21 files changed, 1101 insertions(+), 194 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 0de85fb0a4..581670670b 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -149,6 +149,8 @@ Common findings: - Gateway process stopped: inspect exit status and logs. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. +- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, or supplementary groups must already be able to traverse every parent and write and enter the directory. OpenShell does not create, chown, or chmod a non-default image workdir. +- A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify `OPENSHELL_DOCKER_SUPERVISOR_BIN`, the sibling binary next to `openshell-gateway`, or the configured supervisor image contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and supervisor callback endpoint. - Supervisor image exits before printing `openshell-sandbox --version`: the image should be the scratch supervisor image from `deploy/docker/Dockerfile.supervisor` and must contain a static executable at `/openshell-sandbox`. @@ -175,6 +177,8 @@ Common findings: - Rootless networking unavailable: inspect Podman network configuration. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. +- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, or supplementary groups must already be able to traverse every parent and write and enter the directory. Podman checks the original image in a networkless temporary probe before attaching the workspace volume, so inspect the probe failure in gateway logs. +- If Podman reports probe cleanup or timeout failures, inspect temporary containers with `podman ps -a --filter name=openshell-workspace-probe` and gateway logs. The driver force-removes the probe on every normal success or failure path. - Supervisor cannot call back: check callback endpoint and gateway logs. ### Step 6: Check Kubernetes Helm Gateways diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 74ddce5961..abc13d4598 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -200,20 +200,21 @@ declaration omits the group, the supervisor fills it with the user's numeric primary GID. It does not rewrite the account files. Docker and Podman use an absolute OCI working directory as the workspace. An -empty or root (`/`) declaration falls back to `/sandbox`, which OpenShell -creates inside the container when needed. Protected system trees and -OpenShell-reserved paths cannot become workspaces. `/usr` and `/var` are -protected except for the reviewed application conventions `/usr/src/app`, -`/var/app`, `/var/task`, and `/var/www`. The container runtime starts the -supervisor from `/`; before direct or SSH children start, the supervisor -validates that existing parent directories are traversable by the resolved -identity, creates missing parents with mode `0755`, and transfers ownership of -only the workspace directory to the completed UID/GID. Image-provided contents -and nested mounts retain their ownership. The resolved workspace is also the -child cwd and `HOME`; when `filesystem.include_workdir` is enabled, it becomes -the automatic writable policy path. Podman also uses it as the managed -workspace-volume mount target. Kubernetes/OpenShift keep their `/sandbox` PVC -and `fsGroup` behavior, and VM keeps its `/sandbox` guest initialization path. +empty, root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which +OpenShell creates and owns as a compatibility workspace. Any other workdir must already +exist in the immutable image without symlink components. The completed +identity, including supplementary groups, must already be able to traverse +every parent and write and enter the workdir; OpenShell does not change that +directory's ownership or mode. Filesystem metadata identifies kernel-managed +mounts, while collision checks are derived from actual OpenShell control paths. +Docker performs the check in the final container before workload launch. +Podman performs it in a minimal networkless container from the same pinned +image ID before its managed workspace volume covers the path, then binds the +validated process identity to the final supervisor startup. The resolved +workspace is the child cwd and `HOME`; when `filesystem.include_workdir` is +enabled, it becomes the automatic writable policy path. Kubernetes/OpenShift +keep their `/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` +guest initialization path. Sandbox creation fails before the workload becomes ready when a required image identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index 8756ba291d..196a9bba36 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -5,6 +5,11 @@ use std::path::Path; +use crate::driver_utils::{ + SANDBOX_TOKEN_MOUNT_PATH, SUPERVISOR_CONTAINER_DIR, TLS_CA_MOUNT_PATH, TLS_CERT_MOUNT_PATH, + TLS_KEY_MOUNT_PATH, UPSTREAM_PROXY_AUTH_MOUNT_PATH, +}; + /// `SELinux` relabelling mode for bind mounts. /// /// On hosts with `SELinux` enabled (e.g. Fedora, RHEL) a bind-mounted path @@ -25,28 +30,24 @@ pub enum SelinuxLabel { Private, } -const RESERVED_MOUNT_TARGETS: &[&str] = &[ - "/opt/openshell", - "/etc/openshell", - "/etc/openshell-tls", +/// Paths mounted or created by the local container supervisor. +/// +/// Keep this list tied to concrete `OpenShell` control resources. Workspace +/// safety itself is enforced from the image's ownership and mode metadata, +/// not by guessing which distro paths are sensitive. +const OPENSHELL_CONTROL_PATHS: &[&str] = &[ + SUPERVISOR_CONTAINER_DIR, + TLS_CA_MOUNT_PATH, + TLS_CERT_MOUNT_PATH, + TLS_KEY_MOUNT_PATH, + SANDBOX_TOKEN_MOUNT_PATH, + UPSTREAM_PROXY_AUTH_MOUNT_PATH, + // Kubernetes mounts the sandbox client certificate secret here. + "/etc/openshell-tls/client", + "/etc/openshell-tls/proxy", "/run/netns", ]; -/// Container filesystem trees that image metadata must never turn into an -/// agent-owned writable workspace. -const PROTECTED_WORKSPACE_TREES: &[&str] = &[ - "/bin", "/boot", "/dev", "/etc", "/lib", "/lib32", "/lib64", "/libexec", "/proc", "/root", - "/run", "/sbin", "/sys", "/usr", "/var", -]; - -/// Narrow application workspace conventions allowed inside otherwise -/// protected system trees. -const ALLOWED_WORKSPACE_TREES: &[&str] = &["/usr/src/app", "/var/app", "/var/task", "/var/www"]; - -/// Broad container roots that are unsafe as workspaces themselves, while -/// application-specific descendants remain valid. -const PROTECTED_WORKSPACE_ROOTS: &[&str] = &["/home", "/mnt", "/opt", "/srv"]; - /// Compatibility workspace used when an OCI image has no usable working /// directory and by drivers whose workspace remains fixed. pub const DEFAULT_WORKSPACE_ROOT: &str = "/sandbox"; @@ -136,10 +137,12 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { { return Err("mount target must not contain '..'".to_string()); } - for reserved in RESERVED_MOUNT_TARGETS { - if path_is_or_under(path, Path::new(reserved)) { + for reserved in OPENSHELL_CONTROL_PATHS { + let reserved = Path::new(reserved); + if paths_overlap(path, reserved) { return Err(format!( - "mount target '{target}' conflicts with reserved OpenShell path '{reserved}'" + "mount target '{target}' conflicts with reserved OpenShell path '{}'", + reserved.display() )); } } @@ -181,29 +184,29 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result { } let workspace_root = working_dir.trim_end_matches('/').to_string(); - let workspace_path = Path::new(&workspace_root); - if PROTECTED_WORKSPACE_ROOTS - .iter() - .any(|protected| workspace_path == Path::new(protected)) - || (PROTECTED_WORKSPACE_TREES - .iter() - .any(|protected| path_is_or_under(workspace_path, Path::new(protected))) - && !ALLOWED_WORKSPACE_TREES - .iter() - .any(|allowed| path_is_or_under(workspace_path, Path::new(allowed)))) - || RESERVED_MOUNT_TARGETS.iter().any(|reserved| { - let reserved = Path::new(reserved); - path_is_or_under(workspace_path, reserved) || path_is_or_under(reserved, workspace_path) - }) - { - return Err(format!( - "OCI WorkingDir '{working_dir}' conflicts with a protected container path" - )); + for control_path in OPENSHELL_CONTROL_PATHS { + validate_workspace_control_path(&workspace_root, control_path)?; } Ok(workspace_root) } +/// Reject a workspace that contains or is contained by an `OpenShell` control +/// path. Drivers use this for runtime-configured paths such as the SSH socket. +pub fn validate_workspace_control_path( + workspace_root: &str, + control_path: &str, +) -> Result<(), String> { + let workspace = Path::new(workspace_root); + let control = Path::new(control_path); + if paths_overlap(workspace, control) { + return Err(format!( + "OCI WorkingDir '{workspace_root}' conflicts with OpenShell control path '{control_path}'" + )); + } + Ok(()) +} + /// Reject a user-supplied mount that would replace or contain the resolved /// workspace root. Mounts below the workspace remain valid. pub fn validate_workspace_mount_target(target: &str, workspace_root: &str) -> Result<(), String> { @@ -229,6 +232,10 @@ pub fn path_is_or_under(path: &Path, parent: &Path) -> bool { path == parent || path.starts_with(parent) } +fn paths_overlap(left: &Path, right: &Path) -> bool { + path_is_or_under(left, right) || path_is_or_under(right, left) +} + #[cfg(test)] mod tests { use super::*; @@ -283,32 +290,31 @@ mod tests { } #[test] - fn oci_workspace_root_rejects_protected_container_paths() { + fn oci_workspace_root_rejects_only_openshell_control_path_collisions() { for invalid in [ "/etc", - "/etc/project", - "/usr", - "/usr/bin/project", - "/usr/libexec/project", - "/var", - "/var/log/project", - "/var/lib/app", - "/var/lib/dpkg", "/opt", - "/opt/openshell/project", - "/lib32/project", + "/opt/openshell", + "/opt/openshell/bin/project", + "/etc/openshell/tls/client", + "/etc/openshell/auth", + "/run", + "/run/netns/project", ] { assert!( resolve_oci_workspace_root(invalid).is_err(), - "expected protected workspace '{invalid}' to be rejected" + "expected control-path workspace '{invalid}' to be rejected" ); } for valid in [ "/app", + "/etc/project", "/home/app", "/opt/app", + "/usr/bin/project", "/usr/src/app", + "/var/lib/app", "/var/app/current", "/var/task", "/var/www/app", @@ -323,9 +329,9 @@ mod tests { #[test] fn container_target_rejects_reserved_openshell_tls_legacy_path() { - let err = validate_container_mount_target("/etc/openshell-tls/client").unwrap_err(); + let err = validate_container_mount_target("/etc/openshell-tls/proxy/client").unwrap_err(); - assert!(err.contains("/etc/openshell-tls")); + assert!(err.contains("/etc/openshell-tls/proxy")); } #[test] diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 1549258fa3..ac58d92fbd 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -126,6 +126,15 @@ pub const SANDBOX_GID: &str = "OPENSHELL_SANDBOX_GID"; /// OCI only for the former contract. pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER"; +/// Driver assertion that the image's original workdir was validated before a +/// runtime-managed workspace volume covered it. Set only by the Podman driver. +pub const OCI_WORKSPACE_PREVALIDATED: &str = "OPENSHELL_OCI_WORKSPACE_PREVALIDATED"; + +/// Gateway-projected process policy identity used by local drivers during +/// immutable-image workspace validation. +pub const POLICY_RUN_AS_USER: &str = "OPENSHELL_POLICY_RUN_AS_USER"; +pub const POLICY_RUN_AS_GROUP: &str = "OPENSHELL_POLICY_RUN_AS_GROUP"; + // The corporate upstream-proxy configuration deliberately has no reserved // environment variables: it travels on the supervisor's argv // (`--upstream-proxy` and friends), which a sandbox image cannot forge the diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 840ba84dde..0f659e3bee 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -28,18 +28,16 @@ remain names after validation; a missing group is filled with the user's numeric primary GID. Explicit `process.run_as_user` and `process.run_as_group` values take precedence independently. -An absolute OCI working directory becomes the agent workspace. An empty or -root (`/`) declaration falls back to `/sandbox`, which OpenShell creates when -necessary. The supervisor transfers ownership of only the workspace directory -to the completed identity before direct or SSH children start. Existing -contents and nested bind or volume mounts retain their ownership. The workspace -is the child cwd and `HOME`. Protected system trees and OpenShell-reserved -paths cannot become workspaces. OpenShell protects `/usr` and `/var` except for -the application conventions `/usr/src/app`, `/var/app`, `/var/task`, and -`/var/www`. Existing workspace parents must be traversable by the resolved -identity; OpenShell creates missing parents with mode `0755`. The supervisor -starts from `/`, then reports an invalid, symlinked, or unpreparable workspace -as a readiness failure. +An absolute OCI working directory becomes the agent workspace. An empty, +root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell +creates when necessary and owns as a compatibility workspace. Any other image workdir must +already exist without symlink components. The completed identity, including +supplementary groups, must already be able to traverse every parent and +write and enter the workdir. OpenShell does not change its ownership or mode. +Kernel-managed filesystems are detected from filesystem metadata, and paths +that overlap concrete OpenShell control resources are rejected. The workspace +is the child cwd and `HOME`. The supervisor starts from `/`, then reports an +invalid workdir as a readiness failure. Docker containers join an OpenShell-managed bridge network. The driver injects `host.openshell.internal` and `host.docker.internal` so supervisors have stable diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 7686bea94f..45fde043a2 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2180,6 +2180,13 @@ fn build_environment_for_oci_user( user_env.extend(template.environment.clone()); } user_env.extend(spec.environment.clone()); + for key in [ + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::POLICY_RUN_AS_USER, + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, + ] { + user_env.remove(key); + } environment.extend(user_env.clone()); if !user_env.is_empty() && let Ok(json) = serde_json::to_string(&user_env) @@ -2235,6 +2242,9 @@ fn build_environment_for_oci_user( environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + environment.remove(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); + environment.remove(openshell_core::sandbox_env::POLICY_RUN_AS_USER); + environment.remove(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP); environment.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), oci_user.to_string(), @@ -2369,6 +2379,8 @@ fn build_container_create_body_for_image( let resource_limits = docker_resource_limits(template)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) .map_err(Status::failed_precondition)?; + driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) + .map_err(Status::failed_precondition)?; for mount in &driver_config.mounts { let target = match mount { DockerDriverMountConfig::Bind { target, .. } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 0172f43278..536c9b193c 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -607,11 +607,11 @@ fn container_creation_rejects_invalid_oci_working_dir() { } #[test] -fn container_creation_rejects_protected_oci_working_dir() { +fn container_creation_rejects_openshell_control_path_working_dir() { let metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), - working_dir: "/etc".to_string(), + working_dir: "/opt/openshell/bin/project".to_string(), }; let err = build_container_create_body_for_image( &test_sandbox(), @@ -623,7 +623,7 @@ fn container_creation_rejects_protected_oci_working_dir() { .unwrap_err(); assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("protected container path")); + assert!(err.message().contains("OpenShell control path")); } #[test] @@ -1234,7 +1234,7 @@ fn driver_config_rejects_reserved_mount_targets() { "mounts": [{ "type": "volume", "source": "work-nfs", - "target": "/etc/openshell/auth/custom" + "target": "/etc/openshell/auth" }] }))); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 413b435c6a..6a255028e6 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -17,16 +17,18 @@ Named OCI components remain names after validation; a missing group is filled with the user's numeric primary GID. Explicit `process.run_as_user` and `process.run_as_group` values take precedence independently. -An absolute OCI working directory becomes the agent workspace. An empty or -root (`/`) declaration falls back to `/sandbox`, which OpenShell creates when -necessary. The supervisor transfers ownership of only the workspace directory -to the completed identity before direct or SSH children start. Existing -contents and nested bind or volume mounts retain their ownership. The workspace -is the child cwd and `HOME`, and the Podman-managed workspace volume is mounted -there so normal volume copy-up preserves image content. Protected system trees -and OpenShell-reserved paths cannot become workspaces. The supervisor starts -from `/`, then reports an invalid, symlinked, or unpreparable workspace as a -readiness failure. +An absolute OCI working directory becomes the agent workspace. An empty, +root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell +creates when necessary and owns as a compatibility workspace. For any other workdir, Podman +first starts a minimal, networkless probe from the pinned image ID without the +workspace volume, tokens, or TLS secrets. The probe requires a real directory +with no symlink components and verifies that the completed identity, including +supplementary groups, can already traverse every parent and write and enter the +workdir. It also rejects kernel-managed filesystems and overlaps with concrete +OpenShell control resources. Only then does Podman mount and prepare the +managed workspace volume at that path; normal copy-up preserves image content. +The final supervisor verifies that its policy identity matches the probed +identity. The workspace is the child cwd and `HOME`. For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). @@ -100,13 +102,9 @@ image and volume mounts do not support `subpath` in OpenShell driver config. Mount `source` and `target` values must not contain surrounding whitespace. Mount targets must be absolute container paths and must not replace the resolved workspace root or any of its parents. Nested workspace mounts remain -valid. Mounts also must not overlap OpenShell supervisor files, `/etc/openshell`, -`/etc/openshell-tls`, or `/run/netns`. - -OpenShell protects `/usr` and `/var` as OCI workspaces except for the -application conventions `/usr/src/app`, `/var/app`, `/var/task`, and -`/var/www`. Existing workspace parents must be traversable by the resolved -identity; OpenShell creates missing parents with mode `0755`. +valid. Mounts also must not contain or be contained by concrete OpenShell +control targets such as the supervisor mount, TLS and token files, runtime +socket, or `/run/netns`. Example named-volume usage: diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index bcf6cb577b..567dcb9094 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -427,6 +427,13 @@ fn build_env( user_env.insert(k.clone(), v.clone()); } } + for key in [ + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::POLICY_RUN_AS_USER, + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, + ] { + user_env.remove(key); + } env.extend(user_env.clone()); if !user_env.is_empty() && let Ok(json) = serde_json::to_string(&user_env) @@ -487,6 +494,9 @@ fn build_env( env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); + env.remove(openshell_core::sandbox_env::POLICY_RUN_AS_USER); + env.remove(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP); env.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.into(), oci_user.to_string(), @@ -499,6 +509,17 @@ fn build_env( openshell_core::sandbox_env::SANDBOX_GID.into(), String::new(), ); + for key in [ + openshell_core::sandbox_env::POLICY_RUN_AS_USER, + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, + ] { + env.insert( + key.into(), + spec.and_then(|spec| spec.environment.get(key)) + .cloned() + .unwrap_or_default(), + ); + } // 4. Gateway-minted sandbox JWT. Keep the raw bearer out of container // metadata; the supervisor reads it from a driver-owned bind mount. @@ -942,8 +963,13 @@ pub fn build_container_spec_for_image( .map_or("", |config| config.working_dir.as_str()); let workspace_root = driver_mounts::resolve_oci_workspace_root(oci_working_dir) .map_err(ComputeDriverError::Precondition)?; + driver_mounts::validate_workspace_control_path( + &workspace_root, + &config.sandbox_ssh_socket_path, + ) + .map_err(ComputeDriverError::Precondition)?; - let env = build_env(sandbox, config, requested_image, oci_user); + let mut env = build_env(sandbox, config, requested_image, oci_user); let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts, &workspace_root) @@ -989,6 +1015,10 @@ pub fn build_container_spec_for_image( let mut command = vec!["--workdir".to_string(), workspace_root]; command.extend(upstream_proxy_cli_args(config)); + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED.into(), + "1".into(), + ); let container_spec = ContainerSpec { name, @@ -1209,6 +1239,77 @@ pub fn build_container_spec_for_image( Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) } +/// Build a minimal one-shot container that validates the image's original +/// workdir before the final Podman workspace volume covers it. +pub fn build_workspace_probe_spec( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + inspected_image: &ImageInspect, + probe_name: &str, +) -> Result, ComputeDriverError> { + let image_config = inspected_image.config.as_ref(); + let oci_user = image_config.map_or("", |config| config.user.as_str()); + let oci_working_dir = image_config.map_or("", |config| config.working_dir.as_str()); + let workspace_root = driver_mounts::resolve_oci_workspace_root(oci_working_dir) + .map_err(ComputeDriverError::Precondition)?; + driver_mounts::validate_workspace_control_path( + &workspace_root, + &config.sandbox_ssh_socket_path, + ) + .map_err(ComputeDriverError::Precondition)?; + if workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT { + return Ok(None); + } + + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| ComputeDriverError::Precondition("sandbox.spec is required".into()))?; + let mut command = vec![ + "validate-workspace".to_string(), + "--workdir".to_string(), + workspace_root, + "--oci-user".to_string(), + oci_user.to_string(), + ]; + for (flag, key) in [ + ( + "--run-as-user", + openshell_core::sandbox_env::POLICY_RUN_AS_USER, + ), + ( + "--run-as-group", + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, + ), + ] { + if let Some(value) = spec.environment.get(key).filter(|value| !value.is_empty()) { + command.push(flag.to_string()); + command.push(value.clone()); + } + } + + Ok(Some(serde_json::json!({ + "name": probe_name, + "image": inspected_image.id, + "entrypoint": [SUPERVISOR_BINARY_PATH], + "command": command, + "user": "0:0", + "work_dir": "/", + "image_volumes": [{ + "source": config.supervisor_image, + "destination": SUPERVISOR_MOUNT_DIR, + "rw": false + }], + // Do not materialize OCI VOLUME declarations: the probe must inspect + // the immutable image layer rather than a fresh anonymous volume. + "image_volume_mode": "ignore", + "netns": {"nsmode": "none"}, + "no_new_privileges": true, + "cap_drop": ["ALL"], + "image_pull_policy": "never" + }))) +} + fn hostadd_entries(config: &PodmanComputeConfig) -> Vec { let host_gateway_ip = config.host_gateway_ip.trim(); if host_gateway_ip.is_empty() { @@ -1450,6 +1551,10 @@ mod tests { container["env"][openshell_core::sandbox_env::SANDBOX_GID].as_str(), Some("") ); + assert_eq!( + container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED].as_str(), + Some("1") + ); } #[test] @@ -1471,18 +1576,77 @@ mod tests { } #[test] - fn container_spec_rejects_protected_oci_working_dir() { + fn container_spec_rejects_openshell_control_path_working_dir() { let err = build_container_spec_for_image( &test_sandbox("test-id", "test-name"), &test_config(), None, None, "registry.example/app:latest", - &inspected_image("sha256:immutable", "app:staff", "/etc"), + &inspected_image( + "sha256:immutable", + "app:staff", + "/opt/openshell/bin/project", + ), ) .unwrap_err(); - assert!(err.to_string().contains("protected container path")); + assert!(err.to_string().contains("OpenShell control path")); + } + + #[test] + fn workspace_probe_uses_pinned_image_without_workspace_or_network() { + let mut sandbox = test_sandbox("test-id", "test-name"); + let spec = sandbox.spec.get_or_insert_default(); + spec.environment.insert( + openshell_core::sandbox_env::POLICY_RUN_AS_USER.into(), + "policy-user".into(), + ); + spec.environment.insert( + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP.into(), + "policy-group".into(), + ); + let probe = build_workspace_probe_spec( + &sandbox, + &test_config(), + &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), + "openshell-test-probe", + ) + .unwrap() + .unwrap(); + + assert_eq!(probe["image"], "sha256:immutable"); + assert_eq!(probe["netns"]["nsmode"], "none"); + assert_eq!(probe["image_volume_mode"], "ignore"); + assert!(probe.get("volumes").is_none()); + assert!(probe.get("secrets").is_none()); + assert!(probe.get("env").is_none()); + assert_eq!( + probe["command"], + serde_json::json!([ + "validate-workspace", + "--workdir", + "/workspace/project", + "--oci-user", + "app:staff", + "--run-as-user", + "policy-user", + "--run-as-group", + "policy-group" + ]) + ); + } + + #[test] + fn workspace_probe_skips_sandbox_compatibility_fallback() { + let probe = build_workspace_probe_spec( + &test_sandbox("test-id", "test-name"), + &test_config(), + &inspected_image("sha256:immutable", "sandbox:sandbox", "/"), + "openshell-test-probe", + ) + .unwrap(); + assert!(probe.is_none()); } #[test] @@ -2674,7 +2838,7 @@ mod tests { "mounts": [{ "type": "volume", "source": "work-nfs", - "target": "/etc/openshell/tls/custom" + "target": "/etc/openshell/tls/client" }] }))), ..Default::default() diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 4154c85041..7a565983af 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -77,6 +77,15 @@ fn validated_container_name(sandbox: &DriverSandbox) -> Result String { + const SUFFIX: &str = "-workdir-probe"; + let keep = 255usize.saturating_sub(SUFFIX.len()); + format!( + "{}{SUFFIX}", + &container_name[..container_name.len().min(keep)] + ) +} + fn podman_volume_is_bind_backed(volume: &VolumeInspect) -> bool { (volume.driver.is_empty() || volume.driver == "local") && volume.options.get("o").is_some_and(|options| { @@ -233,6 +242,80 @@ fn resolve_socket_path( } impl PodmanComputeDriver { + async fn validate_image_workspace( + &self, + sandbox: &DriverSandbox, + container_name: &str, + inspected_image: &crate::client::ImageInspect, + ) -> Result<(), ComputeDriverError> { + let probe_name = workspace_probe_name(container_name); + crate::client::validate_name(&probe_name) + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; + let Some(spec) = container::build_workspace_probe_spec( + sandbox, + &self.config, + inspected_image, + &probe_name, + )? + else { + return Ok(()); + }; + + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + self.client + .create_container(&spec) + .await + .map_err(ComputeDriverError::from)?; + let validation = async { + self.client + .start_container(&probe_name) + .await + .map_err(ComputeDriverError::from)?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let inspect = self + .client + .inspect_container(&probe_name) + .await + .map_err(ComputeDriverError::from)?; + if !inspect.state.running + && matches!(inspect.state.status.as_str(), "exited" | "stopped") + { + return if inspect.state.exit_code == 0 { + Ok(()) + } else { + Err(ComputeDriverError::Precondition(format!( + "OCI WorkingDir validation failed for image '{}' (probe exited with code {})", + inspected_image.id, inspect.state.exit_code + ))) + }; + } + if tokio::time::Instant::now() >= deadline { + return Err(ComputeDriverError::Precondition(format!( + "timed out validating OCI WorkingDir for image '{}'", + inspected_image.id + ))); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + .await; + let cleanup = self.client.remove_container(&probe_name).await; + match (validation, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(error)) => Err(ComputeDriverError::from(error)), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(cleanup_error)) => { + warn!( + probe = %probe_name, + %cleanup_error, + "Failed to remove workspace validation probe" + ); + Err(error) + } + } + } + /// Create a new driver, verifying the Podman socket is reachable. pub async fn new(mut config: PodmanComputeConfig) -> Result { const MAX_PING_RETRIES: u32 = 5; @@ -586,6 +669,8 @@ impl PodmanComputeDriver { "podman image '{image}' inspection did not return an immutable image ID" ))); } + self.validate_image_workspace(sandbox, &name, &inspected_image) + .await?; for image in container::podman_driver_image_mount_sources(sandbox, self.config.enable_bind_mounts) .map_err(ComputeDriverError::Precondition)? @@ -1662,6 +1747,85 @@ mod tests { } } + #[tokio::test] + async fn workspace_probe_waits_for_success_and_always_removes_container() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "workspace-probe-success", + vec![ + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"probe","Name":"probe","State":{"Status":"exited","Running":false,"ExitCode":0},"Config":{}}"#, + ), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + let mut sandbox = plain_sandbox("sandbox-probe", "demo"); + sandbox.spec = Some(DriverSandboxSpec::default()); + let image = crate::client::ImageInspect { + id: "sha256:immutable".into(), + config: Some(crate::client::ImageConfig { + user: "1234:1235".into(), + working_dir: "/workspace".into(), + }), + }; + let name = validated_container_name(&sandbox).unwrap(); + + driver + .validate_image_workspace(&sandbox, &name, &image) + .await + .expect("successful probe should pass"); + + handle.await.expect("stub task should finish"); + let requests = request_log.lock().unwrap(); + assert_eq!(requests.len(), 4); + assert!(requests[0].contains("/libpod/containers/create")); + assert!(requests[1].contains("/start")); + assert!(requests[2].contains("/json")); + assert!(requests[3].starts_with("DELETE ")); + let _ = fs::remove_file(socket_path); + } + + #[tokio::test] + async fn workspace_probe_removes_container_after_validation_failure() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "workspace-probe-failure", + vec![ + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"probe","Name":"probe","State":{"Status":"exited","Running":false,"ExitCode":1},"Config":{}}"#, + ), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + let mut sandbox = plain_sandbox("sandbox-probe-fail", "demo"); + sandbox.spec = Some(DriverSandboxSpec::default()); + let image = crate::client::ImageInspect { + id: "sha256:immutable".into(), + config: Some(crate::client::ImageConfig { + user: "1234:1235".into(), + working_dir: "/workspace".into(), + }), + }; + let name = validated_container_name(&sandbox).unwrap(); + + let error = driver + .validate_image_workspace(&sandbox, &name, &image) + .await + .unwrap_err(); + assert!(error.to_string().contains("exited with code 1")); + + handle.await.expect("stub task should finish"); + let requests = request_log.lock().unwrap(); + assert!(requests.last().unwrap().starts_with("DELETE ")); + let _ = fs::remove_file(socket_path); + } + fn secret_delete_request(sandbox_id: &str) -> String { format!( "DELETE {}", diff --git a/crates/openshell-prover/src/lib.rs b/crates/openshell-prover/src/lib.rs index 0fb8757577..913045fe7d 100644 --- a/crates/openshell-prover/src/lib.rs +++ b/crates/openshell-prover/src/lib.rs @@ -105,10 +105,11 @@ mod tests { fn test_filesystem_policy() { let path = testdata_dir().join("policy.yaml"); let model = parse_policy(&path).expect("failed to parse policy"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(None); assert!(readable.contains(&"/usr".to_owned())); assert!(readable.contains(&"/sandbox".to_owned())); assert!(readable.contains(&"/tmp".to_owned())); + assert!(readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); } // 3. Workdir NOT included by default (matches runtime behavior). @@ -121,8 +122,9 @@ filesystem_policy: - /usr "; let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(None); assert!(!readable.contains(&"/sandbox".to_owned())); + assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); } // 4. Workdir excluded when include_workdir: false. @@ -136,8 +138,9 @@ filesystem_policy: - /usr "; let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(None); assert!(!readable.contains(&"/sandbox".to_owned())); + assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); } // 5. No duplicate when workdir already in read_write. @@ -152,12 +155,30 @@ filesystem_policy: - /tmp "; let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(Some("/sandbox")); let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); assert_eq!(sandbox_count, 1); } - // 6. End-to-end: testdata policy with a github credential in scope and a + // 6. A resolved non-default workdir does not replace an explicit path. + #[test] + fn test_include_workdir_preserves_explicit_sandbox_path() { + let yaml = r" +version: 1 +filesystem_policy: + include_workdir: true + read_write: + - /sandbox +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model + .filesystem_policy + .readable_paths(Some("/workspace/project")); + assert!(readable.contains(&"/sandbox".to_owned())); + assert!(readable.contains(&"/workspace/project".to_owned())); + } + + // 7. End-to-end: testdata policy with a github credential in scope and a // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. // The prover output is categorical, not severity-graded. #[test] diff --git a/crates/openshell-prover/src/model.rs b/crates/openshell-prover/src/model.rs index bf52993d47..3769b17e43 100644 --- a/crates/openshell-prover/src/model.rs +++ b/crates/openshell-prover/src/model.rs @@ -268,7 +268,7 @@ impl ReachabilityModel { } fn encode_filesystem(&mut self) { - for path in self.policy.filesystem_policy.readable_paths() { + for path in self.policy.filesystem_policy.readable_paths(None) { let var = Bool::new_const(format!("fs_readable_{path}")); self.solver.assert(&var); self.filesystem_readable.insert(path, var); diff --git a/crates/openshell-prover/src/policy.rs b/crates/openshell-prover/src/policy.rs index aa40d07560..599c8d131a 100644 --- a/crates/openshell-prover/src/policy.rs +++ b/crates/openshell-prover/src/policy.rs @@ -257,18 +257,26 @@ pub struct FilesystemPolicy { pub read_write: Vec, } +/// Symbol used when the prover does not know the image-resolved workspace. +/// +/// Keeping this distinct from `/sandbox` prevents the model from inventing a +/// literal compatibility workspace for images that declare another workdir. +pub const WORKDIR_PATH_SYMBOL: &str = ""; + impl FilesystemPolicy { /// All readable paths (union of `read_only` and `read_write`), with workdir - /// added when `include_workdir` is true and not already present. - pub fn readable_paths(&self) -> Vec { + /// added when `include_workdir` is true and not already present. When the + /// resolved workdir is unavailable, retain it as a symbolic path. + pub fn readable_paths(&self, resolved_workdir: Option<&str>) -> Vec { let mut paths: Vec = self .read_only .iter() .chain(self.read_write.iter()) .cloned() .collect(); - if self.include_workdir && !paths.iter().any(|p| p == "/sandbox") { - paths.push("/sandbox".to_owned()); + let workdir = resolved_workdir.unwrap_or(WORKDIR_PATH_SYMBOL); + if self.include_workdir && !paths.iter().any(|path| path == workdir) { + paths.push(workdir.to_owned()); } paths } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 250ea8c3f7..604d100a82 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -190,6 +190,27 @@ pub async fn run_sandbox( // fill only omitted policy fields from OCI Config.User. #[cfg(unix)] let (resolved_process_identity, workdir_is_home) = { + if std::env::var_os(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED) + .is_some_and(|value| value == "1") + { + for (key, actual) in [ + ( + openshell_core::sandbox_env::POLICY_RUN_AS_USER, + policy.process.run_as_user.as_deref().unwrap_or(""), + ), + ( + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, + policy.process.run_as_group.as_deref().unwrap_or(""), + ), + ] { + let expected = std::env::var(key).unwrap_or_default(); + if expected != actual { + return Err(miette::miette!( + "process identity changed after OCI workspace validation" + )); + } + } + } let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; let workdir_is_home = matches!( &driver_identity, diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 62ae37b5a1..76bbbb5023 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -32,6 +32,7 @@ const COPY_SELF_SUBCOMMAND: &str = "copy-self"; /// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` /// to confirm the cross-sandbox IDOR guard fires. const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; +const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; @@ -231,6 +232,60 @@ struct Args { upstream_proxy_connect_by_hostname: bool, } +#[derive(Parser, Debug)] +#[command(name = "validate-workspace")] +struct ValidateWorkspaceArgs { + #[arg(long)] + workdir: String, + #[arg(long)] + oci_user: String, + #[arg(long)] + run_as_user: Option, + #[arg(long)] + run_as_group: Option, +} + +#[cfg(unix)] +fn validate_workspace(args: &[String]) -> Result<()> { + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use openshell_supervisor_process::identity::{DriverIdentity, resolve_process_identity}; + + let args = ValidateWorkspaceArgs::try_parse_from( + std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), + ) + .into_diagnostic()?; + let mut policy = SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user: args.run_as_user, + run_as_group: args.run_as_group, + }, + }; + let resolved_identity = resolve_process_identity( + &mut policy, + &DriverIdentity::OciUser { + declaration: args.oci_user, + }, + )?; + openshell_supervisor_process::process::validate_oci_workspace_with_process_identity( + &policy, + resolved_identity, + Path::new(&args.workdir), + ) +} + +#[cfg(not(unix))] +fn validate_workspace(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "workspace validation is only supported on Unix" + )) +} + /// Copy the running executable to `dest`, creating parent directories as /// needed and ensuring the result is executable (mode `0755`). /// @@ -479,6 +534,9 @@ fn main() -> Result<()> { std::process::exit(exit); }); } + if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { + return validate_workspace(&raw_args[2..]); + } let args = Args::parse(); @@ -648,6 +706,28 @@ mod tests { use super::*; use std::os::unix::fs::PermissionsExt; + #[cfg(unix)] + #[test] + fn workspace_validation_subcommand_uses_final_policy_identity() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + let args = vec![ + "--workdir".to_string(), + root.display().to_string(), + "--oci-user".to_string(), + "unused".to_string(), + "--run-as-user".to_string(), + "1000".to_string(), + "--run-as-group".to_string(), + "1000".to_string(), + ]; + + validate_workspace(&args).expect("current identity should retain workspace authority"); + } + /// Drives `copy_self`'s file-copy logic against an arbitrary source path /// so tests don't depend on `current_exe()`. fn copy_executable(src: &Path, dest: &Path) -> Result<()> { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index b87f1b5fa6..9814f40aa9 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -2153,9 +2153,24 @@ fn driver_sandbox_spec_from_public( spec: &SandboxSpec, driver_name: &str, ) -> Result> { + let mut environment = spec.environment.clone(); + if matches!(driver_name, "docker" | "podman") { + let process = spec + .policy + .as_ref() + .and_then(|policy| policy.process.as_ref()); + environment.insert( + openshell_core::sandbox_env::POLICY_RUN_AS_USER.to_string(), + process.map_or_else(String::new, |process| process.run_as_user.clone()), + ); + environment.insert( + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP.to_string(), + process.map_or_else(String::new, |process| process.run_as_group.clone()), + ); + } Ok(DriverSandboxSpec { log_level: spec.log_level.clone(), - environment: spec.environment.clone(), + environment, template: spec .template .as_ref() @@ -2912,6 +2927,37 @@ mod tests { assert_eq!(gpu.count, Some(2)); } + #[test] + fn driver_sandbox_spec_projects_process_identity_for_local_image_probe() { + let public = SandboxSpec { + policy: Some(openshell_core::proto::SandboxPolicy { + process: Some(openshell_core::proto::ProcessPolicy { + run_as_user: "app".into(), + run_as_group: "staff".into(), + }), + ..Default::default() + }), + ..Default::default() + }; + + let driver = + driver_sandbox_spec_from_public(&public, "podman").expect("driver spec should map"); + assert_eq!( + driver + .environment + .get(openshell_core::sandbox_env::POLICY_RUN_AS_USER) + .map(String::as_str), + Some("app") + ); + assert_eq!( + driver + .environment + .get(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP) + .map(String::as_str), + Some("staff") + ); + } + #[test] fn select_driver_config_forwards_only_matching_driver_block() { let config = prost_types::Struct { diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 2e5713f186..a0cdae22fb 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -113,6 +113,9 @@ pub(crate) fn prepare_child_sandbox( const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::POLICY_RUN_AS_USER, + openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, openshell_core::sandbox_env::SANDBOX_UID, openshell_core::sandbox_env::SANDBOX_GID, openshell_core::sandbox_env::SANDBOX_TOKEN, @@ -1276,6 +1279,161 @@ fn prepare_oci_workspace( prepare_oci_workspace_with(root, uid, gid, supplementary_gids, &nix::unistd::chown) } +/// Validate that selecting an image-provided OCI workdir does not grant the +/// sandbox identity any filesystem authority it lacked in the immutable image. +/// +/// Every path component must be a real directory (never a symlink), every +/// parent must already be traversable, and the final directory must already be +/// writable and traversable. No ownership or mode bits are changed. +#[cfg(unix)] +pub fn validate_oci_workspace( + root: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> Result<()> { + let root_str = root + .to_str() + .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; + let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) + .map_err(|error| miette::miette!(error))?; + if Path::new(&validated_root) != root + || validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT + { + return Err(miette::miette!( + "image workspace path '{}' must be a normalized absolute non-fallback path", + root.display() + )); + } + + let mut current = PathBuf::from("/"); + validate_workspace_component(¤t, uid, gid, supplementary_gids, false)?; + let components = root + .components() + .skip(1) + .map(|component| match component { + std::path::Component::Normal(component) => Ok(component), + _ => Err(miette::miette!( + "workspace path '{}' must be normalized", + root.display() + )), + }) + .collect::>>()?; + let last_component = components.len().saturating_sub(1); + for (index, component) in components.into_iter().enumerate() { + current.push(component); + validate_workspace_component( + ¤t, + uid, + gid, + supplementary_gids, + index == last_component, + )?; + } + Ok(()) +} + +/// Validate an OCI workdir using the completed process identity selected by +/// policy plus driver-owned OCI metadata. +#[cfg(unix)] +pub fn validate_oci_workspace_with_process_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + workdir: &Path, +) -> Result<()> { + validate_sandbox_user_with_identity(policy, resolved_identity)?; + validate_sandbox_group_with_identity(policy, resolved_identity)?; + let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; + validate_oci_workspace(workdir, uid, gid, &supplementary_gids) +} + +#[cfg(unix)] +fn validate_workspace_component( + path: &Path, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], + is_workspace: bool, +) -> Result<()> { + let metadata = std::fs::symlink_metadata(path).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + miette::miette!( + "image workspace path component '{}' does not exist", + path.display() + ) + } else { + miette::miette!( + "failed to inspect image workspace path component '{}': {error}", + path.display() + ) + } + })?; + if metadata.file_type().is_symlink() { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + path.display() + )); + } + if !metadata.is_dir() { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + path.display() + )); + } + reject_special_workspace_filesystem(path)?; + + let required = if is_workspace { 0o3 } else { 0o1 }; + if !identity_has_permissions(&metadata, uid, gid, supplementary_gids, required) { + let requirement = if is_workspace { + "writable and traversable" + } else { + "traversable" + }; + return Err(miette::miette!( + "workspace path component '{}' is not {requirement} by the sandbox identity in the image", + path.display() + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { + let fs = rustix::fs::statfs(path).into_diagnostic()?; + #[allow(clippy::cast_sign_loss)] + let filesystem_type = fs.f_type as u64; + // Linux filesystem magic values for virtual/kernel-managed filesystems. + // The decision is based on the mounted filesystem, not its conventional + // distro path, so renamed or unusually mounted control filesystems remain + // protected without rejecting ordinary application directories. + const SPECIAL_FILESYSTEMS: &[u64] = &[ + 0x0000_1cd1, // devpts + 0x0000_9fa0, // proc + 0x0102_1994, // tmpfs (including the container /dev tree) + 0x1980_0202, // mqueue + 0x27e0_eb, // cgroup + 0x4249_4e4d, // bpf + 0x6265_6572, // sysfs + 0x6367_7270, // cgroup2 + 0x6462_6720, // debugfs + 0x7363_6673, // securityfs + 0x7472_6163, // tracefs + ]; + if SPECIAL_FILESYSTEMS.contains(&filesystem_type) { + return Err(miette::miette!( + "workspace path component '{}' is on a kernel-managed filesystem (type {filesystem_type:#x})", + path.display() + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +#[allow(clippy::unnecessary_wraps)] +fn reject_special_workspace_filesystem(_path: &Path) -> Result<()> { + Ok(()) +} + /// Prepare only the resolved `OpenShell` workspace directory itself. /// /// Image-provided children retain their declared ownership. This avoids @@ -1365,6 +1523,17 @@ fn identity_can_traverse( uid: Option, gid: Option, supplementary_gids: &[Gid], +) -> bool { + identity_has_permissions(metadata, uid, gid, supplementary_gids, 0o1) +} + +#[cfg(unix)] +fn identity_has_permissions( + metadata: &std::fs::Metadata, + uid: Option, + gid: Option, + supplementary_gids: &[Gid], + required: u32, ) -> bool { let user_id = uid.unwrap_or_else(nix::unistd::geteuid).as_raw(); if user_id == 0 { @@ -1374,15 +1543,15 @@ fn identity_can_traverse( let group_id = gid.unwrap_or_else(nix::unistd::getegid).as_raw(); let mode = metadata.permissions().mode(); if metadata.uid() == user_id { - mode & 0o100 != 0 + mode & (required << 6) == required << 6 } else if metadata.gid() == group_id || supplementary_gids .iter() .any(|supplementary_gid| supplementary_gid.as_raw() == metadata.gid()) { - mode & 0o010 != 0 + mode & (required << 3) == required << 3 } else { - mode & 0o001 != 0 + mode & required == required } } @@ -1489,23 +1658,89 @@ pub fn prepare_filesystem_with_identity( prepare_workspace: bool, ) -> Result<()> { use nix::unistd::chown; - use nix::unistd::{Gid, Uid}; - - let user_name = match policy.process.run_as_user.as_deref() { - Some(name) if !name.is_empty() => Some(name), - _ => None, - }; - let group_name = match policy.process.run_as_group.as_deref() { - Some(name) if !name.is_empty() => Some(name), - _ => None, - }; // If no user/group configured, nothing to do - if user_name.is_none() && group_name.is_none() { + if policy + .process + .run_as_user + .as_deref() + .is_none_or(str::is_empty) + && policy + .process + .run_as_group + .as_deref() + .is_none_or(str::is_empty) + { return Ok(()); } - // Resolve UID: numeric values are passed directly; names resolve via passwd. + let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; + + // Docker and Podman own workspace resolution and must make the selected + // root usable by the final effective identity, including when both policy + // identity fields were explicit. Validate it before processing any + // user-authored read-write paths so an unsafe image path fails first. + // Other drivers retain their preparation. + if prepare_workspace { + let workspace = workdir.ok_or_else(|| { + miette::miette!("local container driver did not supply a workspace workdir") + })?; + let workspace = Path::new(workspace); + if workspace == Path::new(openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) + || std::env::var_os(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED) + .is_some_and(|value| value == "1") + { + info!(path = %workspace.display(), ?uid, ?gid, "Preparing managed workspace"); + prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; + } else { + info!(path = %workspace.display(), ?uid, ?gid, "Validating image workspace authority"); + validate_oci_workspace(workspace, uid, gid, &supplementary_gids)?; + } + } + + // Create missing read_write paths and only chown the ones we created. + for path in &policy.filesystem.read_write { + if prepare_read_write_path(path)? { + debug!( + path = %path.display(), + ?uid, + ?gid, + "Setting ownership on newly created read_write path" + ); + chown(path, uid, gid).into_diagnostic()?; + } + } + + // Retain the existing Kubernetes/OpenShift behavior for driver-injected + // numeric identities. Docker and Podman clear this variable and do not + // receive identity-specific workspace preparation. + if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { + let sandbox_home = Path::new("/sandbox"); + if sandbox_home.exists() { + info!(?uid, ?gid, "Chowning /sandbox for driver-injected UID/GID"); + chown_sandbox_home(sandbox_home, uid, gid)?; + } + } + + Ok(()) +} + +#[cfg(unix)] +fn resolve_filesystem_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result<(Option, Option, Vec)> { + let user_name = policy + .process + .run_as_user + .as_deref() + .filter(|name| !name.is_empty()); + let group_name = policy + .process + .run_as_group + .as_deref() + .filter(|name| !name.is_empty()); + let uid = match resolved_identity.uid() { Some(uid) => Some(Uid::from_raw(uid)), None => match user_name { @@ -1546,45 +1781,7 @@ pub fn prepare_filesystem_with_identity( _ => Vec::new(), }; - // Docker and Podman own workspace resolution and must make the selected - // root usable by the final effective identity, including when both policy - // identity fields were explicit. Validate it before processing any - // user-authored read-write paths so an unsafe image path fails first. - // Other drivers retain their preparation. - if prepare_workspace { - let workspace = workdir.ok_or_else(|| { - miette::miette!("local container driver did not supply a workspace workdir") - })?; - let workspace = Path::new(workspace); - info!(path = %workspace.display(), ?uid, ?gid, "Preparing resolved workspace"); - prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; - } - - // Create missing read_write paths and only chown the ones we created. - for path in &policy.filesystem.read_write { - if prepare_read_write_path(path)? { - debug!( - path = %path.display(), - ?uid, - ?gid, - "Setting ownership on newly created read_write path" - ); - chown(path, uid, gid).into_diagnostic()?; - } - } - - // Retain the existing Kubernetes/OpenShift behavior for driver-injected - // numeric identities. Docker and Podman clear this variable and do not - // receive identity-specific workspace preparation. - if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { - let sandbox_home = Path::new("/sandbox"); - if sandbox_home.exists() { - info!(?uid, ?gid, "Chowning /sandbox for driver-injected UID/GID"); - chown_sandbox_home(sandbox_home, uid, gid)?; - } - } - - Ok(()) + Ok((uid, gid, supplementary_gids)) } #[cfg(not(unix))] @@ -2665,6 +2862,134 @@ mod tests { assert!(child.exists(), "image-provided child should be untouched"); } + #[cfg(unix)] + #[test] + fn validate_oci_workspace_accepts_existing_owner_writable_directory() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + + validate_oci_workspace( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .expect("image owner already has write and traverse authority"); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_accepts_supplementary_group_write_authority() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o070)).unwrap(); + let metadata = std::fs::symlink_metadata(&root).unwrap(); + + validate_oci_workspace( + &root, + Some(Uid::from_raw(metadata.uid().wrapping_add(1))), + Some(Gid::from_raw(metadata.gid().wrapping_add(1))), + &[Gid::from_raw(metadata.gid())], + ) + .expect("supplementary group already has write and traverse authority"); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_unwritable_directory() { + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let metadata = std::fs::symlink_metadata(&root).unwrap(); + + let error = validate_oci_workspace( + &root, + Some(Uid::from_raw(metadata.uid().wrapping_add(1))), + Some(Gid::from_raw(metadata.gid().wrapping_add(1))), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("not writable and traversable")); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_missing_path() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap().join("missing"); + + let error = validate_oci_workspace( + &root, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("does not exist")); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_restrictive_parent() { + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().canonicalize().unwrap().join("private"); + let root = parent.join("project"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + let metadata = std::fs::symlink_metadata(&parent).unwrap(); + + let error = validate_oci_workspace( + &root, + Some(Uid::from_raw(metadata.uid().wrapping_add(1))), + Some(Gid::from_raw(metadata.gid().wrapping_add(1))), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("not traversable")); + } + + #[cfg(unix)] + #[test] + fn validate_oci_workspace_rejects_symlink_component() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let target = base.join("target"); + let link = base.join("link"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &link).unwrap(); + + let error = validate_oci_workspace( + &link, + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("symlink")); + } + + #[cfg(target_os = "linux")] + #[test] + fn validate_oci_workspace_rejects_kernel_filesystem_by_metadata() { + let error = validate_oci_workspace( + Path::new("/proc"), + Some(nix::unistd::geteuid()), + Some(nix::unistd::getegid()), + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("kernel-managed filesystem")); + } + #[cfg(unix)] #[test] fn prepare_oci_workspace_makes_existing_root_owner_writable() { diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 5edf888d8f..edf83d250b 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -168,9 +168,10 @@ Docker mount schema: OpenShell rejects mount `source`, `target`, and Docker volume `subpath` values with surrounding whitespace. OpenShell also rejects mount targets that replace -the workspace root, container root, supervisor files, `/etc/openshell`, -`/etc/openshell-tls`, authentication material, or network namespace paths. These -checks do not make host bind mounts safe. +the workspace root or container root, or contain or are contained by concrete +OpenShell control targets such as the supervisor mount, TLS and token files, +runtime socket, or network namespace mount. These checks do not make host bind +mounts safe. ## Podman Driver @@ -241,9 +242,9 @@ Podman `volume` and `image` mounts do not support `subpath` in OpenShell driver config, and OpenShell rejects `subpath` for those mount types. OpenShell rejects mount `source` and `target` values with surrounding whitespace. OpenShell also rejects mount targets that replace or contain the workspace root, target the -container root or supervisor files, or overlap `/etc/openshell`, -`/etc/openshell-tls`, authentication material, or network namespace paths. -These checks do not make host bind mounts safe. +container root, or contain or are contained by concrete OpenShell control +targets such as the supervisor mount, TLS and token files, runtime socket, or +network namespace mount. These checks do not make host bind mounts safe. ## MicroVM Driver @@ -435,17 +436,21 @@ declared name or numeric components for both direct and SSH children. When does not modify `/etc/passwd` or `/etc/group`. Docker and Podman also inspect OCI `WorkingDir`. An absolute value becomes the -agent workspace; an empty or root (`/`) value falls back to `/sandbox`. -OpenShell creates the resolved directory when needed and transfers ownership of -only that directory to the completed UID/GID. Existing image contents and -nested bind or volume mounts retain their ownership. The resolved workspace is -the cwd and `HOME` for direct and SSH children. Protected system trees and -OpenShell-reserved paths cannot become workspaces. OpenShell protects `/usr` -and `/var` except for `/usr/src/app`, `/var/app`, `/var/task`, `/var/www`, and -their descendants. Existing workspace parents must be traversable by the -resolved identity; OpenShell creates missing parents with mode `0755`. The -supervisor itself starts from `/`, so a missing or invalid workspace is handled -during readiness instead of preventing the container from starting. +agent workspace; an empty, root (`/`), or explicit `/sandbox` value uses the +managed `/sandbox` compatibility workspace. +OpenShell creates and owns that compatibility workspace. Any other workdir must +already exist in the immutable image without symlink components. The completed +UID/GID and supplementary groups must already be able to traverse every parent +and write and enter the workdir. OpenShell does not change that directory's +ownership or mode. It rejects kernel-managed filesystems based on filesystem +metadata and rejects overlap with actual OpenShell control paths. Docker checks +the original image filesystem in the final supervisor. Podman checks the same +pinned image ID in a temporary networkless probe without the workspace volume, +token, or TLS secrets, removes the probe, and only then creates the final +volume-backed sandbox. The resolved workspace is the cwd and `HOME` for direct +and SSH children. The supervisor itself starts from `/`, so a missing or +invalid workspace is handled during readiness instead of preventing the +container runtime from starting it. Sandbox creation fails before readiness if a required `USER` component is missing, malformed, unknown, ambiguous, or resolves to UID/GID 0. An image @@ -477,5 +482,8 @@ Docker and Podman custom images do not need a baked-in `"sandbox"` user. Declare a non-root OCI `USER`, or set both process identity fields explicitly in policy. Named image users require matching account entries; a numeric `UID:GID` pair does not. Declare an absolute OCI `WORKDIR` to select the workspace. Images with -no working directory, or `WORKDIR /`, use OpenShell's `/sandbox` fallback. +no working directory, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's +managed `/sandbox` compatibility workspace. +For any other path, create the directory in the image and grant the final +process identity write and execute permission in the Dockerfile. Kubernetes/OpenShift and VM sandboxes continue to use `/sandbox`. diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index 5494d1078b..e43cdb0514 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -24,10 +24,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ RUN groupadd -g 1235 appstaff && \ useradd -m -u 1234 -g appstaff app -# A custom image may declare a root-owned OCI working directory with existing -# root-owned content. OpenShell prepares only the working directory itself. +# The final image identity already owns the OCI working directory. Existing +# root-owned content remains root-owned. WORKDIR /workspace/project -RUN printf root-owned > root-owned.txt +RUN printf root-owned > root-owned.txt && chown app:appstaff . # Write a marker file so we can verify this is our custom image. # Place under /etc (Landlock baseline read-only path) so the sandbox @@ -47,10 +47,22 @@ USER 2345:2346 CMD ["sleep", "infinity"] "#; +const UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim + +RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -g 3235 appstaff \ + && useradd -m -u 3234 -g appstaff app + +WORKDIR /workspace/project +USER app +CMD ["sleep", "infinity"] +"#; + const MARKER: &str = "custom-image-e2e-marker"; /// A named OCI user can write through direct and SSH children when the image -/// declares a root-owned WorkingDir; existing content retains its ownership. +/// already grants that authority; existing content retains its ownership. #[tokio::test] async fn sandbox_from_custom_dockerfile() { // Step 1: Write a temporary Dockerfile. @@ -200,3 +212,32 @@ async fn sandbox_from_passwd_less_numeric_oci_user() { guard.cleanup().await; } + +#[tokio::test] +async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { + let tmpdir = tempfile::tempdir().expect("create tmpdir"); + let dockerfile_path = tmpdir.path().join("Dockerfile"); + fs::write(&dockerfile_path, UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT).expect("write Dockerfile"); + let dockerfile_str = dockerfile_path.to_str().expect("Dockerfile path is UTF-8"); + + let result = SandboxGuard::create_keep_with_args( + &["--from", dockerfile_str, "--no-tty"], + &["sh", "-c", "echo should-not-run"], + "should-not-run", + ) + .await; + let error = match result { + Ok(mut guard) => { + guard.cleanup().await; + panic!("root-owned workdir must not be made writable for the image user"); + } + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("WorkingDir") + || message.contains("workspace") + || message.contains("readiness"), + "expected workspace authority failure, got: {message}" + ); +} diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index fd63571a0a..2a91f9ce1d 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -34,6 +34,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ && rm -rf /var/lib/apt/lists/* WORKDIR /workspace/project +RUN chown 2234:2235 . USER 2234:2235 CMD ["sleep", "infinity"] "#; diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index aff8d44dfe..47cca331b0 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -58,7 +58,7 @@ impl ImageGuard { "FROM {BASE_IMAGE}\n\ USER 0:0\n\ WORKDIR /workspace/project\n\ - RUN printf root-owned > root-owned.txt\n\ + RUN printf root-owned > root-owned.txt && chown {OCI_UID}:{OCI_GID} .\n\ USER {OCI_UID}:{OCI_GID}\n" ), ) From 1d2f22e7c5413339d458b2ea5c353c1a185165f4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 14:49:24 -0700 Subject: [PATCH 14/16] fix(podman): validate workdirs as sandbox identity Signed-off-by: Matthew Grossman --- .../skills/debug-openshell-cluster/SKILL.md | 2 +- architecture/compute-runtimes.md | 16 ++- crates/openshell-core/src/sandbox_env.rs | 8 ++ crates/openshell-driver-docker/src/lib.rs | 4 + crates/openshell-driver-podman/README.md | 19 +-- crates/openshell-driver-podman/src/client.rs | 89 +++++++++++++ .../openshell-driver-podman/src/container.rs | 70 ++++++++++- crates/openshell-driver-podman/src/driver.rs | 92 ++++++++++++-- crates/openshell-sandbox/src/lib.rs | 48 +++---- crates/openshell-sandbox/src/main.rs | 40 ++++-- crates/openshell-server/src/compute/mod.rs | 21 ++++ .../src/process.rs | 119 +++++++++++++++++- docs/reference/sandbox-compute-drivers.mdx | 14 ++- e2e/rust/tests/podman_oci_identity.rs | 13 +- 14 files changed, 479 insertions(+), 76 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 581670670b..3dbcdc38a8 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -178,7 +178,7 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, or supplementary groups must already be able to traverse every parent and write and enter the directory. Podman checks the original image in a networkless temporary probe before attaching the workspace volume, so inspect the probe failure in gateway logs. -- If Podman reports probe cleanup or timeout failures, inspect temporary containers with `podman ps -a --filter name=openshell-workspace-probe` and gateway logs. The driver force-removes the probe on every normal success or failure path. +- If Podman reports probe cleanup or timeout failures, inspect temporary containers with `podman ps -a --filter name=workdir-probe` and gateway logs. The driver force-removes the probe on every normal success or failure path. - Supervisor cannot call back: check callback endpoint and gateway logs. ### Step 6: Check Kubernetes Helm Gateways diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index abc13d4598..31bcc9df79 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -209,12 +209,16 @@ directory's ownership or mode. Filesystem metadata identifies kernel-managed mounts, while collision checks are derived from actual OpenShell control paths. Docker performs the check in the final container before workload launch. Podman performs it in a minimal networkless container from the same pinned -image ID before its managed workspace volume covers the path, then binds the -validated process identity to the final supervisor startup. The resolved -workspace is the child cwd and `HOME`; when `filesystem.include_workdir` is -enabled, it becomes the automatic writable policy path. Kubernetes/OpenShift -keep their `/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` -guest initialization path. +image ID before its managed workspace volume covers the path. The probe retains +only the capabilities needed to adopt the completed process identity, drops to +that identity, and asks the kernel to validate access. It emits a normalized +identity attestation that the final supervisor must match, including when the +image supplies the default process policy. The driver captures a bounded, +sanitized diagnostic before removing a failed probe. The resolved workspace is +the child cwd and `HOME`; when `filesystem.include_workdir` is enabled, it +becomes the automatic writable policy path. Kubernetes/OpenShift keep their +`/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` guest +initialization path. Sandbox creation fails before the workload becomes ready when a required image identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index ac58d92fbd..9acb6ddf8c 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -130,11 +130,19 @@ pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER"; /// runtime-managed workspace volume covered it. Set only by the Podman driver. pub const OCI_WORKSPACE_PREVALIDATED: &str = "OPENSHELL_OCI_WORKSPACE_PREVALIDATED"; +/// Normalized UID/GID/supplementary-group identity attested by the Podman +/// immutable-image workspace probe. +pub const OCI_WORKSPACE_IDENTITY: &str = "OPENSHELL_OCI_WORKSPACE_IDENTITY"; + /// Gateway-projected process policy identity used by local drivers during /// immutable-image workspace validation. pub const POLICY_RUN_AS_USER: &str = "OPENSHELL_POLICY_RUN_AS_USER"; pub const POLICY_RUN_AS_GROUP: &str = "OPENSHELL_POLICY_RUN_AS_GROUP"; +/// Whether a local driver must discover the initial process identity from the +/// immutable image policy because no policy existed at create time. +pub const POLICY_IDENTITY_FROM_IMAGE: &str = "OPENSHELL_POLICY_IDENTITY_FROM_IMAGE"; + // The corporate upstream-proxy configuration deliberately has no reserved // environment variables: it travels on the supervisor's argv // (`--upstream-proxy` and friends), which a sandbox image cannot forge the diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 45fde043a2..75603a57de 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2181,7 +2181,9 @@ fn build_environment_for_oci_user( } user_env.extend(spec.environment.clone()); for key in [ + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE, openshell_core::sandbox_env::POLICY_RUN_AS_USER, openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, ] { @@ -2242,7 +2244,9 @@ fn build_environment_for_oci_user( environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + environment.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); environment.remove(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); + environment.remove(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE); environment.remove(openshell_core::sandbox_env::POLICY_RUN_AS_USER); environment.remove(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP); environment.insert( diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 6a255028e6..4762d92480 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -21,14 +21,17 @@ An absolute OCI working directory becomes the agent workspace. An empty, root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell creates when necessary and owns as a compatibility workspace. For any other workdir, Podman first starts a minimal, networkless probe from the pinned image ID without the -workspace volume, tokens, or TLS secrets. The probe requires a real directory -with no symlink components and verifies that the completed identity, including -supplementary groups, can already traverse every parent and write and enter the -workdir. It also rejects kernel-managed filesystems and overlaps with concrete -OpenShell control resources. Only then does Podman mount and prepare the -managed workspace volume at that path; normal copy-up preserves image content. -The final supervisor verifies that its policy identity matches the probed -identity. The workspace is the child cwd and `HOME`. +workspace volume, tokens, or TLS secrets. The probe retains only `SETUID` and +`SETGID`, resolves and adopts the completed identity, including supplementary +groups, and uses the kernel to verify that it can traverse every parent and +write and enter the workdir. The path must be a real directory without symlink +components. The probe also rejects kernel-managed filesystems and overlaps with +concrete OpenShell control resources. It emits a normalized identity +attestation; the final supervisor must resolve to the same identity, including +when the image supplies the default process policy. On failure, the driver +captures a bounded, sanitized diagnostic before removing the probe. Only then +does Podman mount and prepare the managed workspace volume at that path; normal +copy-up preserves image content. The workspace is the child cwd and `HOME`. For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 51dadced1d..ee4409ec46 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -25,6 +25,7 @@ const API_TIMEOUT: Duration = Duration::from_secs(30); /// Maximum allowed size for the event stream line buffer (1 MB). const MAX_EVENT_BUFFER: usize = 1_048_576; +const MAX_CONTAINER_LOG_BYTES: usize = 16 * 1024; #[derive(Debug, thiserror::Error)] pub enum PodmanApiError { @@ -352,6 +353,43 @@ impl PodmanClient { Ok((status, bytes)) } + /// Send a request while retaining at most `max_bytes` of the response. + async fn send_request_bounded( + &self, + req: Request>, + timeout: Duration, + max_bytes: usize, + ) -> Result<(hyper::StatusCode, Bytes), PodmanApiError> { + use hyper::body::Body; + + let mut sender = self.connect().await?; + let response = tokio::time::timeout(timeout, sender.send_request(req)) + .await + .map_err(|_| PodmanApiError::Timeout(timeout))? + .map_err(|error| PodmanApiError::Connection(error.to_string()))?; + let status = response.status(); + let deadline = tokio::time::Instant::now() + timeout; + let mut body = response.into_body(); + let mut bytes = Vec::with_capacity(max_bytes.min(4096)); + while bytes.len() < max_bytes { + let frame = tokio::time::timeout_at( + deadline, + std::future::poll_fn(|context| Pin::new(&mut body).poll_frame(context)), + ) + .await + .map_err(|_| PodmanApiError::Timeout(timeout))?; + let Some(frame) = frame else { + break; + }; + let frame = frame.map_err(|error| PodmanApiError::Connection(error.to_string()))?; + if let Some(data) = frame.data_ref() { + let remaining = max_bytes - bytes.len(); + bytes.extend_from_slice(&data[..data.len().min(remaining)]); + } + } + Ok((status, Bytes::from(bytes))) + } + /// Perform a versioned HTTP request and return status + body bytes. async fn request( &self, @@ -503,6 +541,27 @@ impl PodmanClient { .await } + /// Fetch a bounded tail of stdout and stderr from a container. + pub async fn container_logs(&self, name: &str) -> Result { + validate_name(name)?; + let req = Self::build_request( + hyper::Method::GET, + &format!( + "/{API_VERSION}/libpod/containers/{name}/logs?stdout=true&stderr=true&tail=20×tamps=false" + ), + Full::new(Bytes::new()), + None, + ); + let (status, bytes) = self + .send_request_bounded(req, API_TIMEOUT, MAX_CONTAINER_LOG_BYTES) + .await?; + if status.is_success() { + Ok(String::from_utf8_lossy(&bytes).into_owned()) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + /// List containers matching label filters (e.g. `&["openshell.managed=true"]`). pub async fn list_containers( &self, @@ -971,4 +1030,34 @@ mod tests { ); let _ = std::fs::remove_file(socket_path); } + + #[tokio::test] + async fn container_logs_fetches_bounded_stdout_and_stderr_tail() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "container-logs", + vec![StubResponse::new( + StatusCode::OK, + "workspace validation failed\n", + )], + ); + let client = PodmanClient::new(socket_path.clone()); + + let logs = client + .container_logs("workdir-probe") + .await + .expect("container logs should be returned"); + + assert_eq!(logs, "workspace validation failed\n"); + handle.await.expect("stub task should finish"); + assert_eq!( + request_log + .lock() + .expect("request log lock should not be poisoned") + .as_slice(), + [ + "GET /v5.0.0/libpod/containers/workdir-probe/logs?stdout=true&stderr=true&tail=20×tamps=false" + ] + ); + let _ = std::fs::remove_file(socket_path); + } } diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 567dcb9094..0ea8d0dff4 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -428,7 +428,9 @@ fn build_env( } } for key in [ + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE, openshell_core::sandbox_env::POLICY_RUN_AS_USER, openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, ] { @@ -494,7 +496,9 @@ fn build_env( env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); + env.remove(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE); env.remove(openshell_core::sandbox_env::POLICY_RUN_AS_USER); env.remove(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP); env.insert( @@ -940,6 +944,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( id: image.to_string(), config: None, }, + None, ) } @@ -950,6 +955,7 @@ pub fn build_container_spec_for_image( gpu_device_ids: Option<&[String]>, requested_image: &str, inspected_image: &ImageInspect, + workspace_identity: Option<&str>, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); @@ -1015,10 +1021,16 @@ pub fn build_container_spec_for_image( let mut command = vec!["--workdir".to_string(), workspace_root]; command.extend(upstream_proxy_cli_args(config)); - env.insert( - openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED.into(), - "1".into(), - ); + if let Some(identity) = workspace_identity { + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED.into(), + "1".into(), + ); + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.into(), + identity.into(), + ); + } let container_spec = ContainerSpec { name, @@ -1287,6 +1299,13 @@ pub fn build_workspace_probe_spec( command.push(value.clone()); } } + if spec + .environment + .get(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE) + .is_some_and(|value| value == "1") + { + command.push("--discover-policy-identity".to_string()); + } Ok(Some(serde_json::json!({ "name": probe_name, @@ -1306,6 +1325,7 @@ pub fn build_workspace_probe_spec( "netns": {"nsmode": "none"}, "no_new_privileges": true, "cap_drop": ["ALL"], + "cap_add": ["SETUID", "SETGID"], "image_pull_policy": "never" }))) } @@ -1517,6 +1537,7 @@ mod tests { None, "registry.example/app:latest", &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), + Some("1000:1000:"), ) .unwrap(); @@ -1555,6 +1576,10 @@ mod tests { container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED].as_str(), Some("1") ); + assert_eq!( + container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY].as_str(), + Some("1000:1000:") + ); } #[test] @@ -1566,6 +1591,7 @@ mod tests { None, "registry.example/app:latest", &inspected_image("sha256:immutable", "app:staff", "relative/workspace"), + None, ) .unwrap_err(); @@ -1588,6 +1614,7 @@ mod tests { "app:staff", "/opt/openshell/bin/project", ), + None, ) .unwrap_err(); @@ -1618,6 +1645,8 @@ mod tests { assert_eq!(probe["image"], "sha256:immutable"); assert_eq!(probe["netns"]["nsmode"], "none"); assert_eq!(probe["image_volume_mode"], "ignore"); + assert_eq!(probe["cap_drop"], serde_json::json!(["ALL"])); + assert_eq!(probe["cap_add"], serde_json::json!(["SETUID", "SETGID"])); assert!(probe.get("volumes").is_none()); assert!(probe.get("secrets").is_none()); assert!(probe.get("env").is_none()); @@ -1637,6 +1666,36 @@ mod tests { ); } + #[test] + fn workspace_probe_discovers_image_policy_identity_when_requested() { + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec.get_or_insert_default().environment.insert( + openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE.into(), + "1".into(), + ); + + let probe = build_workspace_probe_spec( + &sandbox, + &test_config(), + &inspected_image("sha256:immutable", "app:staff", "/home/app/project"), + "openshell-test-probe", + ) + .unwrap() + .unwrap(); + + assert_eq!( + probe["command"], + serde_json::json!([ + "validate-workspace", + "--workdir", + "/home/app/project", + "--oci-user", + "app:staff", + "--discover-policy-identity" + ]) + ); + } + #[test] fn workspace_probe_skips_sandbox_compatibility_fallback() { let probe = build_workspace_probe_spec( @@ -1677,6 +1736,7 @@ mod tests { None, "registry.example/app:latest", &inspected_image("sha256:immutable", "app:staff", "/workspace"), + None, ) .unwrap_err(); assert!( @@ -1704,6 +1764,7 @@ mod tests { None, "registry.example/app:latest", &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), + None, ) .unwrap_err(); assert!( @@ -1731,6 +1792,7 @@ mod tests { None, "registry.example/app:latest", &inspected_image("sha256:immutable", "app:staff", "/workspace"), + None, ) .expect("nested workspace mounts remain supported"); } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 7a565983af..afa26a51ad 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -221,6 +221,50 @@ fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError { ComputeDriverError::Precondition(err.to_string()) } +const WORKSPACE_IDENTITY_MARKER: &str = "OPENSHELL_WORKSPACE_IDENTITY="; + +fn parse_workspace_identity(logs: &str) -> Result { + let identity = logs + .lines() + .find_map(|line| line.trim().strip_prefix(WORKSPACE_IDENTITY_MARKER)) + .ok_or_else(|| { + ComputeDriverError::Precondition( + "OCI WorkingDir validation probe did not report its process identity".to_string(), + ) + })?; + let mut fields = identity.split(':'); + let uid = fields.next().unwrap_or_default(); + let gid = fields.next().unwrap_or_default(); + let groups = fields.next().unwrap_or_default(); + let valid = fields.next().is_none() + && !uid.is_empty() + && !gid.is_empty() + && uid.bytes().all(|byte| byte.is_ascii_digit()) + && gid.bytes().all(|byte| byte.is_ascii_digit()) + && groups + .split(',') + .all(|group| group.is_empty() || group.bytes().all(|byte| byte.is_ascii_digit())); + if !valid { + return Err(ComputeDriverError::Precondition( + "OCI WorkingDir validation probe reported an invalid process identity".to_string(), + )); + } + Ok(identity.to_string()) +} + +fn sanitized_probe_diagnostic(logs: &str) -> String { + let sanitized = logs + .chars() + .filter(|character| !character.is_control() || matches!(character, '\n' | '\t')) + .collect::(); + let sanitized = sanitized.trim(); + if sanitized.is_empty() { + "no probe diagnostic was emitted".to_string() + } else { + sanitized.chars().take(2048).collect() + } +} + /// Resolve the socket to connect to: explicit configuration wins, otherwise /// fall back to `detect`. Returns an error if neither resolves. /// @@ -247,7 +291,7 @@ impl PodmanComputeDriver { sandbox: &DriverSandbox, container_name: &str, inspected_image: &crate::client::ImageInspect, - ) -> Result<(), ComputeDriverError> { + ) -> Result, ComputeDriverError> { let probe_name = workspace_probe_name(container_name); crate::client::validate_name(&probe_name) .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; @@ -258,7 +302,7 @@ impl PodmanComputeDriver { &probe_name, )? else { - return Ok(()); + return Ok(None); }; let _lifecycle_guard = self.container_lifecycle_lock.lock().await; @@ -281,12 +325,18 @@ impl PodmanComputeDriver { if !inspect.state.running && matches!(inspect.state.status.as_str(), "exited" | "stopped") { + let logs = self.client.container_logs(&probe_name).await; return if inspect.state.exit_code == 0 { - Ok(()) + let logs = logs.map_err(ComputeDriverError::from)?; + parse_workspace_identity(&logs).map(Some) } else { + let diagnostic = logs.map_or_else( + |error| format!("unable to read probe diagnostic: {error}"), + |logs| sanitized_probe_diagnostic(&logs), + ); Err(ComputeDriverError::Precondition(format!( - "OCI WorkingDir validation failed for image '{}' (probe exited with code {})", - inspected_image.id, inspect.state.exit_code + "OCI WorkingDir validation failed for image '{}' (probe exited with code {}): {diagnostic}", + inspected_image.id, inspect.state.exit_code, ))) }; } @@ -302,8 +352,8 @@ impl PodmanComputeDriver { .await; let cleanup = self.client.remove_container(&probe_name).await; match (validation, cleanup) { - (Ok(()), Ok(())) => Ok(()), - (Ok(()), Err(error)) => Err(ComputeDriverError::from(error)), + (Ok(identity), Ok(())) => Ok(identity), + (Ok(_), Err(error)) => Err(ComputeDriverError::from(error)), (Err(error), Ok(())) => Err(error), (Err(error), Err(cleanup_error)) => { warn!( @@ -669,7 +719,8 @@ impl PodmanComputeDriver { "podman image '{image}' inspection did not return an immutable image ID" ))); } - self.validate_image_workspace(sandbox, &name, &inspected_image) + let workspace_identity = self + .validate_image_workspace(sandbox, &name, &inspected_image) .await?; for image in container::podman_driver_image_mount_sources(sandbox, self.config.enable_bind_mounts) @@ -736,6 +787,7 @@ impl PodmanComputeDriver { gpu_devices.as_deref(), image, &inspected_image, + workspace_identity.as_deref(), ) { Ok(spec) => spec, Err(e) => { @@ -1758,6 +1810,7 @@ mod tests { StatusCode::OK, r#"{"Id":"probe","Name":"probe","State":{"Status":"exited","Running":false,"ExitCode":0},"Config":{}}"#, ), + StubResponse::new(StatusCode::OK, "OPENSHELL_WORKSPACE_IDENTITY=1234:1235:\n"), StubResponse::new(StatusCode::NO_CONTENT, ""), ], ); @@ -1773,18 +1826,20 @@ mod tests { }; let name = validated_container_name(&sandbox).unwrap(); - driver + let identity = driver .validate_image_workspace(&sandbox, &name, &image) .await .expect("successful probe should pass"); + assert_eq!(identity.as_deref(), Some("1234:1235:")); handle.await.expect("stub task should finish"); let requests = request_log.lock().unwrap(); - assert_eq!(requests.len(), 4); + assert_eq!(requests.len(), 5); assert!(requests[0].contains("/libpod/containers/create")); assert!(requests[1].contains("/start")); assert!(requests[2].contains("/json")); - assert!(requests[3].starts_with("DELETE ")); + assert!(requests[3].contains("/logs?")); + assert!(requests[4].starts_with("DELETE ")); let _ = fs::remove_file(socket_path); } @@ -1799,6 +1854,10 @@ mod tests { StatusCode::OK, r#"{"Id":"probe","Name":"probe","State":{"Status":"exited","Running":false,"ExitCode":1},"Config":{}}"#, ), + StubResponse::new( + StatusCode::OK, + "image workspace path component '/workspace' is not writable and traversable\n", + ), StubResponse::new(StatusCode::NO_CONTENT, ""), ], ); @@ -1819,6 +1878,7 @@ mod tests { .await .unwrap_err(); assert!(error.to_string().contains("exited with code 1")); + assert!(error.to_string().contains("not writable and traversable")); handle.await.expect("stub task should finish"); let requests = request_log.lock().unwrap(); @@ -2009,3 +2069,13 @@ mod tests { let _ = fs::remove_file(socket_path); } } +#[test] +fn workspace_identity_parser_requires_a_valid_marker_line() { + assert_eq!( + parse_workspace_identity("probe startup\nOPENSHELL_WORKSPACE_IDENTITY=1234:1235:7,8\n") + .unwrap(), + "1234:1235:7,8" + ); + assert!(parse_workspace_identity("prefix OPENSHELL_WORKSPACE_IDENTITY=1234:1235:\n").is_err()); + assert!(parse_workspace_identity("OPENSHELL_WORKSPACE_IDENTITY=1234:bad:\n").is_err()); +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 604d100a82..93a104276b 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -190,27 +190,6 @@ pub async fn run_sandbox( // fill only omitted policy fields from OCI Config.User. #[cfg(unix)] let (resolved_process_identity, workdir_is_home) = { - if std::env::var_os(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED) - .is_some_and(|value| value == "1") - { - for (key, actual) in [ - ( - openshell_core::sandbox_env::POLICY_RUN_AS_USER, - policy.process.run_as_user.as_deref().unwrap_or(""), - ), - ( - openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, - policy.process.run_as_group.as_deref().unwrap_or(""), - ), - ] { - let expected = std::env::var(key).unwrap_or_default(); - if expected != actual { - return Err(miette::miette!( - "process identity changed after OCI workspace validation" - )); - } - } - } let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; let workdir_is_home = matches!( &driver_identity, @@ -220,6 +199,21 @@ pub async fn run_sandbox( &mut policy, &driver_identity, )?; + if std::env::var_os(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED) + .is_some_and(|value| value == "1") + { + let expected = std::env::var(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY) + .map_err(|_| miette::miette!("Podman workspace identity attestation is missing"))?; + let actual = + openshell_supervisor_process::process::resolved_workspace_identity_attestation( + &policy, resolved, + )?; + if expected != actual { + return Err(miette::miette!( + "process identity changed after OCI workspace validation" + )); + } + } (resolved, workdir_is_home) }; #[cfg(not(unix))] @@ -2140,6 +2134,18 @@ fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolic discover_policy_from_path(primary) } +/// Discover only the process identity portion of the immutable image policy. +/// +/// The Podman workspace probe uses this when no policy existed at create time, +/// matching the final supervisor's disk-policy discovery before the gateway +/// backfills that policy. +pub fn discover_process_policy_from_disk_or_default() -> openshell_core::policy::ProcessPolicy { + discover_policy_from_disk_or_default() + .process + .map(Into::into) + .unwrap_or_default() +} + /// Try to read a sandbox policy YAML from `path`, falling back to the /// hardcoded restrictive default if the file is missing or invalid. fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 76bbbb5023..59c467d54c 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -243,6 +243,8 @@ struct ValidateWorkspaceArgs { run_as_user: Option, #[arg(long)] run_as_group: Option, + #[arg(long)] + discover_policy_identity: bool, } #[cfg(unix)] @@ -256,15 +258,20 @@ fn validate_workspace(args: &[String]) -> Result<()> { std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), ) .into_diagnostic()?; + let process = if args.discover_policy_identity { + openshell_sandbox::discover_process_policy_from_disk_or_default() + } else { + ProcessPolicy { + run_as_user: args.run_as_user, + run_as_group: args.run_as_group, + } + }; let mut policy = SandboxPolicy { version: 0, filesystem: FilesystemPolicy::default(), network: NetworkPolicy::default(), landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: args.run_as_user, - run_as_group: args.run_as_group, - }, + process, }; let resolved_identity = resolve_process_identity( &mut policy, @@ -272,11 +279,14 @@ fn validate_workspace(args: &[String]) -> Result<()> { declaration: args.oci_user, }, )?; - openshell_supervisor_process::process::validate_oci_workspace_with_process_identity( - &policy, - resolved_identity, - Path::new(&args.workdir), - ) + let identity = + openshell_supervisor_process::process::validate_oci_workspace_as_process_identity( + &policy, + resolved_identity, + Path::new(&args.workdir), + )?; + println!("OPENSHELL_WORKSPACE_IDENTITY={identity}"); + Ok(()) } #[cfg(not(unix))] @@ -709,6 +719,14 @@ mod tests { #[cfg(unix)] #[test] fn workspace_validation_subcommand_uses_final_policy_identity() { + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid < 1000 || gid < 1000 { + // Process policies intentionally reject system identities. Local + // macOS users commonly have IDs below 1000, so this test cannot + // exercise the credential transition for those accounts. + return; + } let dir = tempfile::tempdir_in("/tmp").unwrap(); std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); let root = dir.path().canonicalize().unwrap().join("workspace"); @@ -720,9 +738,9 @@ mod tests { "--oci-user".to_string(), "unused".to_string(), "--run-as-user".to_string(), - "1000".to_string(), + uid.to_string(), "--run-as-group".to_string(), - "1000".to_string(), + gid.to_string(), ]; validate_workspace(&args).expect("current identity should retain workspace authority"); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 9814f40aa9..faab932015 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -2167,6 +2167,10 @@ fn driver_sandbox_spec_from_public( openshell_core::sandbox_env::POLICY_RUN_AS_GROUP.to_string(), process.map_or_else(String::new, |process| process.run_as_group.clone()), ); + environment.insert( + openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE.to_string(), + u8::from(spec.policy.is_none()).to_string(), + ); } Ok(DriverSandboxSpec { log_level: spec.log_level.clone(), @@ -2956,6 +2960,23 @@ mod tests { .map(String::as_str), Some("staff") ); + assert_eq!( + driver + .environment + .get(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE) + .map(String::as_str), + Some("0") + ); + + let without_policy = + driver_sandbox_spec_from_public(&SandboxSpec::default(), "podman").unwrap(); + assert_eq!( + without_policy + .environment + .get(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE) + .map(String::as_str), + Some("1") + ); } #[test] diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index a0cdae22fb..40c1735efe 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -113,7 +113,9 @@ pub(crate) fn prepare_child_sandbox( const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE, openshell_core::sandbox_env::POLICY_RUN_AS_USER, openshell_core::sandbox_env::POLICY_RUN_AS_GROUP, openshell_core::sandbox_env::SANDBOX_UID, @@ -1347,6 +1349,98 @@ pub fn validate_oci_workspace_with_process_identity( validate_oci_workspace(workdir, uid, gid, &supplementary_gids) } +/// Drop a one-shot workspace probe to the completed sandbox identity, validate +/// using the kernel's real path-access checks, and return a normalized identity +/// attestation for the final supervisor. +#[cfg(unix)] +pub fn validate_oci_workspace_as_process_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + workdir: &Path, +) -> Result { + validate_sandbox_user_with_identity(policy, resolved_identity)?; + validate_sandbox_group_with_identity(policy, resolved_identity)?; + let (uid, gid, mut supplementary_gids) = + resolve_filesystem_identity(policy, resolved_identity)?; + let uid = uid.ok_or_else(|| miette::miette!("workspace probe UID is unresolved"))?; + let gid = gid.ok_or_else(|| miette::miette!("workspace probe GID is unresolved"))?; + supplementary_gids.sort_unstable_by_key(|group| group.as_raw()); + supplementary_gids.dedup(); + + if nix::unistd::geteuid().is_root() { + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "haiku", + target_os = "redox" + )))] + nix::unistd::setgroups(&supplementary_gids).into_diagnostic()?; + nix::unistd::setgid(gid).into_diagnostic()?; + nix::unistd::setuid(uid).into_diagnostic()?; + } + + let effective_identity = (nix::unistd::geteuid(), nix::unistd::getegid()); + if effective_identity != (uid, gid) { + return Err(miette::miette!( + "workspace probe privilege drop failed: expected {uid}:{gid}, got {}:{}", + effective_identity.0, + effective_identity.1 + )); + } + validate_oci_workspace( + workdir, + Some(effective_identity.0), + Some(effective_identity.1), + &supplementary_gids, + )?; + Ok(workspace_identity_attestation( + Some(uid), + Some(gid), + &supplementary_gids, + )) +} + +/// Normalize a completed process identity for comparison with the immutable +/// image probe. +#[cfg(unix)] +pub fn workspace_identity_attestation( + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> String { + let mut groups = supplementary_gids + .iter() + .map(|group| group.as_raw()) + .collect::>(); + groups.sort_unstable(); + groups.dedup(); + format!( + "{}:{}:{}", + uid.map_or_else(String::new, |value| value.as_raw().to_string()), + gid.map_or_else(String::new, |value| value.as_raw().to_string()), + groups + .iter() + .map(u32::to_string) + .collect::>() + .join(",") + ) +} + +/// Resolve and normalize the completed policy/driver identity without changing +/// process credentials. +#[cfg(unix)] +pub fn resolved_workspace_identity_attestation( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result { + let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; + Ok(workspace_identity_attestation( + uid, + gid, + &supplementary_gids, + )) +} + #[cfg(unix)] fn validate_workspace_component( path: &Path, @@ -1399,9 +1493,6 @@ fn validate_workspace_component( #[cfg(target_os = "linux")] fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { - let fs = rustix::fs::statfs(path).into_diagnostic()?; - #[allow(clippy::cast_sign_loss)] - let filesystem_type = fs.f_type as u64; // Linux filesystem magic values for virtual/kernel-managed filesystems. // The decision is based on the mounted filesystem, not its conventional // distro path, so renamed or unusually mounted control filesystems remain @@ -1411,7 +1502,7 @@ fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { 0x0000_9fa0, // proc 0x0102_1994, // tmpfs (including the container /dev tree) 0x1980_0202, // mqueue - 0x27e0_eb, // cgroup + 0x0027_e0eb, // cgroup 0x4249_4e4d, // bpf 0x6265_6572, // sysfs 0x6367_7270, // cgroup2 @@ -1419,6 +1510,9 @@ fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { 0x7363_6673, // securityfs 0x7472_6163, // tracefs ]; + let fs = rustix::fs::statfs(path).into_diagnostic()?; + #[allow(clippy::cast_sign_loss)] + let filesystem_type = fs.f_type as u64; if SPECIAL_FILESYSTEMS.contains(&filesystem_type) { return Err(miette::miette!( "workspace path component '{}' is on a kernel-managed filesystem (type {filesystem_type:#x})", @@ -2898,6 +2992,23 @@ mod tests { .expect("supplementary group already has write and traverse authority"); } + #[cfg(unix)] + #[test] + fn workspace_identity_attestation_sorts_and_deduplicates_groups() { + assert_eq!( + workspace_identity_attestation( + Some(Uid::from_raw(1000)), + Some(Gid::from_raw(1001)), + &[ + Gid::from_raw(1003), + Gid::from_raw(1002), + Gid::from_raw(1003), + ], + ), + "1000:1001:1002,1003" + ); + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_unwritable_directory() { diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index edf83d250b..55f91c1ac2 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -446,11 +446,15 @@ ownership or mode. It rejects kernel-managed filesystems based on filesystem metadata and rejects overlap with actual OpenShell control paths. Docker checks the original image filesystem in the final supervisor. Podman checks the same pinned image ID in a temporary networkless probe without the workspace volume, -token, or TLS secrets, removes the probe, and only then creates the final -volume-backed sandbox. The resolved workspace is the cwd and `HOME` for direct -and SSH children. The supervisor itself starts from `/`, so a missing or -invalid workspace is handled during readiness instead of preventing the -container runtime from starting it. +token, or TLS secrets. The probe retains only `SETUID` and `SETGID`, drops to +the completed process identity, and uses the kernel to validate access. The +final supervisor must match the probe's normalized identity attestation, +including when the image supplies the default process policy. Podman captures +a bounded, sanitized failure diagnostic, removes the probe, and only then +creates the final volume-backed sandbox. The resolved workspace is the cwd and +`HOME` for direct and SSH children. The supervisor itself starts from `/`, so a +missing or invalid workspace is handled during readiness instead of preventing +the container runtime from starting it. Sandbox creation fails before readiness if a required `USER` component is missing, malformed, unknown, ambiguous, or resolves to UID/GID 0. An image diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 47cca331b0..8dcc25155c 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -57,7 +57,10 @@ impl ImageGuard { format!( "FROM {BASE_IMAGE}\n\ USER 0:0\n\ - WORKDIR /workspace/project\n\ + RUN mkdir -p /home/app/project && \ + chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ + chmod 0700 /home/app\n\ + WORKDIR /home/app/project\n\ RUN printf root-owned > root-owned.txt && chown {OCI_UID}:{OCI_GID} .\n\ USER {OCI_UID}:{OCI_GID}\n" ), @@ -189,8 +192,8 @@ async fn podman_uses_oci_identity_workspace_and_inspected_image_id() { "sh", "-c", "set -eu; \ - test \"$(pwd -P)\" = /workspace/project; \ - test \"$HOME\" = /workspace/project; \ + test \"$(pwd -P)\" = /home/app/project; \ + test \"$HOME\" = /home/app/project; \ test \"$(cat root-owned.txt)\" = root-owned; \ test \"$(stat -c %u:%g .)\" = 2345:2346; \ test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ @@ -215,8 +218,8 @@ async fn podman_uses_oci_identity_workspace_and_inspected_image_id() { "-c", "set -eu; \ test \"$(id -u):$(id -g)\" = 2345:2346; \ - test \"$(pwd -P)\" = /workspace/project; \ - test \"$HOME\" = /workspace/project; \ + test \"$(pwd -P)\" = /home/app/project; \ + test \"$HOME\" = /home/app/project; \ test -f direct-workspace-write; \ touch ssh-workspace-write; \ echo podman-ssh-identity-ok", From ed0619ac55022bc17ac1be741c455112aec43337 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 29 Jul 2026 16:04:45 -0700 Subject: [PATCH 15/16] fix(podman): narrow workspace probe lifecycle lock Signed-off-by: Matthew Grossman --- .github/workflows/e2e-test.yml | 6 +- crates/openshell-driver-podman/src/driver.rs | 138 +++++++++++++++---- 2 files changed, 113 insertions(+), 31 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index d8f33f7016..5a2ae8e0c1 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -131,10 +131,10 @@ jobs: fail-fast: false matrix: include: - # Ubuntu 24.04 matches the environment reported in #2069 and ships - # Podman 4.x. The probe records whether AppArmor blocks the drop. + # Ubuntu 24.04 matches the AppArmor environment reported in #2069. + # GitHub's runner image now supplies the supported Podman 5.x line. - runner: ubuntu-24.04 - podman_major: "4" + podman_major: "5" # Ubuntu 26.04 provides the supported Podman 5.x coverage for # comparison with the Ubuntu 24.04 environment. - runner: ubuntu-26.04 diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index afa26a51ad..eff2d5e6c8 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -225,31 +225,34 @@ const WORKSPACE_IDENTITY_MARKER: &str = "OPENSHELL_WORKSPACE_IDENTITY="; fn parse_workspace_identity(logs: &str) -> Result { let identity = logs - .lines() - .find_map(|line| line.trim().strip_prefix(WORKSPACE_IDENTITY_MARKER)) + .match_indices(WORKSPACE_IDENTITY_MARKER) + .filter_map(|(start, _)| { + logs[start + WORKSPACE_IDENTITY_MARKER.len()..] + .split(|character: char| character.is_control()) + .next() + }) + .find(|identity| workspace_identity_is_valid(identity)) .ok_or_else(|| { ComputeDriverError::Precondition( "OCI WorkingDir validation probe did not report its process identity".to_string(), ) })?; + Ok(identity.to_string()) +} + +fn workspace_identity_is_valid(identity: &str) -> bool { let mut fields = identity.split(':'); let uid = fields.next().unwrap_or_default(); let gid = fields.next().unwrap_or_default(); let groups = fields.next().unwrap_or_default(); - let valid = fields.next().is_none() + fields.next().is_none() && !uid.is_empty() && !gid.is_empty() && uid.bytes().all(|byte| byte.is_ascii_digit()) && gid.bytes().all(|byte| byte.is_ascii_digit()) && groups .split(',') - .all(|group| group.is_empty() || group.bytes().all(|byte| byte.is_ascii_digit())); - if !valid { - return Err(ComputeDriverError::Precondition( - "OCI WorkingDir validation probe reported an invalid process identity".to_string(), - )); - } - Ok(identity.to_string()) + .all(|group| group.is_empty() || group.bytes().all(|byte| byte.is_ascii_digit())) } fn sanitized_probe_diagnostic(logs: &str) -> String { @@ -305,16 +308,19 @@ impl PodmanComputeDriver { return Ok(None); }; - let _lifecycle_guard = self.container_lifecycle_lock.lock().await; - self.client - .create_container(&spec) - .await - .map_err(ComputeDriverError::from)?; - let validation = async { + let start = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; self.client - .start_container(&probe_name) + .create_container(&spec) .await .map_err(ComputeDriverError::from)?; + self.client + .start_container(&probe_name) + .await + .map_err(ComputeDriverError::from) + }; + let validation = async { + start?; let deadline = tokio::time::Instant::now() + Duration::from_secs(30); loop { let inspect = self @@ -350,7 +356,10 @@ impl PodmanComputeDriver { } } .await; - let cleanup = self.client.remove_container(&probe_name).await; + let cleanup = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + self.client.remove_container(&probe_name).await + }; match (validation, cleanup) { (Ok(identity), Ok(())) => Ok(identity), (Ok(_), Err(error)) => Err(ComputeDriverError::from(error)), @@ -1843,6 +1852,68 @@ mod tests { let _ = fs::remove_file(socket_path); } + #[tokio::test] + async fn workspace_probe_releases_lifecycle_lock_while_polling() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "workspace-probe-lock-scope", + vec![ + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"probe","Name":"probe","State":{"Status":"running","Running":true,"ExitCode":0},"Config":{}}"#, + ), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"probe","Name":"probe","State":{"Status":"exited","Running":false,"ExitCode":0},"Config":{}}"#, + ), + StubResponse::new(StatusCode::OK, "OPENSHELL_WORKSPACE_IDENTITY=1234:1235:\n"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + let mut sandbox = plain_sandbox("sandbox-probe-lock", "demo"); + sandbox.spec = Some(DriverSandboxSpec::default()); + let image = crate::client::ImageInspect { + id: "sha256:immutable".into(), + config: Some(crate::client::ImageConfig { + user: "1234:1235".into(), + working_dir: "/workspace".into(), + }), + }; + let name = validated_container_name(&sandbox).unwrap(); + let probe_driver = driver.clone(); + let probe = tokio::spawn(async move { + probe_driver + .validate_image_workspace(&sandbox, &name, &image) + .await + }); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if request_log.lock().unwrap().len() >= 3 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("probe should enter its polling interval"); + + let unrelated_guard = tokio::time::timeout( + Duration::from_millis(25), + driver.container_lifecycle_lock.lock(), + ) + .await + .expect("unrelated lifecycle work should proceed while the probe polls"); + drop(unrelated_guard); + + let identity = probe.await.unwrap().unwrap(); + assert_eq!(identity.as_deref(), Some("1234:1235:")); + handle.await.expect("stub task should finish"); + let _ = fs::remove_file(socket_path); + } + #[tokio::test] async fn workspace_probe_removes_container_after_validation_failure() { let (socket_path, request_log, handle) = spawn_podman_stub( @@ -2068,14 +2139,25 @@ mod tests { ); let _ = fs::remove_file(socket_path); } -} -#[test] -fn workspace_identity_parser_requires_a_valid_marker_line() { - assert_eq!( - parse_workspace_identity("probe startup\nOPENSHELL_WORKSPACE_IDENTITY=1234:1235:7,8\n") - .unwrap(), - "1234:1235:7,8" - ); - assert!(parse_workspace_identity("prefix OPENSHELL_WORKSPACE_IDENTITY=1234:1235:\n").is_err()); - assert!(parse_workspace_identity("OPENSHELL_WORKSPACE_IDENTITY=1234:bad:\n").is_err()); + + #[test] + fn workspace_identity_parser_accepts_plain_and_multiplexed_logs() { + assert_eq!( + parse_workspace_identity("probe startup\nOPENSHELL_WORKSPACE_IDENTITY=1234:1235:7,8\n") + .unwrap(), + "1234:1235:7,8" + ); + assert_eq!( + parse_workspace_identity("\u{1}\0\0\0\0\0\0.OPENSHELL_WORKSPACE_IDENTITY=1234:1235:\n") + .unwrap(), + "1234:1235:" + ); + assert!( + parse_workspace_identity( + "OPENSHELL_WORKSPACE_IDENTITY=1234:bad:\n\ + OPENSHELL_WORKSPACE_IDENTITY=1234:1235:bad\n" + ) + .is_err() + ); + } } From 59f179e527872ac13d430bab814586a00b6559f5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 30 Jul 2026 09:40:13 -0700 Subject: [PATCH 16/16] fix(sandbox): harden OCI workspace validation Signed-off-by: Matthew Grossman --- .../skills/debug-openshell-cluster/SKILL.md | 1 + architecture/compute-runtimes.md | 3 +- crates/openshell-driver-docker/README.md | 3 + crates/openshell-driver-docker/src/lib.rs | 33 ++- crates/openshell-driver-docker/src/tests.rs | 44 ++++ .../openshell-driver-kubernetes/src/driver.rs | 25 +- .../openshell-driver-podman/src/container.rs | 44 +++- .../src/identity.rs | 91 +++++++ .../src/process.rs | 237 +++++++++++++----- docs/reference/sandbox-compute-drivers.mdx | 4 +- e2e/rust/tests/custom_image.rs | 4 + 11 files changed, 414 insertions(+), 75 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 3dbcdc38a8..d4bdd9b515 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -150,6 +150,7 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, or supplementary groups must already be able to traverse every parent and write and enter the directory. OpenShell does not create, chown, or chmod a non-default image workdir. +- Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify `OPENSHELL_DOCKER_SUPERVISOR_BIN`, the sibling binary next to `openshell-gateway`, or the configured supervisor image contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and supervisor callback endpoint. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 31bcc9df79..7eb34967c0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -207,7 +207,8 @@ identity, including supplementary groups, must already be able to traverse every parent and write and enter the workdir; OpenShell does not change that directory's ownership or mode. Filesystem metadata identifies kernel-managed mounts, while collision checks are derived from actual OpenShell control paths. -Docker performs the check in the final container before workload launch. +Docker performs the check in the final container before workload launch and +rejects image `VOLUME` declarations that would mask the workdir ancestry. Podman performs it in a minimal networkless container from the same pinned image ID before its managed workspace volume covers the path. The probe retains only the capabilities needed to adopt the completed process identity, drops to diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 0f659e3bee..0123a6ee85 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -34,6 +34,9 @@ creates when necessary and owns as a compatibility workspace. Any other image wo already exist without symlink components. The completed identity, including supplementary groups, must already be able to traverse every parent and write and enter the workdir. OpenShell does not change its ownership or mode. +Image `VOLUME` declarations must not cover the workdir or one of its parents +because Docker would mount the volume before the supervisor could validate the +immutable image path. Kernel-managed filesystems are detected from filesystem metadata, and paths that overlap concrete OpenShell control resources are rejected. The workspace is the child cwd and `HOME`. The supervisor starts from `/`, then reports an diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 75603a57de..4cbc9cfbeb 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -220,6 +220,7 @@ struct DockerImageMetadata { id: String, user: String, working_dir: String, + volumes: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -1323,12 +1324,13 @@ impl DockerComputeDriver { "docker image '{image}' inspection did not return an immutable image ID" )) })?; - let (user, working_dir) = inspect.config.map_or_else( - || (String::new(), String::new()), + let (user, working_dir, volumes) = inspect.config.map_or_else( + || (String::new(), String::new(), Vec::new()), |config| { ( config.user.unwrap_or_default(), config.working_dir.unwrap_or_default(), + config.volumes.unwrap_or_default(), ) }, ); @@ -1336,6 +1338,7 @@ impl DockerComputeDriver { id, user, working_dir, + volumes, }) } @@ -2244,11 +2247,20 @@ fn build_environment_for_oci_user( environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - environment.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); - environment.remove(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); environment.remove(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE); environment.remove(openshell_core::sandbox_env::POLICY_RUN_AS_USER); environment.remove(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP); + // Docker never uses the Podman prevalidation contract. Emit explicit + // driver-owned values so image-baked ENV entries cannot select the + // managed-workspace mutation path for an image-provided workdir. + environment.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED.to_string(), + "0".to_string(), + ); + environment.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.to_string(), + String::new(), + ); environment.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), oci_user.to_string(), @@ -2361,6 +2373,7 @@ fn build_container_create_body_with_gpu_devices( id: template.image.clone(), user: String::new(), working_dir: String::new(), + volumes: Vec::new(), }, ) } @@ -2385,6 +2398,18 @@ fn build_container_create_body_for_image( .map_err(Status::failed_precondition)?; driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + for volume in &image.volumes { + driver_mounts::validate_container_mount_target(volume).map_err(|error| { + Status::failed_precondition(format!( + "invalid image-declared volume '{volume}': {error}" + )) + })?; + driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err(|_| { + Status::failed_precondition(format!( + "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" + )) + })?; + } for mount in &driver_config.mounts { let target = match mount { DockerDriverMountConfig::Bind { target, .. } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 536c9b193c..25db62703d 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -540,6 +540,11 @@ fn build_environment_protects_oci_identity_metadata() { (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), (openshell_core::sandbox_env::SANDBOX_UID, "9999"), (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + (openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, "1"), + ( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, + "9999:9999:", + ), ] { spec.environment.insert(key.to_string(), value.to_string()); } @@ -552,6 +557,14 @@ fn build_environment_protects_oci_identity_metadata() { ))); assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); + assert!(env.contains(&format!( + "{}=0", + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED + ))); + assert!(env.contains(&format!( + "{}=", + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY + ))); assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); } @@ -563,6 +576,7 @@ fn container_creation_uses_inspected_immutable_image() { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), working_dir: "/workspace/project".to_string(), + volumes: Vec::new(), }; let body = build_container_create_body_for_image( &sandbox, @@ -592,6 +606,7 @@ fn container_creation_rejects_invalid_oci_working_dir() { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), working_dir: "relative/workspace".to_string(), + volumes: Vec::new(), }; let err = build_container_create_body_for_image( &test_sandbox(), @@ -612,6 +627,7 @@ fn container_creation_rejects_openshell_control_path_working_dir() { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), working_dir: "/opt/openshell/bin/project".to_string(), + volumes: Vec::new(), }; let err = build_container_create_body_for_image( &test_sandbox(), @@ -626,12 +642,39 @@ fn container_creation_rejects_openshell_control_path_working_dir() { assert!(err.message().contains("OpenShell control path")); } +#[test] +fn container_creation_rejects_image_volume_that_masks_working_dir() { + let sandbox = test_sandbox(); + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/workspace/project".to_string(), + volumes: vec!["/workspace".to_string()], + }; + + let error = build_container_create_body_for_image( + &sandbox, + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + ) + .unwrap_err(); + + assert!( + error + .message() + .contains("masks OCI WorkingDir '/workspace/project'") + ); +} + #[test] fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { let metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), working_dir: "/workspace".to_string(), + volumes: Vec::new(), }; let root_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ "mounts": [{"type": "tmpfs", "target": "/workspace"}] @@ -656,6 +699,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( .unwrap(); let nested_metadata = DockerImageMetadata { working_dir: "/workspace/project".to_string(), + volumes: Vec::new(), ..metadata.clone() }; let err = build_container_create_body_for_image( diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 56361ef59c..8b8f5a7353 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3025,7 +3025,15 @@ fn apply_resolved_identity_env(env: &mut Vec, uid: u32, gid: remove_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER); remove_env(env, openshell_core::sandbox_env::SANDBOX_UID); remove_env(env, openshell_core::sandbox_env::SANDBOX_GID); + remove_env(env, openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); + remove_env(env, openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); upsert_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER, ""); + upsert_env( + env, + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + "0", + ); + upsert_env(env, openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, ""); upsert_env( env, openshell_core::sandbox_env::SANDBOX_UID, @@ -3940,7 +3948,9 @@ mod tests { {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "spoofed"}, {"name": openshell_core::sandbox_env::SANDBOX_UID, "value": "9999"}, {"name": openshell_core::sandbox_env::SANDBOX_GID, "value": "9999"}, - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "duplicate"} + {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "duplicate"}, + {"name": openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, "value": "1"}, + {"name": openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, "value": "9999:9999:"} ] }] } @@ -3961,6 +3971,8 @@ mod tests { openshell_core::sandbox_env::OCI_IMAGE_USER, openshell_core::sandbox_env::SANDBOX_UID, openshell_core::sandbox_env::SANDBOX_GID, + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, ] { assert_eq!( env.iter().filter(|item| item["name"] == name).count(), @@ -3980,6 +3992,17 @@ mod tests { rendered_env(agent, openshell_core::sandbox_env::SANDBOX_GID), Some("1600") ); + assert_eq!( + rendered_env( + agent, + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED + ), + Some("0") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY), + Some("") + ); } #[test] diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 0ea8d0dff4..cb8ad563ee 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -496,11 +496,20 @@ fn build_env( env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); - env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED); env.remove(openshell_core::sandbox_env::POLICY_IDENTITY_FROM_IMAGE); env.remove(openshell_core::sandbox_env::POLICY_RUN_AS_USER); env.remove(openshell_core::sandbox_env::POLICY_RUN_AS_GROUP); + // The final spec overwrites these only after a successful immutable-image + // probe. Explicit false/empty defaults prevent image ENV from forging the + // attestation contract on the /sandbox compatibility path. + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED.into(), + "0".into(), + ); + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.into(), + String::new(), + ); env.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.into(), oci_user.to_string(), @@ -1526,6 +1535,14 @@ mod tests { (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), (openshell_core::sandbox_env::SANDBOX_UID, "9999"), (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + ( + openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED, + "forged", + ), + ( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, + "9999:9999:", + ), ] { spec.environment.insert(key.to_string(), value.to_string()); } @@ -1582,6 +1599,29 @@ mod tests { ); } + #[test] + fn compatibility_workspace_emits_non_prevalidated_attestation() { + let container = build_container_spec_for_image( + &test_sandbox("test-id", "test-name"), + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", ""), + None, + ) + .unwrap(); + + assert_eq!( + container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_PREVALIDATED].as_str(), + Some("0") + ); + assert_eq!( + container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY].as_str(), + Some("") + ); + } + #[test] fn container_spec_rejects_invalid_oci_working_dir() { let err = build_container_spec_for_image( diff --git a/crates/openshell-supervisor-process/src/identity.rs b/crates/openshell-supervisor-process/src/identity.rs index 6a9a785542..543d1245c8 100644 --- a/crates/openshell-supervisor-process/src/identity.rs +++ b/crates/openshell-supervisor-process/src/identity.rs @@ -332,6 +332,68 @@ fn find_group_by_name(path: &Path, name: &str) -> Result> { }) } +/// Resolve supplementary groups declared for an OCI named user without +/// consulting NSS. Numeric OCI users have no trustworthy group-membership +/// name and therefore receive no supplementary groups. +pub fn resolve_oci_supplementary_gids(declaration: &str, primary_gid: u32) -> Result> { + resolve_oci_supplementary_gids_at(declaration, primary_gid, Path::new(GROUP_PATH)) +} + +fn resolve_oci_supplementary_gids_at( + declaration: &str, + primary_gid: u32, + group_path: &Path, +) -> Result> { + let (user, _) = split_oci_declaration(declaration); + validate_component(user, "OCI user")?; + if user.parse::().is_ok() { + return Ok(Vec::new()); + } + + let content = read_account_file(group_path)?; + let mut gids = vec![primary_gid]; + for line in content.lines() { + if line.is_empty() || line.starts_with('#') { + continue; + } + if line.len() > MAX_ACCOUNT_LINE_SIZE { + return Err(miette::miette!( + "account file '{}' contains an oversized line", + group_path.display() + )); + } + let fields = line.split(':').collect::>(); + if fields.len() != 4 + || fields + .iter() + .any(|field| field.len() > MAX_ACCOUNT_FIELD_SIZE) + { + return Err(miette::miette!( + "group membership entry in '{}' is malformed", + group_path.display() + )); + } + if !fields[3].split(',').any(|member| member == user) { + continue; + } + let gid = fields[2].parse::().map_err(|_| { + miette::miette!( + "group membership GID in '{}' is malformed", + group_path.display() + ) + })?; + if gid == 0 { + return Err(miette::miette!( + "OCI user '{user}' is a member of prohibited GID 0" + )); + } + gids.push(gid); + } + gids.sort_unstable(); + gids.dedup(); + Ok(gids) +} + fn find_unique( path: &Path, mut select: impl FnMut(&[&str]) -> Option>, @@ -665,6 +727,35 @@ mod tests { ); } + #[test] + fn named_oci_user_resolves_bounded_supplementary_groups() { + let (_dir, _passwd, group) = account_files( + "", + "primary:x:1235:\nvideo:x:44:app,other\naudio:x:63:other\nrender:x:107:app\n", + ); + + let gids = resolve_oci_supplementary_gids_at("app:primary", 1235, &group).unwrap(); + assert_eq!(gids, vec![44, 107, 1235]); + } + + #[test] + fn numeric_oci_user_has_no_named_supplementary_groups() { + let dir = tempdir().unwrap(); + let missing_group = dir.path().join("missing-group"); + + let gids = resolve_oci_supplementary_gids_at("1234:1235", 1235, &missing_group).unwrap(); + assert!(gids.is_empty()); + } + + #[test] + fn oci_supplementary_membership_rejects_root_group() { + let (_dir, _passwd, group) = account_files("", "root:x:0:app\n"); + + let error = + resolve_oci_supplementary_gids_at("app", 1235, &group).expect_err("GID 0 must fail"); + assert!(error.to_string().contains("prohibited GID 0")); + } + #[test] fn missing_unknown_ambiguous_and_root_identities_fail() { let (_dir, passwd, group) = account_files( diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 40c1735efe..72225a417c 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -1387,6 +1387,9 @@ pub fn validate_oci_workspace_as_process_identity( effective_identity.1 )); } + #[cfg(target_os = "linux")] + validate_oci_workspace_as_effective_identity(workdir)?; + #[cfg(not(target_os = "linux"))] validate_oci_workspace( workdir, Some(effective_identity.0), @@ -1491,12 +1494,143 @@ fn validate_workspace_component( Ok(()) } +#[cfg(target_os = "linux")] +fn validate_oci_workspace_as_effective_identity(root: &Path) -> Result<()> { + use rustix::fs::{Access, AtFlags, FileType, Mode, OFlags}; + + let root_str = root + .to_str() + .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; + let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) + .map_err(|error| miette::miette!(error))?; + if Path::new(&validated_root) != root + || validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT + { + return Err(miette::miette!( + "image workspace path '{}' must be a normalized absolute non-fallback path", + root.display() + )); + } + + let open_flags = OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let mut current_path = PathBuf::from("/"); + let mut current_fd = rustix::fs::open("/", open_flags, Mode::empty()).into_diagnostic()?; + reject_special_workspace_filesystem_fd(¤t_fd, ¤t_path)?; + rustix::fs::accessat( + ¤t_fd, + ".", + Access::EXEC_OK, + AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(|error| { + miette::miette!( + "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", + current_path.display() + ) + })?; + + let components = root + .components() + .skip(1) + .map(|component| match component { + std::path::Component::Normal(component) => Ok(component), + _ => Err(miette::miette!( + "workspace path '{}' must be normalized", + root.display() + )), + }) + .collect::>>()?; + let last_component = components.len().saturating_sub(1); + + for (index, component) in components.into_iter().enumerate() { + current_path.push(component); + let stat = rustix::fs::statat(¤t_fd, component, AtFlags::SYMLINK_NOFOLLOW).map_err( + |error| { + if error == rustix::io::Errno::NOENT { + miette::miette!( + "image workspace path component '{}' does not exist", + current_path.display() + ) + } else { + miette::miette!( + "failed to inspect image workspace path component '{}': {error}", + current_path.display() + ) + } + }, + )?; + let file_type = FileType::from_raw_mode(stat.st_mode); + if file_type.is_symlink() { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + current_path.display() + )); + } + if !file_type.is_dir() { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + current_path.display() + )); + } + + let is_workspace = index == last_component; + let access = if is_workspace { + Access::WRITE_OK | Access::EXEC_OK + } else { + Access::EXEC_OK + }; + rustix::fs::accessat( + ¤t_fd, + component, + access, + AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(|error| { + let requirement = if is_workspace { + "writable and traversable" + } else { + "traversable" + }; + miette::miette!( + "workspace path component '{}' is not {requirement} by the sandbox identity in the image: {error}", + current_path.display() + ) + })?; + + current_fd = rustix::fs::openat(¤t_fd, component, open_flags, Mode::empty()) + .map_err(|error| { + miette::miette!( + "failed to open image workspace path component '{}': {error}", + current_path.display() + ) + })?; + reject_special_workspace_filesystem_fd(¤t_fd, ¤t_path)?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn reject_special_workspace_filesystem_fd(fd: &impl std::os::fd::AsFd, path: &Path) -> Result<()> { + let fs = rustix::fs::fstatfs(fd).into_diagnostic()?; + #[allow(clippy::cast_sign_loss)] + reject_special_workspace_filesystem_type(path, fs.f_type as u64) +} + #[cfg(target_os = "linux")] fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { // Linux filesystem magic values for virtual/kernel-managed filesystems. // The decision is based on the mounted filesystem, not its conventional // distro path, so renamed or unusually mounted control filesystems remain // protected without rejecting ordinary application directories. + let fs = rustix::fs::statfs(path).into_diagnostic()?; + #[allow(clippy::cast_sign_loss)] + let filesystem_type = fs.f_type as u64; + reject_special_workspace_filesystem_type(path, filesystem_type) +} + +#[cfg(target_os = "linux")] +fn reject_special_workspace_filesystem_type(path: &Path, filesystem_type: u64) -> Result<()> { const SPECIAL_FILESYSTEMS: &[u64] = &[ 0x0000_1cd1, // devpts 0x0000_9fa0, // proc @@ -1510,9 +1644,6 @@ fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { 0x7363_6673, // securityfs 0x7472_6163, // tracefs ]; - let fs = rustix::fs::statfs(path).into_diagnostic()?; - #[allow(clippy::cast_sign_loss)] - let filesystem_type = fs.f_type as u64; if SPECIAL_FILESYSTEMS.contains(&filesystem_type) { return Err(miette::miette!( "workspace path component '{}' is on a kernel-managed filesystem (type {filesystem_type:#x})", @@ -1859,7 +1990,7 @@ fn resolve_filesystem_identity( }; let supplementary_gids = match user_name { - Some(name) if name.parse::().is_err() && resolved_identity.uid().is_none() => { + Some(name) if name.parse::().is_err() => { let primary_gid = if let Some(gid) = gid { gid } else { @@ -1870,7 +2001,14 @@ fn resolve_filesystem_identity( .ok_or_else(|| miette::miette!("Failed to resolve user from UID {uid}"))? .gid }; - named_user_supplementary_groups(name, primary_gid)? + if resolved_identity.uid().is_some() { + crate::identity::resolve_oci_supplementary_gids(name, primary_gid.as_raw())? + .into_iter() + .map(Gid::from_raw) + .collect() + } else { + named_user_supplementary_groups(name, primary_gid)? + } } _ => Vec::new(), }; @@ -1891,19 +2029,6 @@ pub fn drop_privileges(policy: &SandboxPolicy) -> Result<()> { drop_privileges_with_identity(policy, ResolvedProcessIdentity::default()) } -#[cfg(unix)] -fn should_clear_supplementary_groups( - current_uid: Uid, - target_uid: Uid, - user_name: Option<&str>, - resolved_identity: ResolvedProcessIdentity, -) -> bool { - resolved_identity.uses_oci_user_fallback() - && target_uid != current_uid - && !(user_name.is_some_and(|name| name.parse::().is_err()) - && resolved_identity.uid().is_none()) -} - #[cfg(unix)] #[allow(clippy::similar_names)] pub fn drop_privileges_with_identity( @@ -1999,23 +2124,21 @@ pub fn drop_privileges_with_identity( }; if target_uid != nix::unistd::geteuid() { - if should_clear_supplementary_groups( - nix::unistd::geteuid(), - target_uid, - user_name, - resolved_identity, - ) { - // OCI-derived users do not have a trustworthy NSS - // supplementary-group source. Clear the root supervisor's - // inherited groups before changing UID/GID. Platform-resolved and - // explicit numeric identities retain their pre-OCI behavior. + if resolved_identity.uses_oci_user_fallback() { + // OCI named users use the bounded /etc/group parser shared with + // workspace validation. Numeric OCI users resolve to an empty + // list. Never retain the root supervisor's inherited groups. #[cfg(not(any( target_os = "macos", target_os = "ios", target_os = "haiku", target_os = "redox" )))] - nix::unistd::setgroups(&[]).into_diagnostic()?; + { + let (_, _, supplementary_gids) = + resolve_filesystem_identity(policy, resolved_identity)?; + nix::unistd::setgroups(&supplementary_gids).into_diagnostic()?; + } } else if let Some(ref user_name) = initgroups_name { let user_cstr = CString::new(user_name.as_str()) .map_err(|_| miette::miette!("Invalid user name"))?; @@ -2242,43 +2365,6 @@ mod tests { assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); } - #[test] - #[cfg(unix)] - fn only_oci_numeric_user_paths_clear_supplementary_groups_before_uid_drop() { - let current_uid = Uid::from_raw(0); - let target_uid = Uid::from_raw(1234); - - assert!(!should_clear_supplementary_groups( - current_uid, - target_uid, - Some("1234"), - ResolvedProcessIdentity::default(), - )); - assert!(should_clear_supplementary_groups( - current_uid, - target_uid, - Some("1234"), - ResolvedProcessIdentity::new(None, Some(1235)), - )); - } - - #[test] - #[cfg(unix)] - fn supplementary_group_clearing_preserves_explicit_named_user_behavior() { - assert!(!should_clear_supplementary_groups( - Uid::from_raw(0), - Uid::from_raw(1234), - Some("app"), - ResolvedProcessIdentity::default(), - )); - assert!(!should_clear_supplementary_groups( - Uid::from_raw(1234), - Uid::from_raw(1234), - Some("1234"), - ResolvedProcessIdentity::new(None, Some(1235)), - )); - } - #[test] fn full_enforcement_uses_privileged_setup_and_child_sandbox() { assert!(ProcessEnforcementMode::Full.uses_privileged_process_setup()); @@ -3045,6 +3131,25 @@ mod tests { assert!(error.to_string().contains("does not exist")); } + #[cfg(target_os = "linux")] + #[test] + fn effective_identity_validation_uses_kernel_access_checks() { + if nix::unistd::geteuid().is_root() { + return; + } + + let dir = tempfile::tempdir_in("/tmp").unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + validate_oci_workspace_as_effective_identity(&root) + .expect("current identity can write and enter its directory"); + + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o500)).unwrap(); + let error = validate_oci_workspace_as_effective_identity(&root).unwrap_err(); + assert!(error.to_string().contains("not writable and traversable")); + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_restrictive_parent() { diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 55f91c1ac2..55e6fecdb8 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -444,7 +444,9 @@ UID/GID and supplementary groups must already be able to traverse every parent and write and enter the workdir. OpenShell does not change that directory's ownership or mode. It rejects kernel-managed filesystems based on filesystem metadata and rejects overlap with actual OpenShell control paths. Docker checks -the original image filesystem in the final supervisor. Podman checks the same +the original image filesystem in the final supervisor and rejects image +`VOLUME` declarations that would mask the workdir or one of its parents before +validation. Podman checks the same pinned image ID in a temporary networkless probe without the workspace volume, token, or TLS secrets. The probe retains only `SETUID` and `SETGID`, drops to the completed process identity, and uses the kernel to validate access. The diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index e43cdb0514..80fdac4c8c 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -55,6 +55,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ && useradd -m -u 3234 -g appstaff app WORKDIR /workspace/project +# Image metadata must not be able to forge the driver's successful-validation +# attestation. +ENV OPENSHELL_OCI_WORKSPACE_PREVALIDATED=1 +ENV OPENSHELL_OCI_WORKSPACE_IDENTITY=3234:3235: USER app CMD ["sleep", "infinity"] "#;