diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 0de85fb0a4..acdaa7b3b9 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -176,6 +176,14 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. +- Gateway exits before becoming healthy with a callback-listener discovery + error: inspect `podman info --debug`, the configured Podman network, and the + host's IPv4 default route. Rootless pasta uses the private source address + selected by that route; rootful Podman uses the bridge gateway address. +- Rootless slirp4netns, another named helper, or missing helper metadata + requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` + cannot bypass slirp4netns host-loopback isolation. Do not work around + discovery failures by broadening the primary gateway listener to `0.0.0.0`. ### Step 6: Check Kubernetes Helm Gateways @@ -392,6 +400,8 @@ openshell logs |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | +| Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | +| Admin, health, reflection, or HTTP request is denied on a Docker/Podman callback address | Negotiated callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs | | Docker GPU e2e fails before GPU sandbox comparison | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` | | Kubernetes gateway pod pending | PVC unbound, taint, selector, or insufficient resources | `kubectl -n openshell describe pod ` | diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index d8f33f7016..e96a56075b 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -121,24 +121,15 @@ jobs: run: ${{ matrix.cmd }} e2e-podman-rootless: - name: E2E (rust-podman-rootless, ${{ matrix.runner }}) + name: E2E (rust-podman-rootless, ubuntu-26.04) # Run directly on the Ubuntu host so the test observes the host's AppArmor # and unprivileged-user-namespace policy. A privileged job container masks # the restrictions that production rootless Podman installations enforce. - runs-on: ${{ matrix.runner }} + # Ubuntu 26.04 provides the supported Podman 5.x and pasta combination. + # Re-add older/slirp4netns environments when direct callbacks through a + # rootless-network namespace relay are supported. + runs-on: ubuntu-26.04 timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - # Ubuntu 24.04 matches the environment reported in #2069 and ships - # Podman 4.x. The probe records whether AppArmor blocks the drop. - - runner: ubuntu-24.04 - podman_major: "4" - # Ubuntu 26.04 provides the supported Podman 5.x coverage for - # comparison with the Ubuntu 24.04 environment. - - runner: ubuntu-26.04 - podman_major: "5" env: IMAGE_TAG: ${{ inputs.image-tag }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -188,7 +179,6 @@ jobs: passt \ pkg-config \ podman \ - slirp4netns \ uidmap - name: Configure rootless Podman @@ -209,10 +199,11 @@ jobs: set -euo pipefail podman_version="$(podman version --format '{{.Client.Version}}')" case "$podman_version" in - "${{ matrix.podman_major }}".*) ;; - *) echo "ERROR: expected Podman ${{ matrix.podman_major }}.x, found $podman_version" >&2; exit 1 ;; + 5.*) ;; + *) echo "ERROR: expected Podman 5.x, found $podman_version" >&2; exit 1 ;; esac test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true" + test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta" test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1" echo "=== host ===" uname -a diff --git a/Cargo.lock b/Cargo.lock index 31e2104987..2358c72422 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3976,6 +3976,7 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", + "url", ] [[package]] @@ -4217,6 +4218,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "socket2 0.6.3", "sqlx", "tempfile", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index 4ec6a0d44f..78af61d438 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat # Unix/Process nix = { version = "0.29", features = ["signal", "process", "user", "fs", "term"] } rustix = { version = "1.1", features = ["process"] } +socket2 = "0.6" # Serialization serde = { version = "1", features = ["derive"] } diff --git a/architecture/gateway.md b/architecture/gateway.md index c8b323ea13..7b2a8cc8f3 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -37,6 +37,22 @@ health, metrics, or tunnel routes. The plaintext service router also rejects browser requests whose Fetch Metadata, Origin, or Referer headers indicate a cross-origin or sibling-subdomain request. +Docker and Podman may negotiate additional listeners that make the gateway +reachable from their local sandbox network topology. Those listeners accept +only gRPC methods classified as sandbox-callable by the gateway's generated +authorization metadata. They reject user and administrator APIs, health, +reflection, non-callback inference APIs, and HTTP routes before normal request +authentication. The operator-configured primary listener retains the full +multiplexed API surface. + +The `rpc_auth` classification is also the source of truth for negotiated +listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on +these listeners. Review such changes as both authorization and network-surface +changes. Listener requirements are currently authorized only for the built-in +Docker and Podman drivers. Operator-granted listener capabilities for external +drivers are tracked in +[#2539](https://github.com/NVIDIA/OpenShell/issues/2539). + Operators can configure a gateway-wide gRPC request rate limit. The limit is applied only to gRPC API traffic after protocol multiplexing; health, metrics, and local sandbox-service HTTP routes are not rate limited by this control. @@ -623,7 +639,12 @@ system entry instead of pretending to delete package-manager owned state. - Podman-backed macOS gateways use gvproxy's host-loopback IP for sandbox host aliases by default so stale Podman machine images do not need Podman's `host-gateway` resolver. Linux Podman keeps the resolver unless - `host_gateway_ip` is configured. + `host_gateway_ip` is configured. Rootful Podman can request its exact bridge + gateway listener. Rootless Podman explicitly reporting pasta requests the + private IPv4 source selected by the host default route rather than an + arbitrary private interface. Slirp4netns, other helpers, and missing helper + metadata fail closed for local callbacks until a rootless-network namespace + relay is available. - Gateway restarts recover persisted objects from storage, but live relay streams must be re-established by supervisors. - User-facing behavior changes must update published docs in `docs/`; this file diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index a196ba6ad9..502f4ae8fe 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -39,12 +39,14 @@ use openshell_core::progress::{ use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, watch_sandboxes_event, + DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, + StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, + WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, + gateway_listener_requirement::Selector, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -399,14 +401,6 @@ impl DockerComputeDriver { Ok(driver) } - #[must_use] - pub fn gateway_bind_addresses(&self) -> Vec { - match self.config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], - DockerGatewayRoute::HostGateway => Vec::new(), - } - } - fn capabilities(&self) -> GetCapabilitiesResponse { openshell_core::driver_utils::build_capabilities_response( "docker", @@ -1386,6 +1380,24 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(self.capabilities())) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + let requirements = match self.config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => { + vec![GatewayListenerRequirement { + reason: "docker managed bridge gateway".to_string(), + selector: Some(Selector::ExactBindAddress(bind_address.to_string())), + }] + } + DockerGatewayRoute::HostGateway => Vec::new(), + }; + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements, + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index a86a9936f5..fdf850dc6a 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -13,8 +13,9 @@ use openshell_core::progress::{ PROGRESS_STEP_STARTING_SANDBOX, }; use openshell_core::proto::compute::v1::{ - DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GpuResourceRequirements, - ResourceRequirements, + DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, + GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, + gateway_listener_requirement::Selector, }; use std::fs; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -158,6 +159,43 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr } } +#[tokio::test] +async fn gateway_listener_requirements_report_managed_bridge_address() { + let config = runtime_config(); + let expected_address = match config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address, + DockerGatewayRoute::HostGateway => panic!("test config must use a managed bridge"), + }; + let driver = test_driver_with_config(config); + + let response = driver + .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) + .await + .unwrap() + .into_inner(); + + assert_eq!(response.requirements.len(), 1); + assert_eq!( + response.requirements[0].selector, + Some(Selector::ExactBindAddress(expected_address.to_string())) + ); +} + +#[tokio::test] +async fn gateway_listener_requirements_are_empty_for_host_gateway_route() { + let mut config = runtime_config(); + config.gateway_route = DockerGatewayRoute::HostGateway; + let driver = test_driver_with_config(config); + + let response = driver + .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) + .await + .unwrap() + .into_inner(); + + assert!(response.requirements.is_empty()); +} + #[test] fn container_visible_endpoint_rewrites_loopback_hosts() { assert_eq!( diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index fccfa9464b..6eeb51cd73 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -6,7 +6,8 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, @@ -40,6 +41,15 @@ impl ComputeDriver for ComputeDriverService { .map_err(Status::internal) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: Vec::new(), + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index ed798c0ab2..5f108fd9cc 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } +url = { workspace = true } [dev-dependencies] prost-types = { workspace = true } diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 93c5ba0964..28357f1c43 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -255,6 +255,31 @@ if config.grpc_endpoint.is_empty() { The bridge gateway IP is not a stable substitute in rootless mode because it can live inside the user namespace rather than on the host. +Before the gateway binds its serving sockets, the driver reports the callback +listener required by the selected topology: + +- Rootful Linux Podman reports the configured bridge's gateway address exactly. +- Rootless Linux Podman explicitly reporting pasta requests the private IPv4 + source address selected by the host's default route. This avoids guessing + among private interfaces on a multihomed host. +- Rootless Linux Podman reporting slirp4netns, another named helper, or no + helper cannot use a direct local callback listener. The driver fails startup + unless `grpc_endpoint` names an explicitly remote endpoint. Supporting + slirp4netns requires a relay inside Podman's rootless network namespace. +- Podman Machine requests IPv4 loopback because gvproxy terminates the host + forwarding path there. +- An explicitly remote callback endpoint requests no additional local listener. + +On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman +and rootless pasta because the driver maps both local callback aliases to that +literal. Other rootless helpers still fail closed. Podman Machine requests +gateway loopback because its configured address is guest-visible and gvproxy +terminates that route on host loopback. The gateway validates and binds every +accepted callback listener; the primary listener can remain on loopback. +Negotiated callback listeners expose only the gateway's sandbox-callable gRPC +methods. Operator, health, reflection, and HTTP requests must use the primary +listener. + ### Layer 3 Inner Sandbox Network Namespace Inside the container, the supervisor creates another network namespace for the diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 952938d47a..5291f09eac 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -262,6 +262,8 @@ pub struct HostInfo { #[serde(default)] pub network_backend: String, #[serde(default)] + pub rootless_network_cmd: String, + #[serde(default)] pub security: SecurityInfo, } @@ -902,6 +904,24 @@ mod tests { assert!(validate_name(&exact_name).is_ok()); } + #[test] + fn system_info_parses_rootless_network_helper() { + let info: SystemInfo = serde_json::from_str( + r#"{ + "host": { + "cgroupVersion": "v2", + "networkBackend": "netavark", + "rootlessNetworkCmd": "pasta", + "security": {"rootless": true} + } + }"#, + ) + .unwrap(); + + assert!(info.host.security.rootless); + assert_eq!(info.host.rootless_network_cmd, "pasta"); + } + #[tokio::test] async fn inspect_volume_parses_driver_options() { let (socket_path, request_log, handle) = spawn_podman_stub( diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index aa3df9cbd7..39c312c24b 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -16,13 +16,21 @@ use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; +#[cfg(target_os = "linux")] +use openshell_core::proto::compute::v1::GatewayDefaultRouteInterfaceRequirement; +#[cfg(target_os = "macos")] +use openshell_core::proto::compute::v1::GatewayLoopbackInterfaceRequirement; use openshell_core::proto::compute::v1::{ - DriverSandbox, GetCapabilitiesResponse, GpuResourceRequirements, + DriverSandbox, GatewayListenerRequirement, GetCapabilitiesResponse, GpuResourceRequirements, + gateway_listener_requirement::Selector, }; +#[cfg(target_os = "linux")] +use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use tracing::{debug, info, warn}; +use url::Url; impl From for ComputeDriverError { fn from(value: PodmanApiError) -> Self { @@ -39,9 +47,13 @@ impl From for ComputeDriverError { pub struct PodmanComputeDriver { client: PodmanClient, config: PodmanComputeConfig, - /// The host's IP on the bridge network. Sandbox containers use this to - /// reach the gateway server when no explicit gRPC endpoint is configured. + /// The host's IP on the bridge network, when that bridge exists in the + /// gateway's network namespace (notably rootful Podman). network_gateway_ip: Option, + /// Whether Podman's service is running without root privileges. + rootless: bool, + /// Rootless network helper reported by Podman, such as `pasta`. + rootless_network_cmd: String, gpu_selector: Arc, gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, } @@ -52,6 +64,8 @@ impl std::fmt::Debug for PodmanComputeDriver { .field("socket_path", &self.config.socket_path) .field("default_image", &self.config.default_image) .field("network_name", &self.config.network_name) + .field("rootless", &self.rootless) + .field("rootless_network_cmd", &self.rootless_network_cmd) .field("gpu_inventory", &self.gpu_selector.device_ids()) .finish() } @@ -289,7 +303,7 @@ impl PodmanComputeDriver { } // Verify cgroups v2, detect rootless mode, and log system info. - match client.system_info().await { + let (rootless, rootless_network_cmd) = match client.system_info().await { Ok(info) => { if info.host.cgroup_version != "v2" { return Err(PodmanApiError::Connection(format!( @@ -303,15 +317,17 @@ impl PodmanComputeDriver { cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, rootless = info.host.security.rootless, + rootless_network_cmd = %info.host.rootless_network_cmd, "Connected to Podman" ); + (info.host.security.rootless, info.host.rootless_network_cmd) } Err(e) => { return Err(PodmanApiError::Connection(format!( "failed to query Podman system info: {e}" ))); } - } + }; // Rootless pre-flight: warn if subuid/subgid ranges look missing. // Not a hard error because some systems configure these via LDAP or @@ -322,10 +338,7 @@ impl PodmanComputeDriver { // Ensure the bridge network exists. client.ensure_network(&config.network_name).await?; - let network_gateway_ip = client - .network_gateway_ip(&config.network_name) - .await - .unwrap_or(None); + let network_gateway_ip = client.network_gateway_ip(&config.network_name).await?; info!( network = %config.network_name, gateway_ip = ?network_gateway_ip, @@ -368,6 +381,8 @@ impl PodmanComputeDriver { client, config, network_gateway_ip, + rootless, + rootless_network_cmd, gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -378,8 +393,8 @@ impl PodmanComputeDriver { /// The host's IP on the bridge network, if available. /// - /// Used by the server to auto-detect the gRPC callback endpoint when - /// no explicit `--grpc-endpoint` is configured. + /// Used to request the exact rootful gateway callback listener when no + /// explicit host-gateway override is configured. #[must_use] pub fn network_gateway_ip(&self) -> Option<&str> { self.network_gateway_ip.as_deref() @@ -394,6 +409,96 @@ impl PodmanComputeDriver { )) } + /// Report the gateway exposure needed by Podman's standard local callback aliases. + /// + /// Rootful Podman binds the exact bridge address behind the sandbox alias. + /// Rootless pasta follows the host's default-route interface, while Podman + /// Machine forwards the alias to gateway loopback. Other rootless helpers + /// cannot use a direct host listener. + pub fn gateway_listener_requirements( + &self, + ) -> Result, ComputeDriverError> { + let endpoint = Url::parse(&self.config.grpc_endpoint).map_err(|err| { + ComputeDriverError::Precondition(format!( + "invalid Podman gateway callback endpoint '{}': {err}", + self.config.grpc_endpoint + )) + })?; + let uses_local_callback_alias = endpoint.host_str().is_some_and(|host| { + matches!(host, "host.containers.internal" | "host.openshell.internal") + }); + if !uses_local_callback_alias { + return Ok(Vec::new()); + } + let callback_port = endpoint.port_or_known_default().ok_or_else(|| { + ComputeDriverError::Precondition(format!( + "Podman gateway callback endpoint '{}' has no port", + self.config.grpc_endpoint + )) + })?; + if callback_port != self.config.gateway_port { + return Err(ComputeDriverError::Precondition(format!( + "Podman local callback endpoint '{}' uses port {callback_port}, but the gateway primary listener uses port {}; configure grpc_endpoint with the gateway primary listener port", + self.config.grpc_endpoint, self.config.gateway_port + ))); + } + + #[cfg(target_os = "linux")] + { + if self.rootless { + validate_rootless_local_callback_helper(&self.rootless_network_cmd)?; + + if self.config.host_gateway_ip.trim().is_empty() { + return Ok(vec![GatewayListenerRequirement { + reason: + "Podman rootless pasta callback uses the host default-route interface" + .to_string(), + selector: Some(Selector::DefaultRouteInterface( + GatewayDefaultRouteInterfaceRequirement {}, + )), + }]); + } + } + + let gateway_ip = if self.config.host_gateway_ip.trim().is_empty() { + self.network_gateway_ip.as_deref().ok_or_else(|| { + ComputeDriverError::Precondition(format!( + "Podman network '{}' did not report a host bridge gateway address for local callback alias '{}'", + self.config.network_name, + endpoint.host_str().unwrap_or_default() + )) + })? + } else { + self.config.host_gateway_ip.trim() + }; + let gateway_ip = gateway_ip.parse::().map_err(|err| { + ComputeDriverError::Precondition(format!( + "Podman callback gateway address '{gateway_ip}' is invalid: {err}" + )) + })?; + Ok(vec![GatewayListenerRequirement { + reason: format!("Podman network '{}' host gateway", self.config.network_name), + selector: Some(Selector::ExactBindAddress( + SocketAddr::new(gateway_ip, callback_port).to_string(), + )), + }]) + } + #[cfg(target_os = "macos")] + { + Ok(vec![GatewayListenerRequirement { + reason: "Podman machine callback forwarding terminates on gateway loopback" + .to_string(), + selector: Some(Selector::LoopbackInterface( + GatewayLoopbackInterfaceRequirement {}, + )), + }]) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + Ok(Vec::new()) + } + } + #[must_use] pub fn default_image(&self) -> &str { &self.config.default_image @@ -892,6 +997,8 @@ impl PodmanComputeDriver { client, config, network_gateway_ip: None, + rootless: false, + rootless_network_cmd: String::new(), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -950,6 +1057,25 @@ fn check_subuid_range() { } } +#[cfg(any(target_os = "linux", test))] +fn validate_rootless_local_callback_helper( + rootless_network_cmd: &str, +) -> Result<(), ComputeDriverError> { + let rootless_network_cmd = rootless_network_cmd.trim(); + if rootless_network_cmd == "pasta" { + return Ok(()); + } + + let reported = if rootless_network_cmd.is_empty() { + "" + } else { + rootless_network_cmd + }; + Err(ComputeDriverError::Precondition(format!( + "Podman rootless network helper '{reported}' does not support direct local gateway callbacks; configure pasta or use an explicitly remote grpc_endpoint" + ))) +} + #[cfg(test)] mod tests { use super::*; @@ -1168,6 +1294,240 @@ mod tests { assert_eq!(cfg.grpc_endpoint, "https://gateway.internal:9000"); } + #[test] + fn rootless_slirp_allows_remote_callback_endpoint() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "https://gateway.internal:9000".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "slirp4netns".to_string(); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert!(requirements.is_empty()); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootful_local_callback_alias_requests_discovered_network_gateway() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.network_gateway_ip = Some("10.89.1.1".to_string()); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert_eq!(requirements.len(), 1); + assert_eq!( + requirements[0].selector, + Some(Selector::ExactBindAddress("10.89.1.1:17670".to_string())) + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn configured_host_gateway_overrides_discovered_network_gateway() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.containers.internal:17670".to_string(), + host_gateway_ip: "10.90.1.1".to_string(), + ..PodmanComputeConfig::default() + }); + driver.network_gateway_ip = Some("10.89.1.1".to_string()); + driver.rootless = true; + driver.rootless_network_cmd = "pasta".to_string(); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert_eq!( + requirements[0].selector, + Some(Selector::ExactBindAddress("10.90.1.1:17670".to_string())) + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootless_pasta_requests_default_route_interface() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "pasta".to_string(); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert!(matches!( + requirements[0].selector, + Some(Selector::DefaultRouteInterface(_)) + )); + } + + #[test] + fn rootless_non_pasta_helpers_are_rejected() { + for (rootless_network_cmd, reported) in [ + ("slirp4netns", "slirp4netns"), + ("", ""), + ("unknown-helper", "unknown-helper"), + ] { + let err = validate_rootless_local_callback_helper(rootless_network_cmd).unwrap_err(); + + assert!(matches!(err, ComputeDriverError::Precondition(_))); + assert!(err.to_string().contains(reported)); + assert!(err.to_string().contains("configure pasta")); + assert!(err.to_string().contains("remote grpc_endpoint")); + } + } + + #[test] + fn rootless_pasta_is_accepted_for_local_callbacks() { + validate_rootless_local_callback_helper("pasta").unwrap(); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootless_slirp_rejects_explicit_host_gateway_override() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + host_gateway_ip: "10.90.1.1".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "slirp4netns".to_string(); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!(matches!(err, ComputeDriverError::Precondition(_))); + assert!(err.to_string().contains("slirp4netns")); + } + + #[tokio::test] + async fn constructor_preserves_network_gateway_discovery_error() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "network-gateway-error", + vec![ + StubResponse::new(StatusCode::OK, ""), + StubResponse::new( + StatusCode::OK, + r#"{ + "host": { + "cgroupVersion": "v2", + "networkBackend": "netavark", + "security": {"rootless": false} + } + }"#, + ), + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message":"network gateway inspection failed"}"#, + ), + ], + ); + let config = PodmanComputeConfig { + socket_path: Some(socket_path.clone()), + ..PodmanComputeConfig::default() + }; + let network_name = config.network_name.clone(); + + let Err(err) = PodmanComputeDriver::new(config).await else { + panic!("network gateway discovery failure should prevent startup"); + }; + + assert!( + err.to_string() + .contains("network gateway inspection failed"), + "unexpected startup error: {err}" + ); + handle.await.expect("stub task should finish"); + assert_eq!( + request_log + .lock() + .expect("request log lock should not be poisoned") + .as_slice(), + [ + "GET /_ping".to_string(), + format!("GET {}", api_path("/libpod/info")), + format!("POST {}", api_path("/libpod/networks/create")), + format!( + "GET {}", + api_path(&format!("/libpod/networks/{network_name}/json")) + ), + ] + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootful_local_callback_alias_requires_concrete_gateway_address() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!( + err.to_string() + .contains("did not report a host bridge gateway address") + ); + } + + #[test] + #[cfg(target_os = "macos")] + fn podman_machine_callback_alias_requests_loopback_listener() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert_eq!(requirements.len(), 1); + assert!(matches!( + requirements[0].selector, + Some(Selector::LoopbackInterface(_)) + )); + } + + #[test] + fn explicit_remote_callback_does_not_request_gateway_listener() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "https://gateway.example.test:9443".to_string(), + gateway_port: 17670, + ..PodmanComputeConfig::default() + }); + + assert!(driver.gateway_listener_requirements().unwrap().is_empty()); + } + + #[test] + fn local_callback_alias_requires_primary_listener_port() { + for grpc_endpoint in [ + "http://host.openshell.internal:17671", + "http://host.containers.internal", + ] { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: grpc_endpoint.to_string(), + gateway_port: 17670, + ..PodmanComputeConfig::default() + }); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!( + matches!(err, ComputeDriverError::Precondition(_)), + "mismatched local callback port should fail precondition: {err}" + ); + assert!( + err.to_string() + .contains("gateway primary listener uses port 17670"), + "unexpected error for {grpc_endpoint}: {err}" + ); + } + } + #[test] fn local_podman_cdi_gpu_inventory_maps_nvidia_device_nodes() { let root = std::env::temp_dir().join(format!( diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 8e68a91e72..2d0792d447 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -6,7 +6,8 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, @@ -40,6 +41,18 @@ impl ComputeDriver for ComputeDriverService { .map_err(Status::from) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: self + .driver + .gateway_listener_requirements() + .map_err(Status::from)?, + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 7af0ddc389..841457f38c 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -41,6 +41,7 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, @@ -2989,6 +2990,15 @@ impl ComputeDriver for VmDriver { Ok(Response::new(self.capabilities())) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: Vec::new(), + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 4c4f289ed6..569a924f59 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -36,6 +36,7 @@ k8s-openapi = { workspace = true } # Async runtime tokio = { workspace = true } +socket2 = { workspace = true } # gRPC tonic = { workspace = true, features = ["channel", "tls-native-roots"] } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index b87f1b5fa6..5d3dda0a72 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -27,11 +27,14 @@ use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetSandboxRequest, + DriverSandboxTemplate, GatewayListenerRequirement as ProtoGatewayListenerRequirement, + GetCapabilitiesRequest, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, ResourceRequirements as DriverSandboxResourceRequirements, ValidateSandboxCreateRequest, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, - compute_driver_server::ComputeDriver, watch_sandboxes_event, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, + watch_sandboxes_event, }; use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, @@ -65,6 +68,41 @@ type SharedComputeDriver = const DELETE_PHASE_CAS_RETRY_LIMIT: usize = 3; +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum GatewayListenerRequirement { + Exact { + address: SocketAddr, + driver_name: String, + reason: String, + }, + DefaultRouteInterface { + driver_name: String, + reason: String, + }, + LoopbackInterface { + driver_name: String, + reason: String, + }, +} + +impl GatewayListenerRequirement { + pub fn driver_name(&self) -> &str { + match self { + Self::Exact { driver_name, .. } + | Self::DefaultRouteInterface { driver_name, .. } + | Self::LoopbackInterface { driver_name, .. } => driver_name, + } + } + + pub fn reason(&self) -> &str { + match self { + Self::Exact { reason, .. } + | Self::DefaultRouteInterface { reason, .. } + | Self::LoopbackInterface { reason, .. } => reason, + } + } +} + /// Serializes request-side deletes for the same stable sandbox ID. /// /// Watch events deliberately do not use these gates, so a slow driver delete @@ -288,6 +326,14 @@ impl ComputeDriver for RemoteComputeDriver { client.get_capabilities(request).await } + async fn get_gateway_listener_requirements( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.get_gateway_listener_requirements(request).await + } + async fn validate_sandbox_create( &self, request: Request, @@ -370,7 +416,7 @@ pub struct ComputeRuntime { supervisor_sessions: Arc, sync_lock: Arc>, delete_gates: Arc, - gateway_bind_addresses: Vec, + gateway_listener_requirements: Vec, replica_id: String, } @@ -393,7 +439,6 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - gateway_bind_addresses: Vec, ) -> Result { let capabilities = driver .get_capabilities(Request::new(GetCapabilitiesRequest {})) @@ -408,11 +453,64 @@ impl ComputeRuntime { "Compute driver connected" ); let driver_info = ComputeDriverInfoSnapshot { - name: driver_name, + name: driver_name.clone(), driver_name: capabilities.driver_name, driver_version: capabilities.driver_version, }; let default_image = capabilities.default_image; + let gateway_listener_requirements = match driver + .get_gateway_listener_requirements(Request::new( + GetGatewayListenerRequirementsRequest {}, + )) + .await + { + Ok(response) => response + .into_inner() + .requirements + .into_iter() + .map(|requirement: ProtoGatewayListenerRequirement| { + let Some(selector) = requirement.selector else { + return Err(ComputeError::Message(format!( + "compute driver '{driver_name}' returned a gateway listener requirement without a selector" + ))); + }; + match selector { + Selector::ExactBindAddress(bind_address) => { + let address = bind_address.parse::().map_err(|err| { + ComputeError::Message(format!( + "compute driver '{driver_name}' returned invalid gateway listener address '{bind_address}': {err}" + )) + })?; + Ok(GatewayListenerRequirement::Exact { + address, + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + } + Selector::DefaultRouteInterface(_) => { + Ok(GatewayListenerRequirement::DefaultRouteInterface { + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + } + Selector::LoopbackInterface(_) => { + Ok(GatewayListenerRequirement::LoopbackInterface { + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + } + } + }) + .collect::, ComputeError>>()?, + Err(status) if status.code() == Code::Unimplemented => { + debug!( + driver = %driver_name, + "Compute driver does not implement gateway listener requirements" + ); + Vec::new() + } + Err(status) => return Err(compute_error_from_status(status)), + }; Ok(Self { driver, driver_info, @@ -427,7 +525,7 @@ impl ComputeRuntime { supervisor_sessions, sync_lock: Arc::new(Mutex::new(())), delete_gates: Arc::new(DeleteGateRegistry::default()), - gateway_bind_addresses, + gateway_listener_requirements, replica_id: lease::replica_id(), }) } @@ -471,7 +569,6 @@ impl ComputeRuntime { .await .map_err(|err| ComputeError::Message(err.to_string()))?, ); - let gateway_bind_addresses = driver.gateway_bind_addresses(); let shutdown_cleanup: Arc = driver.clone(); let startup_resume: Arc = driver.clone(); let driver: SharedComputeDriver = driver; @@ -486,7 +583,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - gateway_bind_addresses, ) .await } @@ -514,7 +610,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), ) .await } @@ -539,7 +634,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), ) .await } @@ -567,7 +661,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), ) .await } @@ -588,8 +681,8 @@ impl ComputeRuntime { } #[must_use] - pub fn gateway_bind_addresses(&self) -> &[SocketAddr] { - &self.gateway_bind_addresses + pub(crate) fn gateway_listener_requirements(&self) -> &[GatewayListenerRequirement] { + &self.gateway_listener_requirements } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { @@ -2752,6 +2845,15 @@ impl ComputeDriver for NoopTestDriver { )) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new( + GetGatewayListenerRequirementsResponse::default(), + )) + } + async fn validate_sandbox_create( &self, _request: Request, @@ -2847,7 +2949,7 @@ pub async fn new_test_runtime_for_driver(store: Arc, driver_name: &str) - supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), delete_gates: Arc::new(DeleteGateRegistry::default()), - gateway_bind_addresses: Vec::new(), + gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), } } @@ -3001,6 +3103,15 @@ mod tests { })) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new( + GetGatewayListenerRequirementsResponse::default(), + )) + } + async fn validate_sandbox_create( &self, _request: Request, @@ -3183,6 +3294,15 @@ mod tests { })) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new( + GetGatewayListenerRequirementsResponse::default(), + )) + } + async fn validate_sandbox_create( &self, _request: Request, @@ -3316,7 +3436,7 @@ mod tests { supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), delete_gates: Arc::new(DeleteGateRegistry::default()), - gateway_bind_addresses: Vec::new(), + gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), } } @@ -6085,7 +6205,11 @@ mod tests { let socket_path = dir.path().join("compute-driver.sock"); let driver = FakeComputeDriver::new() .with_driver_name("fake-remote-driver") - .with_default_image("openshell/sandbox:remote"); + .with_default_image("openshell/sandbox:remote") + .with_gateway_listener_requirement( + "172.19.0.1:17670", + "external driver managed bridge", + ); let _server = driver.serve_uds(&socket_path).unwrap(); let endpoint = connect_remote_compute_driver("external-test", &socket_path) @@ -6102,6 +6226,14 @@ mod tests { ) .await .unwrap(); + assert_eq!( + runtime.gateway_listener_requirements(), + &[GatewayListenerRequirement::Exact { + address: "172.19.0.1:17670".parse().unwrap(), + driver_name: "external-test".to_string(), + reason: "external driver managed bridge".to_string(), + }] + ); let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); sandbox.spec = Some(SandboxSpec { @@ -6138,10 +6270,14 @@ mod tests { ); let calls = driver.calls(); - assert_eq!(calls.len(), 4, "unexpected calls: {calls:?}"); + assert_eq!(calls.len(), 5, "unexpected calls: {calls:?}"); assert!(matches!(calls[0], FakeComputeDriverCall::GetCapabilities)); + assert!(matches!( + calls[1], + FakeComputeDriverCall::GetGatewayListenerRequirements + )); - let validated = match &calls[1] { + let validated = match &calls[2] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), } => sandbox, @@ -6158,7 +6294,7 @@ mod tests { assert!(driver_config.fields.contains_key("pool")); assert!(!driver_config.fields.contains_key("network_mode")); - let created = match &calls[2] { + let created = match &calls[3] { FakeComputeDriverCall::CreateSandbox { sandbox: Some(sandbox), } => sandbox, @@ -6167,7 +6303,7 @@ mod tests { assert_eq!(created.id, "sb-uds"); assert_eq!(created.name, "uds-sandbox"); - match &calls[3] { + match &calls[4] { FakeComputeDriverCall::DeleteSandbox { sandbox_id, sandbox_name, @@ -6179,6 +6315,43 @@ mod tests { } } + #[tokio::test] + #[cfg(unix)] + async fn remote_compute_driver_accepts_unimplemented_listener_requirements_api() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("compute-driver.sock"); + let driver = FakeComputeDriver::new() + .with_driver_name("legacy-remote-driver") + .without_gateway_listener_requirements_api(); + let _server = driver.serve_uds(&socket_path).unwrap(); + + let endpoint = connect_remote_compute_driver("external-test", &socket_path) + .await + .unwrap(); + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + store, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + + assert!(runtime.gateway_listener_requirements().is_empty()); + assert_eq!( + driver.calls(), + vec![ + FakeComputeDriverCall::GetCapabilities, + FakeComputeDriverCall::GetGatewayListenerRequirements, + ] + ); + } + #[tokio::test] async fn create_sandbox_returns_resource_version_one() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index c4e0cbc959..2980fbe561 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -720,7 +720,8 @@ version = 2 /// `load()` path that the gateway uses at runtime, catching: /// - template corruption or unknown fields (`deny_unknown_fields`) /// - schema drift (version bump or field renames) - /// - accidental changes to the bind address or compute driver list + /// - accidental addition of a wildcard bind-address override + /// - accidental changes to the compute driver list #[test] fn rpm_default_config_parses_and_has_podman_defaults() { let path = @@ -729,20 +730,12 @@ version = 2 load(&path).expect("deploy/rpm/gateway.toml.default must parse against current schema"); let gw = &config.openshell.gateway; - let addr = gw - .bind_address - .expect("bind_address must be explicitly set in the RPM default config"); - assert!( - addr.ip().is_unspecified(), - "RPM default bind_address must be 0.0.0.0 so Podman sandbox containers \ - can reach the gateway over the host network bridge, got {addr}" - ); - assert_eq!( - addr.port(), - openshell_core::config::DEFAULT_SERVER_PORT, - "RPM default port must match DEFAULT_SERVER_PORT ({})", - openshell_core::config::DEFAULT_SERVER_PORT - ); + if let Some(addr) = gw.bind_address { + assert!( + !addr.ip().is_unspecified(), + "RPM default config must not expose the primary listener on every interface" + ); + } let drivers = gw .compute_drivers diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 0957c4ebee..08a875389f 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use openshell_core::{Error, Result}; -use std::net::SocketAddr; +use crate::compute::GatewayListenerRequirement; +use openshell_core::{ComputeDriverKind, Error, Result}; +use socket2::{Domain, Protocol, Socket, Type}; +use std::net::{IpAddr, SocketAddr}; use tokio::net::TcpListener; use tracing::info; @@ -24,6 +26,14 @@ pub struct GatewayListenerSpec { pub address: SocketAddr, pub scope: GatewayListenerScope, covered_addresses: Vec, + provenance: Option, +} + +/// Diagnostic source of a driver-requested listener. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GatewayListenerProvenance { + pub driver_name: String, + pub reason: String, } /// A gateway listener together with the context needed to serve it. @@ -38,6 +48,7 @@ impl GatewayListenerSpec { address, scope, covered_addresses: Vec::new(), + provenance: None, } } @@ -59,52 +70,248 @@ impl GatewayListenerSpec { fn gateway_listener_specs( bind_address: SocketAddr, - extra_addresses: &[SocketAddr], -) -> Vec { + requirements: &[GatewayListenerRequirement], +) -> Result> { + let needs_default_route_resolution = requirements.iter().any(|requirement| { + matches!( + requirement, + GatewayListenerRequirement::DefaultRouteInterface { .. } + ) + }); + let default_route_ip = if needs_default_route_resolution { + Some(gateway_default_route_ip()?) + } else { + None + }; + gateway_listener_specs_with_default_route_ip(bind_address, requirements, default_route_ip) +} + +fn gateway_listener_specs_with_default_route_ip( + bind_address: SocketAddr, + requirements: &[GatewayListenerRequirement], + default_route_ip: Option, +) -> Result> { let mut specs = vec![GatewayListenerSpec::new( bind_address, GatewayListenerScope::Primary, )]; - for address in extra_addresses { - let scope = GatewayListenerScope::ComputeDriverCallback; - if let Some(existing) = specs - .iter() - .position(|existing| listener_covers(existing.address, *address)) + + // Resolve exact requirements first so they can satisfy a later semantic + // requirement regardless of driver response ordering. + for requirement in requirements { + let GatewayListenerRequirement::Exact { address, .. } = requirement else { + continue; + }; + validate_gateway_listener_requirement(bind_address, requirement)?; + add_callback_listener_spec(&mut specs, *address, requirement); + } + + for requirement in requirements { + let GatewayListenerRequirement::DefaultRouteInterface { .. } = requirement else { + continue; + }; + validate_gateway_listener_requirement(bind_address, requirement)?; + let Some(ip) = default_route_ip else { + return Err(Error::config(format!( + "compute driver '{}' requested the gateway default-route interface, but no IPv4 source address was resolved (reason: {})", + requirement.driver_name(), + requirement.reason() + ))); + }; + if !gateway_default_route_ip_is_usable(ip) { + return Err(Error::config(format!( + "compute driver '{}' requested the gateway default-route interface, but its resolved address {ip} is not a private IPv4 address (reason: {})", + requirement.driver_name(), + requirement.reason() + ))); + } + let address = SocketAddr::new(ip, bind_address.port()); + validate_resolved_gateway_listener(bind_address, address)?; + add_callback_listener_spec(&mut specs, address, requirement); + } + + for requirement in requirements { + let GatewayListenerRequirement::LoopbackInterface { .. } = requirement else { + continue; + }; + validate_gateway_listener_requirement(bind_address, requirement)?; + let address = SocketAddr::from(([127, 0, 0, 1], bind_address.port())); + validate_resolved_gateway_listener(bind_address, address)?; + add_callback_listener_spec(&mut specs, address, requirement); + } + + Ok(specs) +} + +fn add_callback_listener_spec( + specs: &mut Vec, + address: SocketAddr, + requirement: &GatewayListenerRequirement, +) { + let scope = GatewayListenerScope::ComputeDriverCallback; + if let Some(existing) = specs + .iter_mut() + .find(|existing| listener_covers(existing.address, address)) + { + if existing.address != address + && !existing + .covered_addresses + .iter() + .any(|covered| covered.address == address) { - let existing = &mut specs[existing]; - if existing.address != *address - && !existing - .covered_addresses - .iter() - .any(|covered| covered.address == *address) - { - existing.covered_addresses.push(CoveredGatewayAddress { - address: *address, - scope, - }); - } - } else { - specs.push(GatewayListenerSpec::new(*address, scope)); + existing + .covered_addresses + .push(CoveredGatewayAddress { address, scope }); } + return; } - specs + specs.push(callback_listener_spec(address, requirement)); +} + +fn callback_listener_spec( + address: SocketAddr, + requirement: &GatewayListenerRequirement, +) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::ComputeDriverCallback, + covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: requirement.driver_name().to_string(), + reason: requirement.reason().to_string(), + }), + } +} + +fn validate_gateway_listener_requirement( + primary_listener: SocketAddr, + requirement: &GatewayListenerRequirement, +) -> Result<()> { + match requirement { + GatewayListenerRequirement::Exact { + address, + driver_name, + .. + } if driver_name == ComputeDriverKind::Docker.as_str() + || driver_name == ComputeDriverKind::Podman.as_str() => + { + validate_resolved_gateway_listener(primary_listener, *address) + } + GatewayListenerRequirement::DefaultRouteInterface { driver_name, .. } + | GatewayListenerRequirement::LoopbackInterface { driver_name, .. } + if driver_name == ComputeDriverKind::Podman.as_str() => + { + Ok(()) + } + _ => Err(Error::config(format!( + "compute driver '{}' is not authorized to request this gateway listener selector", + requirement.driver_name() + ))), + } +} + +fn validate_resolved_gateway_listener( + primary_listener: SocketAddr, + requested_listener: SocketAddr, +) -> Result<()> { + if requested_listener.ip().is_unspecified() { + return Err(Error::config(format!( + "compute driver requested wildcard gateway listener {requested_listener}" + ))); + } + if requested_listener.ip().is_multicast() { + return Err(Error::config(format!( + "compute driver requested multicast gateway listener {requested_listener}" + ))); + } + if requested_listener.port() == 0 { + return Err(Error::config(format!( + "compute driver requested zero-port gateway listener {requested_listener}" + ))); + } + if requested_listener.port() != primary_listener.port() { + return Err(Error::config(format!( + "compute driver requested gateway listener {requested_listener} with port {}, but the primary listener uses port {}", + requested_listener.port(), + primary_listener.port() + ))); + } + Ok(()) +} + +fn gateway_default_route_ip_is_usable(address: IpAddr) -> bool { + matches!(address, IpAddr::V4(address) if address.is_private()) +} + +#[cfg(target_os = "linux")] +fn gateway_default_route_ip() -> Result { + // UDP connect performs a local route lookup without sending a packet. The + // selected source address follows the IPv4 default route, matching pasta's + // default upstream-interface selection. + let socket = + std::net::UdpSocket::bind((std::net::Ipv4Addr::UNSPECIFIED, 0)).map_err(|err| { + Error::config(format!("failed to open default-route probe socket: {err}")) + })?; + socket + .connect((std::net::Ipv4Addr::new(192, 0, 2, 1), 9)) + .map_err(|err| Error::config(format!("failed to resolve IPv4 default route: {err}")))?; + socket + .local_addr() + .map(|address| address.ip()) + .map_err(|err| Error::config(format!("failed to read IPv4 default-route address: {err}"))) +} + +#[cfg(not(target_os = "linux"))] +fn gateway_default_route_ip() -> Result { + Err(Error::config( + "default-route gateway listener requirements are supported only on Linux", + )) } pub async fn bind_gateway_listeners( bind_address: SocketAddr, - extra_addresses: &[SocketAddr], + requirements: &[GatewayListenerRequirement], ) -> Result> { - let specs = gateway_listener_specs(bind_address, extra_addresses); + let specs = gateway_listener_specs(bind_address, requirements)?; let mut listeners = Vec::with_capacity(specs.len()); - for spec in specs { - let listener = TcpListener::bind(spec.address) + for spec in &specs { + let ipv6_only = matches!( + spec.address.ip(), + IpAddr::V6(address) if address.is_unspecified() + ) && specs.iter().any(|candidate| { + candidate.address.port() == spec.address.port() && candidate.address.is_ipv4() + }); + let listener = bind_gateway_listener(spec.address, ipv6_only) .await .map_err(|e| Error::transport(format!("failed to bind to {}: {e}", spec.address)))?; let local_addr = listener.local_addr().unwrap_or(spec.address); - info!(address = %local_addr, "Server listening"); + match spec.scope { + GatewayListenerScope::Primary => { + info!( + address = %local_addr, + listener_purpose = "primary", + authorization_scope = "full-multiplexed-api", + "Gateway listener bound" + ); + } + GatewayListenerScope::ComputeDriverCallback => { + let provenance = spec + .provenance + .as_ref() + .expect("callback listener spec must include provenance"); + info!( + address = %local_addr, + listener_purpose = "compute-driver-callback", + driver = %provenance.driver_name, + reason = %provenance.reason, + authorization_scope = "sandbox-callable-grpc-only", + "Gateway listener bound" + ); + } + } listeners.push(BoundGatewayListener { listener, - spec: spec.bind_to(local_addr), + spec: spec.clone().bind_to(local_addr), }); } Ok(listeners) @@ -140,6 +347,24 @@ fn resolve_ephemeral_port( } } +async fn bind_gateway_listener( + address: SocketAddr, + ipv6_only: bool, +) -> std::io::Result { + if ipv6_only { + let socket = Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP))?; + socket.set_reuse_address(true)?; + socket.set_only_v6(true)?; + socket.set_nonblocking(true)?; + socket.bind(&address.into())?; + socket.listen(1024)?; + let listener: std::net::TcpListener = socket.into(); + return TcpListener::from_std(listener); + } + + TcpListener::bind(address).await +} + fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { if existing == requested { return true; @@ -149,8 +374,8 @@ fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { } match (existing.ip(), requested.ip()) { - (std::net::IpAddr::V4(existing), std::net::IpAddr::V4(_)) => existing.is_unspecified(), - (std::net::IpAddr::V6(existing), std::net::IpAddr::V6(_)) => existing.is_unspecified(), + (IpAddr::V4(existing), IpAddr::V4(_)) => existing.is_unspecified(), + (IpAddr::V6(existing), IpAddr::V6(_)) => existing.is_unspecified(), _ => false, } } @@ -158,9 +383,11 @@ fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { #[cfg(test)] mod tests { use super::{ - CoveredGatewayAddress, GatewayListenerScope, GatewayListenerSpec, bind_gateway_listeners, - gateway_listener_specs, + CoveredGatewayAddress, GatewayListenerProvenance, GatewayListenerScope, + GatewayListenerSpec, bind_gateway_listeners, gateway_listener_specs, + gateway_listener_specs_with_default_route_ip, }; + use crate::compute::GatewayListenerRequirement; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::net::TcpListener; @@ -169,9 +396,13 @@ mod tests { fn gateway_listener_specs_track_driver_address_covered_by_wildcard() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let requirements = [ + docker_listener_requirement(docker), + docker_listener_requirement(docker), + ]; assert_eq!( - gateway_listener_specs(primary, &[docker, docker]), + gateway_listener_specs(primary, &requirements).unwrap(), vec![GatewayListenerSpec { address: primary, scope: GatewayListenerScope::Primary, @@ -179,6 +410,7 @@ mod tests { address: docker, scope: GatewayListenerScope::ComputeDriverCallback, }], + provenance: None, }] ); } @@ -188,7 +420,8 @@ mod tests { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let loopback: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let [spec] = gateway_listener_specs(primary, &[docker]) + let [spec] = gateway_listener_specs(primary, &[docker_listener_requirement(docker)]) + .unwrap() .try_into() .unwrap(); @@ -206,24 +439,206 @@ mod tests { fn gateway_listener_specs_preserve_driver_callback_scope() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let requirements = [ + docker_listener_requirement(docker), + docker_listener_requirement(docker), + ]; assert_eq!( - gateway_listener_specs(primary, &[docker, docker]), + gateway_listener_specs(primary, &requirements).unwrap(), vec![ GatewayListenerSpec { address: primary, scope: GatewayListenerScope::Primary, covered_addresses: Vec::new(), + provenance: None, }, GatewayListenerSpec { address: docker, scope: GatewayListenerScope::ComputeDriverCallback, covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: "docker".to_string(), + reason: "managed bridge".to_string(), + }), }, ] ); } + #[test] + fn gateway_listener_specs_reject_unauthorized_external_driver() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let requirement = GatewayListenerRequirement::Exact { + address: "172.18.0.1:8080".parse().unwrap(), + driver_name: "external-test".to_string(), + reason: "external bridge".to_string(), + }; + + let err = gateway_listener_specs(primary, &[requirement]).unwrap_err(); + assert!(err.to_string().contains("not authorized")); + } + + #[test] + fn gateway_listener_specs_reject_invalid_exact_addresses() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + for address in [ + "0.0.0.0:8080", + "224.0.0.1:8080", + "172.18.0.1:0", + "172.18.0.1:9090", + ] { + let requirement = docker_listener_requirement(address.parse().unwrap()); + assert!( + gateway_listener_specs(primary, &[requirement]).is_err(), + "{address} should be rejected" + ); + } + } + + #[test] + fn gateway_listener_specs_use_exact_podman_network_gateway() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)]) + .unwrap(), + vec![ + primary_listener_spec(primary), + callback_listener_spec(podman_gateway, "podman", "Podman managed bridge",), + ] + ); + } + + #[test] + fn gateway_listener_specs_track_podman_exact_when_primary_covers_it() { + let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); + let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)],) + .unwrap(), + vec![primary_listener_spec_with_covered(primary, podman_gateway,)] + ); + } + + #[test] + fn gateway_listener_specs_resolve_podman_default_route_source() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let default_route_ip = "192.168.20.20".parse().unwrap(); + + assert_eq!( + gateway_listener_specs_with_default_route_ip( + primary, + &[podman_default_route_listener_requirement()], + Some(default_route_ip), + ) + .unwrap(), + vec![ + primary_listener_spec(primary), + callback_listener_spec( + "192.168.20.20:8080".parse().unwrap(), + "podman", + "rootless pasta upstream interface", + ), + ] + ); + } + + #[test] + fn gateway_listener_specs_reject_public_default_route_source() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + + let err = gateway_listener_specs_with_default_route_ip( + primary, + &[podman_default_route_listener_requirement()], + Some("203.0.113.20".parse().unwrap()), + ) + .unwrap_err(); + + assert!(err.to_string().contains("not a private IPv4 address")); + } + + #[test] + fn gateway_listener_specs_track_default_route_when_primary_is_ipv4_wildcard() { + let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); + let default_route_ip = "192.168.20.20".parse().unwrap(); + let callback = "192.168.20.20:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs_with_default_route_ip( + primary, + &[podman_default_route_listener_requirement()], + Some(default_route_ip), + ) + .unwrap(), + vec![primary_listener_spec_with_covered(primary, callback)] + ); + } + + #[test] + fn gateway_listener_specs_resolve_podman_loopback_separately() { + let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + vec![ + primary_listener_spec(primary), + callback_listener_spec( + "127.0.0.1:8080".parse().unwrap(), + "podman", + "Podman machine host forwarder", + ), + ] + ); + } + + #[test] + fn gateway_listener_specs_track_podman_loopback_when_wildcard_primary_covers_it() { + let primary = "0.0.0.0:8080".parse().unwrap(); + let loopback = "127.0.0.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + vec![primary_listener_spec_with_covered(primary, loopback)] + ); + } + + #[test] + fn gateway_listener_specs_skip_podman_loopback_when_primary_is_same_address() { + let primary = "127.0.0.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + vec![primary_listener_spec(primary)] + ); + } + + #[test] + fn gateway_listener_specs_do_not_use_ipv6_listener_for_ipv4_loopback_requirement() { + for primary in ["[::1]:8080", "[::]:8080"] { + let primary = primary.parse().unwrap(); + let specs = + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(); + + assert_eq!(specs.len(), 2); + assert_eq!(specs[1].address, SocketAddr::from(([127, 0, 0, 1], 8080))); + } + } + + #[test] + fn gateway_listener_specs_reject_cross_driver_selector_authority() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let requirement = GatewayListenerRequirement::LoopbackInterface { + driver_name: "docker".to_string(), + reason: "wrong selector".to_string(), + }; + + let err = gateway_listener_specs(primary, &[requirement]).unwrap_err(); + assert!(err.to_string().contains("not authorized")); + } + #[tokio::test] async fn failed_bind_does_not_return_partially_bound_listeners() { let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -232,7 +647,11 @@ mod tests { let primary_address: SocketAddr = "127.0.0.1:0".parse().unwrap(); let result: openshell_core::Result<()> = async { - let _listeners = bind_gateway_listeners(primary_address, &[occupied_address]).await?; + let _listeners = bind_gateway_listeners( + primary_address, + &[docker_listener_requirement(occupied_address)], + ) + .await?; continuation_reached.store(true, Ordering::SeqCst); Ok(()) } @@ -247,4 +666,96 @@ mod tests { "binding must fail before returning a partial listener set" ); } + + #[tokio::test] + #[cfg(target_os = "linux")] + async fn gateway_listeners_bind_ipv6_wildcard_and_ipv4_callback_on_same_port() { + let probe = TcpListener::bind("[::1]:0") + .await + .expect("IPv6 loopback probe should bind"); + let port = probe.local_addr().unwrap().port(); + drop(probe); + + let primary = format!("[::]:{port}").parse().unwrap(); + let listeners = bind_gateway_listeners(primary, &[podman_loopback_listener_requirement()]) + .await + .expect("IPv6 wildcard and IPv4 callback listeners should both bind"); + + assert_eq!(listeners.len(), 2); + assert_eq!(listeners[0].address, primary); + assert_eq!( + listeners[1].address, + SocketAddr::from(([127, 0, 0, 1], port)) + ); + } + + fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + GatewayListenerRequirement::Exact { + address, + driver_name: "docker".to_string(), + reason: "managed bridge".to_string(), + } + } + + fn podman_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + GatewayListenerRequirement::Exact { + address, + driver_name: "podman".to_string(), + reason: "Podman managed bridge".to_string(), + } + } + + fn podman_default_route_listener_requirement() -> GatewayListenerRequirement { + GatewayListenerRequirement::DefaultRouteInterface { + driver_name: "podman".to_string(), + reason: "rootless pasta upstream interface".to_string(), + } + } + + fn podman_loopback_listener_requirement() -> GatewayListenerRequirement { + GatewayListenerRequirement::LoopbackInterface { + driver_name: "podman".to_string(), + reason: "Podman machine host forwarder".to_string(), + } + } + + fn primary_listener_spec(address: SocketAddr) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::Primary, + covered_addresses: Vec::new(), + provenance: None, + } + } + + fn primary_listener_spec_with_covered( + address: SocketAddr, + covered_address: SocketAddr, + ) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::Primary, + covered_addresses: vec![CoveredGatewayAddress { + address: covered_address, + scope: GatewayListenerScope::ComputeDriverCallback, + }], + provenance: None, + } + } + + fn callback_listener_spec( + address: SocketAddr, + driver_name: &str, + reason: &str, + ) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::ComputeDriverCallback, + covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: driver_name.to_string(), + reason: reason.to_string(), + }), + } + } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 485e564e4c..a476d32f49 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -72,8 +72,6 @@ use tracing::{debug, error, info, warn}; pub(crate) static TEST_ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); use compute::ComputeRuntime; -#[cfg(test)] -use gateway_listener::GatewayListenerSpec; use gateway_listener::{BoundGatewayListener, GatewayListenerScope, bind_gateway_listeners}; pub use grpc::OpenShellService; pub use http::{health_router, http_router, metrics_router, service_http_router}; @@ -430,8 +428,11 @@ pub(crate) async fn run_server( // snapshot on its first poll. ensure_default_workspace(&store).await?; - let gateway_listeners = - bind_gateway_listeners(config.bind_address, state.compute.gateway_bind_addresses()).await?; + let gateway_listeners = bind_gateway_listeners( + config.bind_address, + state.compute.gateway_listener_requirements(), + ) + .await?; if let Err(err) = state.compute.resume_persisted_sandboxes().await { warn!(error = %err, "Failed to resume persisted sandboxes during startup"); @@ -643,8 +644,12 @@ fn allow_plaintext_service_http( enabled: bool, listen_addr: SocketAddr, peer_addr: SocketAddr, + listener_scope: GatewayListenerScope, ) -> bool { - enabled && listen_addr.ip().is_loopback() && peer_addr.ip().is_loopback() + enabled + && matches!(listener_scope, GatewayListenerScope::Primary) + && listen_addr.ip().is_loopback() + && peer_addr.ip().is_loopback() } fn spawn_gateway_connection( @@ -664,6 +669,7 @@ fn spawn_gateway_connection( enable_loopback_service_http, listen_addr, addr, + listener_scope, ) => { if let Err(e) = service @@ -678,7 +684,12 @@ fn spawn_gateway_connection( } } Ok(ConnectionProtocol::PlainHttp) => { - warn!(client = %addr, listen = %listen_addr, "Rejected plaintext HTTP on non-loopback gateway listener"); + warn!( + client = %addr, + listen = %listen_addr, + scope = ?listener_scope, + "Rejected plaintext HTTP on gateway listener" + ); } Ok(ConnectionProtocol::Tls | ConnectionProtocol::Unknown) => { // acceptor.acceptor() snapshots the current TLS config; @@ -996,10 +1007,10 @@ pub(crate) async fn ensure_default_workspace(store: &Store) -> Result<()> { mod tests { use super::{ BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, GatewayListenerScope, - GatewayListenerSpec, MultiplexService, ServerState, TlsAcceptor, - allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, - kubernetes_sandbox_jwt_expiry_disabled, serve_gateway_listener, + MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, + bind_gateway_listeners, classify_initial_bytes, configured_compute_driver, + is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, + serve_gateway_listener, }; use openshell_core::{ ComputeDriverKind, Config, @@ -1017,7 +1028,11 @@ mod tests { use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; - use crate::tls_test_utils::{generate_test_certs_with_ca, install_rustls_provider}; + use crate::{ + compute::GatewayListenerRequirement, + gateway_listener::GatewayListenerSpec, + tls_test_utils::{generate_test_certs_with_ca, install_rustls_provider}, + }; fn test_driver_startup<'a>( config: &'a Config, @@ -1193,11 +1208,23 @@ mod tests { let peer: SocketAddr = "127.0.0.1:54000".parse().unwrap(); let wildcard: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let remote_peer: SocketAddr = "192.0.2.10:54000".parse().unwrap(); + let primary = GatewayListenerScope::Primary; + let callback = GatewayListenerScope::ComputeDriverCallback; - assert!(allow_plaintext_service_http(true, loopback, peer)); - assert!(!allow_plaintext_service_http(false, loopback, peer)); - assert!(!allow_plaintext_service_http(true, wildcard, peer)); - assert!(!allow_plaintext_service_http(true, loopback, remote_peer)); + assert!(allow_plaintext_service_http(true, loopback, peer, primary)); + assert!(!allow_plaintext_service_http( + false, loopback, peer, primary + )); + assert!(!allow_plaintext_service_http(true, wildcard, peer, primary)); + assert!(!allow_plaintext_service_http( + true, + loopback, + remote_peer, + primary + )); + assert!(!allow_plaintext_service_http( + true, loopback, peer, callback + )); } #[tokio::test] @@ -1482,7 +1509,11 @@ mod tests { let primary_address: SocketAddr = "127.0.0.1:0".parse().unwrap(); let result: openshell_core::Result<()> = async { - let _listeners = bind_gateway_listeners(primary_address, &[occupied_address]).await?; + let _listeners = bind_gateway_listeners( + primary_address, + &[docker_listener_requirement(occupied_address)], + ) + .await?; resume_attempted.store(true, Ordering::SeqCst); Ok(()) } @@ -1497,4 +1528,12 @@ mod tests { "persisted sandbox resume must not run before every gateway listener is bound" ); } + + fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + GatewayListenerRequirement::Exact { + address, + driver_name: "docker".to_string(), + reason: "managed bridge".to_string(), + } + } } diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index e7e22b7174..2fed379ddf 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -7,7 +7,7 @@ //! to either the gRPC service or HTTP endpoints based on the request headers. use bytes::{Bytes, BytesMut}; -use http::{Extensions, HeaderValue, Request, Response}; +use http::{Extensions, HeaderValue, Request, Response, StatusCode}; use http_body::Body; use http_body_util::{BodyExt, Full, LengthLimitError, Limited, StreamBody}; use hyper::body::Incoming; @@ -149,7 +149,7 @@ impl MultiplexService { .await } - /// Serve a connection and preserve its listener purpose in request + /// Serve a connection and preserve its listener scope in request /// extensions for downstream routing and policy decisions. pub(crate) async fn serve_on_listener( &self, @@ -180,7 +180,7 @@ impl MultiplexService { .await } - /// Serve a TLS connection and preserve its listener purpose in request + /// Serve a TLS connection and preserve its listener scope in request /// extensions for downstream routing and policy decisions. pub(crate) async fn serve_with_peer_identity_on_listener( &self, @@ -260,7 +260,7 @@ impl MultiplexService { } /// Serve a plaintext service HTTP connection and preserve its listener - /// purpose in request extensions. + /// scope in request extensions. pub(crate) async fn serve_service_http_on_listener( &self, stream: S, @@ -994,6 +994,38 @@ impl MultiplexedService { } } +fn listener_allows_request( + listener_scope: Option<&GatewayListenerScope>, + is_grpc: bool, + path: &str, +) -> bool { + match listener_scope { + Some(GatewayListenerScope::ComputeDriverCallback) => { + is_grpc && crate::auth::sandbox_methods::is_sandbox_callable(path) + } + Some(GatewayListenerScope::Primary) | None => true, + } +} + +fn callback_listener_rejection(is_grpc: bool) -> Response { + if is_grpc { + let response: Response = tonic::Status::permission_denied( + "compute-driver callback listeners accept sandbox callback RPCs only", + ) + .into_http(); + let (parts, body) = response.into_parts(); + let body = body.map_err(Into::into).boxed_unsync(); + Response::from_parts(parts, BoxBody(body)) + } else { + Response::builder() + .status(StatusCode::FORBIDDEN) + .body(boxed_body_from_bytes(Bytes::from_static( + b"compute-driver callback listeners accept gRPC callbacks only", + ))) + .expect("static callback listener rejection response must be valid") + } +} + impl hyper::service::Service> for MultiplexedService where G: tower::Service, Response = Response> + Clone + Send + 'static, @@ -1017,6 +1049,15 @@ where .get("content-type") .is_some_and(|v| v.as_bytes().starts_with(b"application/grpc")); + if !listener_allows_request( + req.extensions().get::(), + is_grpc, + req.uri().path(), + ) { + let response = callback_listener_rejection(is_grpc); + return Box::pin(async move { Ok(response) }); + } + if is_grpc { let method = grpc_method_from_path(req.uri().path()); let start = Instant::now(); @@ -1204,6 +1245,98 @@ mod tests { ); } + fn callback_listener_scope() -> GatewayListenerScope { + GatewayListenerScope::ComputeDriverCallback + } + + #[test] + fn callback_listener_allows_sandbox_callback_rpcs() { + let scope = callback_listener_scope(); + let callback_paths = [ + "/openshell.v1.OpenShell/ConnectSupervisor", + "/openshell.v1.OpenShell/RelayStream", + "/openshell.v1.OpenShell/GetSandboxConfig", + "/openshell.v1.OpenShell/ReportPolicyStatus", + "/openshell.v1.OpenShell/PushSandboxLogs", + "/openshell.v1.OpenShell/GetSandboxProviderEnvironment", + "/openshell.v1.OpenShell/SubmitPolicyAnalysis", + "/openshell.v1.OpenShell/RefreshSandboxToken", + "/openshell.inference.v1.Inference/GetInferenceBundle", + ]; + + for path in callback_paths { + assert!( + listener_allows_request(Some(&scope), true, path), + "callback listener should allow {path}" + ); + } + } + + #[test] + fn callback_listener_surface_matches_rpc_auth_metadata() { + let scope = callback_listener_scope(); + + for path in crate::auth::method_authz::all_paths() { + assert_eq!( + listener_allows_request(Some(&scope), true, path), + crate::auth::method_authz::is_sandbox_callable(path), + "callback listener exposure must follow rpc_auth metadata for {path}" + ); + } + } + + #[test] + fn callback_listener_rejects_non_callback_routes() { + let scope = callback_listener_scope(); + let rejected_grpc_paths = [ + "/grpc.health.v1.Health/Check", + "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo", + "/openshell.v1.OpenShell/ListSandboxes", + "/openshell.v1.OpenShell/DeleteSandbox", + "/openshell.v1.OpenShell/CreateProvider", + "/openshell.inference.v1.Inference/GetInferenceRoute", + "/openshell.inference.v1.Inference/SetInferenceRoute", + ]; + + for path in rejected_grpc_paths { + assert!( + !listener_allows_request(Some(&scope), true, path), + "callback listener should reject {path}" + ); + } + assert!(!listener_allows_request(Some(&scope), false, "/health")); + assert!(!listener_allows_request(Some(&scope), false, "/service")); + } + + #[test] + fn primary_listener_routing_is_unchanged() { + let primary = GatewayListenerScope::Primary; + let paths = [ + "/grpc.health.v1.Health/Check", + "/openshell.v1.OpenShell/ListSandboxes", + "/openshell.inference.v1.Inference/GetInferenceRoute", + "/health", + "/service", + ]; + + for path in paths { + assert!(listener_allows_request(Some(&primary), true, path)); + assert!(listener_allows_request(Some(&primary), false, path)); + assert!(listener_allows_request(None, true, path)); + assert!(listener_allows_request(None, false, path)); + } + } + + #[test] + fn callback_listener_rejections_use_protocol_appropriate_statuses() { + let grpc = callback_listener_rejection(true); + assert_eq!(grpc.status(), StatusCode::OK); + assert_eq!(grpc.headers().get("grpc-status").unwrap(), "7"); + + let http = callback_listener_rejection(false); + assert_eq!(http.status(), StatusCode::FORBIDDEN); + } + #[derive(Clone)] struct PostCommitTestInterceptor; @@ -1305,6 +1438,12 @@ mod tests { } async fn start_http_server_with_middleware() -> std::net::SocketAddr { + start_http_server_with_middleware_on_listener(GatewayListenerScope::Primary).await + } + + async fn start_http_server_with_middleware_on_listener( + listener_scope: GatewayListenerScope, + ) -> std::net::SocketAddr { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -1312,6 +1451,7 @@ mod tests { let http_service = request_id_middleware!(http_service); let service = MultiplexedService::new(http_service.clone(), http_service); + let service = GatewayListenerContextService::new(service, listener_scope); tokio::spawn(async move { loop { @@ -1330,8 +1470,9 @@ mod tests { addr } - async fn http1_get( + async fn http1_request( addr: std::net::SocketAddr, + method: &str, path: &str, headers: &[(&str, &str)], ) -> Response { @@ -1345,7 +1486,7 @@ mod tests { }); let mut builder = Request::builder() - .method("GET") + .method(method) .uri(format!("http://{addr}{path}")); for (k, v) in headers { builder = builder.header(*k, *v); @@ -1354,6 +1495,47 @@ mod tests { sender.send_request(req).await.unwrap() } + async fn http1_get( + addr: std::net::SocketAddr, + path: &str, + headers: &[(&str, &str)], + ) -> Response { + http1_request(addr, "GET", path, headers).await + } + + #[tokio::test] + async fn callback_listener_filter_is_applied_before_route_dispatch() { + let addr = start_http_server_with_middleware_on_listener(callback_listener_scope()).await; + + let health = http1_get(addr, "/healthz", &[]).await; + assert_eq!(health.status(), StatusCode::FORBIDDEN); + + let admin = http1_request( + addr, + "POST", + "/openshell.v1.OpenShell/ListSandboxes", + &[("content-type", "application/grpc")], + ) + .await; + assert_eq!(admin.status(), StatusCode::OK); + assert_eq!(admin.headers().get("grpc-status").unwrap(), "7"); + + let callback = http1_request( + addr, + "POST", + "/openshell.v1.OpenShell/ConnectSupervisor", + &[("content-type", "application/grpc")], + ) + .await; + assert_ne!( + callback + .headers() + .get("grpc-status") + .and_then(|value| value.to_str().ok()), + Some("7") + ); + } + #[tokio::test] async fn intercepted_grpc_body_collection_rejects_oversized_body() { let oversized = Bytes::from(vec![0_u8; MAX_INTERCEPTED_GRPC_BODY_SIZE + 1]); diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 9cd80d6ed8..2bfa9998a2 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -8,10 +8,12 @@ use futures::{Stream, stream}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverSandbox, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, - StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, - WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, + DriverSandbox, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, }; use std::collections::HashMap; #[cfg(unix)] @@ -33,6 +35,7 @@ type WatchStream = Pin #[derive(Debug, Clone, PartialEq)] pub enum FakeComputeDriverCall { GetCapabilities, + GetGatewayListenerRequirements, ValidateSandboxCreate { sandbox: Option, }, @@ -65,6 +68,8 @@ struct FakeComputeDriverState { driver_name: String, driver_version: String, default_image: String, + gateway_listener_requirements: Vec, + gateway_listener_requirements_supported: bool, sandboxes: HashMap, calls: Vec, } @@ -83,6 +88,8 @@ impl FakeComputeDriver { driver_name: "fake-compute-driver".to_string(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), + gateway_listener_requirements: Vec::new(), + gateway_listener_requirements_supported: true, sandboxes: HashMap::new(), calls: Vec::new(), })), @@ -107,6 +114,29 @@ impl FakeComputeDriver { self } + #[must_use] + pub fn with_gateway_listener_requirement( + self, + bind_address: impl Into, + reason: impl Into, + ) -> Self { + self.with_state(|state| { + state + .gateway_listener_requirements + .push(GatewayListenerRequirement { + reason: reason.into(), + selector: Some(Selector::ExactBindAddress(bind_address.into())), + }); + }); + self + } + + #[must_use] + pub fn without_gateway_listener_requirements_api(self) -> Self { + self.with_state(|state| state.gateway_listener_requirements_supported = false); + self + } + #[must_use] pub fn calls(&self) -> Vec { self.with_state(|state| state.calls.clone()) @@ -194,6 +224,24 @@ impl ComputeDriver for FakeComputeDriver { Ok(Response::new(response)) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + self.with_state(|state| { + state + .calls + .push(FakeComputeDriverCall::GetGatewayListenerRequirements); + state + .gateway_listener_requirements_supported + .then(|| GetGatewayListenerRequirementsResponse { + requirements: state.gateway_listener_requirements.clone(), + }) + .map(Response::new) + .ok_or_else(|| Status::unimplemented("listener requirements unsupported")) + }) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index a144cac8ed..4fc18e6215 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -20,14 +20,13 @@ The defaults are tuned for rootless Podman use: version = 1 [openshell.gateway] -bind_address = "0.0.0.0:17670" compute_drivers = ["podman"] ``` -`bind_address = "0.0.0.0:17670"` is required because Podman sandbox -containers reach the gateway over the host network bridge and cannot -connect to `127.0.0.1` inside the gateway's network namespace. mTLS is -enabled by default and protects all connections. +The RPM does not override `bind_address`. The primary listener uses the +built-in `127.0.0.1:17670` default. The Podman driver reports the callback +interface it needs, and the gateway adds a separate listener scoped to that +interface. This keeps the general API off unrelated host interfaces. `compute_drivers = ["podman"]` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning @@ -43,8 +42,8 @@ To apply environment variable overrides that persist across upgrades without editing the TOML file, add them to `~/.config/openshell/gateway.env`: ```shell -# Example: restrict to loopback only -OPENSHELL_BIND_ADDRESS=127.0.0.1 +# Example: explicitly expose the primary listener on one host interface +OPENSHELL_BIND_ADDRESS=192.168.1.10 ``` To override the path to the TOML config file entirely: @@ -63,8 +62,9 @@ systemctl --user edit openshell-gateway ## TLS (mTLS) The RPM enables mutual TLS by default. The gateway requires a valid -client certificate for all API connections and listens on -`0.0.0.0:17670` by default (see "Default configuration" above). +client certificate for all API connections. Its primary listener uses +`127.0.0.1:17670`; Podman callback traffic uses the additional listener +described in "Default configuration" above. ### Auto-generated certificates @@ -214,7 +214,7 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| -| `bind_address` | `0.0.0.0:17670` (RPM default) | Address for the gRPC/HTTP API. | +| `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | | `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | | `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | @@ -235,7 +235,6 @@ settings: version = 1 [openshell.gateway] -bind_address = "0.0.0.0:17670" compute_drivers = ["podman"] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" diff --git a/deploy/rpm/QUICKSTART.md b/deploy/rpm/QUICKSTART.md index c6634ced95..442458d09d 100644 --- a/deploy/rpm/QUICKSTART.md +++ b/deploy/rpm/QUICKSTART.md @@ -65,11 +65,11 @@ On first start, the gateway automatically generates: - A self-signed PKI bundle (CA, server cert, client cert) for mTLS -> **Note:** The RPM default configuration binds to `0.0.0.0:17670` so -> Podman sandbox containers can reach the gateway over the host network -> bridge. Mutual TLS (mTLS) is enabled automatically on first start, -> requiring a valid client certificate for every connection. See -> CONFIGURATION.md for details. +> **Note:** The primary gateway listener uses the loopback default, +> `127.0.0.1:17670`. The Podman driver requests a separate callback listener +> scoped to the interface its sandboxes can reach. Mutual TLS (mTLS) is +> enabled automatically on first start, requiring a valid client certificate +> for every connection. See CONFIGURATION.md for details. Verify the service is running: diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 68a1f49464..103ce3bf9d 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -85,7 +85,7 @@ Generate certificates that include the server's hostname or IP in the SANs. See "Using externally-managed certificates" in CONFIGURATION.md. Then change `bind_address` in `~/.config/openshell/gateway.toml` to the interface the remote CLI -can reach, for example `0.0.0.0:17670`, and restart the gateway. +can reach, for example `192.168.1.10:17670`, and restart the gateway. After placing the server and client certs, register from the remote CLI: @@ -274,11 +274,12 @@ Other breaking changes in this release: - **Default bind address changed from `0.0.0.0` to `127.0.0.1`.** If you relied on network-accessible access without an explicit bind - address, add the following to `~/.config/openshell/gateway.toml`: + address, bind the specific reachable interface in + `~/.config/openshell/gateway.toml`: ```toml [openshell.gateway] - bind_address = "0.0.0.0:17670" + bind_address = "192.168.1.10:17670" ``` Also update your firewall rule if applicable: diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index d853799640..cd7e0d99c3 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -18,11 +18,9 @@ version = 1 [openshell.gateway] -# Podman sandbox containers reach the gateway over the host network bridge, -# which requires binding to all interfaces. Override to 127.0.0.1:17670 if -# you don't use Podman or want loopback-only access (e.g. behind a reverse -# proxy). mTLS is enabled by default and protects all connections. -bind_address = "0.0.0.0:17670" +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 27a1d3d45a..02de4b39df 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -29,6 +29,8 @@ Package-managed gateways do not require a TOML file. Create one at the package's | Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | +The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. + ## Layout The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Shared keys set at gateway scope are inherited into driver tables when not overridden. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 6700a22c9f..1673743338 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -107,6 +107,12 @@ It overrides the gateway's configured default runtime class for that sandbox, while a typed `SandboxTemplate.runtime_class_name` value from the API still takes precedence. +Docker and Podman callback listeners accept only supervisor callback gRPC +methods. Use the gateway's primary endpoint for CLI, administrator, health, +reflection, inference-route management, and HTTP requests. A +`PermissionDenied` response from one of the sandbox-visible callback addresses +is expected for those requests. + ## Docker Driver [Docker](https://www.docker.com/get-started/)-backed sandboxes run as containers on the gateway host. Use Docker for local development, single-machine gateways, and hosts that already use Docker Desktop or Docker Engine. @@ -186,7 +192,7 @@ Podman sandboxes default to a 45-second graceful stop window before Podman escal For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. +On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts diff --git a/e2e/python/test_security_tls.py b/e2e/python/test_security_tls.py index fa9059fa49..529404a6e4 100644 --- a/e2e/python/test_security_tls.py +++ b/e2e/python/test_security_tls.py @@ -234,11 +234,14 @@ def test_plaintext_connection_rejected( stub = openshell_pb2_grpc.OpenShellStub(channel) with pytest.raises(grpc.RpcError) as exc_info: stub.Health(openshell_pb2.HealthRequest(), timeout=10) - # Plaintext to a TLS port will fail at the transport level. + # The loopback listener may intentionally accept plaintext service + # HTTP. A gRPC request is still rejected, either at the transport + # boundary or as an unimplemented HTTP route. assert exc_info.value.code() in ( grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.UNKNOWN, grpc.StatusCode.INTERNAL, - ), f"expected transport failure, got {exc_info.value.code()}" + grpc.StatusCode.UNIMPLEMENTED, + ), f"expected plaintext gRPC rejection, got {exc_info.value.code()}" finally: channel.close() diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 64062b74d6..f312b70038 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -498,7 +498,6 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - --bind-address 0.0.0.0 --port "${HOST_PORT}" --drivers docker --tls-cert "${PKI_DIR}/server/tls.crt" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index e9e9fb2be2..d40ea37e6c 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -408,10 +408,10 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # Start from the RPM default template so this e2e test exercises the same -# TOML config path that RPM users get on first start. The template sets -# bind_address = "0.0.0.0:17670" and compute_drivers = ["podman"]; those -# values must be correct for Podman e2e to pass, which means a regression -# to the template (wrong bind address, wrong driver) will surface here. +# TOML config path that RPM users get on first start. The template leaves +# bind_address unset and sets compute_drivers = ["podman"], so this test +# exercises the built-in loopback listener plus the callback listener +# requested by the Podman driver. # # We append the driver-specific table and override the port via CLI flag # (CLI > TOML in the merge precedence) so the test can use an ephemeral port. @@ -445,8 +445,8 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # bind_address and compute_drivers come from the RPM template; no CLI flags - # needed. Port is overridden via CLI (CLI > TOML) for ephemeral port selection. + # compute_drivers comes from the RPM template, while bind_address uses the + # built-in loopback default. Override only the port for ephemeral selection. --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" --tls-cert "${PKI_DIR}/server/tls.crt" diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index c99b7756b0..e3f18af19f 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -20,6 +20,13 @@ service ComputeDriver { // Report driver capabilities and defaults. rpc GetCapabilities(GetCapabilitiesRequest) returns (GetCapabilitiesResponse); + // Report additional gateway listeners required by this driver instance. + // + // A requirement is not authorization to expose the gateway. The gateway + // owns validation, authorization, and the authoritative bind. + rpc GetGatewayListenerRequirements(GetGatewayListenerRequirementsRequest) + returns (GetGatewayListenerRequirementsResponse); + // Validate a sandbox before create-time provisioning. rpc ValidateSandboxCreate(ValidateSandboxCreateRequest) returns (ValidateSandboxCreateResponse); @@ -57,6 +64,32 @@ message GetCapabilitiesResponse { string default_image = 3; } +message GetGatewayListenerRequirementsRequest {} + +message GatewayListenerRequirement { + // Untrusted human-readable driver rationale for diagnostics. + string reason = 1; + + oneof selector { + // Concrete IP:port address requested by the driver. The port must match + // the gateway's configured primary listener port. + string exact_bind_address = 2; + // Ask the gateway to bind the IPv4 address selected by its default route. + // This matches rootless pasta's default upstream-interface selection. + GatewayDefaultRouteInterfaceRequirement default_route_interface = 3; + // Ask the gateway to ensure an IPv4 loopback listener is present. This + // covers runtimes whose host forwarder terminates on gateway loopback. + GatewayLoopbackInterfaceRequirement loopback_interface = 4; + } +} + +message GatewayDefaultRouteInterfaceRequirement {} +message GatewayLoopbackInterfaceRequirement {} + +message GetGatewayListenerRequirementsResponse { + repeated GatewayListenerRequirement requirements = 1; +} + // Driver-owned sandbox model used for create requests and platform observations. // // This intentionally omits gateway-owned lifecycle fields such as the public diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 3e94afe108..5d3adae2a8 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -306,15 +306,6 @@ if [[ "${DRIVER}" == "podman" ]]; then SUPERVISOR_IMAGE="${OPENSHELL_SUPERVISOR_IMAGE:-openshell/supervisor:dev}" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" export OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" - - # Rootless Podman containers reach the host via pasta's local connection - # bypass, which translates to host L4 sockets. The gateway must listen on - # 0.0.0.0 so pasta can reach it — 127.0.0.1 is not routable through pasta. - if [[ -z "${OPENSHELL_BIND_ADDRESS:-}" ]]; then - if podman info --format '{{.Host.Security.Rootless}}' 2>/dev/null | grep -q true; then - export OPENSHELL_BIND_ADDRESS="0.0.0.0" - fi - fi fi if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then