feat(observability): export gateway traces over OTLP

Cargo.toml

@@ -60,6 +60,13 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 tracing-appender = "0.2"
 
+# OpenTelemetry — OTLP/gRPC export. Kept in lockstep with the workspace's
+# tonic 0.14 / prost 0.14 via opentelemetry-proto's `grpc-tonic` feature.
+opentelemetry = "0.32"
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+tracing-opentelemetry = { version = "0.33", default-features = false, features = ["tracing-log"] }
+
 # Metrics
 metrics = "0.24"
 metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] }

architecture/gateway.md

@@ -596,6 +596,36 @@ Driver-specific values that are not part of the inheritance allowlist
 (e.g. Podman `socket_path`, VM `vcpus`) only come from the driver's own
 table.
 
+### OTLP export
+
+The gateway already uses Rust's `tracing` framework for structured log events
+and request-span context consumed by stdout and the sandbox log bus. OTLP export
+adds an OpenTelemetry layer to the same subscriber. That layer turns selected
+`tracing` spans into distributed traces; it does not export log events or
+replace the existing logging paths.
+
+`[openshell.gateway.otlp]` is the only enablement path for OpenTelemetry
+export: the table's presence is the on-switch, and `OTEL_EXPORTER_OTLP_ENDPOINT`
+is ignored so enablement has a single source. TOML decides whether and where
+to export; the SDK's `OTEL_*` variables tune how. Transport is OTLP over gRPC
+only.
+
+Span emission requires no per-handler instrumentation. The `tower_http`
+`TraceLayer` in `multiplex.rs` opens a span per inbound request, and that span
+continues incoming W3C trace context when present or starts a new trace
+otherwise. It is named for the RPC and carries the request ID that also appears
+in the gateway's logs — the identifier that lets an operator pivot between a
+trace and its log lines. Store and compute-driver spans become children of the
+request span. Reconciliation, provider refresh, and driver-watch loops create
+their own operation spans because they have no inbound request to provide a
+parent. gRPC status is recorded when response trailers arrive.
+
+Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP
+failure stops the gateway from serving: a malformed endpoint is logged at
+startup and disables export. Export is best-effort — the SDK logs runtime
+failures, and a failed batch is dropped rather than retried. Buffered spans
+flush after the server loop exits so `SIGTERM` does not drop in-flight traces.
+
 ### Package-managed gateway registry
 
 The CLI reads its active-gateway and per-gateway metadata from

crates/openshell-server/Cargo.toml

@@ -69,6 +69,12 @@ anyhow = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 
+# OpenTelemetry (OTLP trace export, opt-in via [openshell.gateway.otlp])
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
+opentelemetry-otlp = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+
 # Metrics
 metrics = { workspace = true }
 metrics-exporter-prometheus = { workspace = true }
@@ -118,6 +124,11 @@ rcgen = { version = "0.13", features = ["crypto", "pem"] }
 tokio-tungstenite = { workspace = true }
 futures-util = "0.3"
 wiremock = "0.6"
+# `testing` provides InMemorySpanExporter, so span assertions do not need a
+# collector, a network hop, or a flush barrier.
+opentelemetry_sdk = { workspace = true, features = ["testing"] }
+# Supports asserting that the gateway sends spans over OTLP.
+opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] }
 
 [lints]
 workspace = true

crates/openshell-server/src/cli.rs

@@ -10,7 +10,7 @@ use openshell_core::ComputeDriverKind;
 use openshell_core::config::DEFAULT_SERVER_PORT;
 use std::net::{IpAddr, SocketAddr};
 use std::path::PathBuf;
-use tracing::{info, warn};
+use tracing::{error, info, warn};
 use tracing_subscriber::EnvFilter;
 
 use crate::certgen;
@@ -440,9 +440,15 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> {
     let prepared = prepare_server_config(&mut args, &matches)?;
 
     let tracing_log_bus = TracingLogBus::new();
-    tracing_log_bus.install_subscriber(
+    let otlp_config = prepared
+        .config_file
+        .as_ref()
+        .and_then(|f| f.openshell.gateway.otlp.as_ref());
+    let (tracing_handle, setup_error) = crate::tracing_setup::install(
         EnvFilter::try_from_default_env()
             .unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)),
+        &tracing_log_bus,
+        otlp_config,
     );
 
     let has_client_ca = prepared
@@ -468,6 +474,18 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> {
     if has_oidc {
         info!("OIDC authentication enabled");
     }
+    if let Some(err) = &setup_error {
+        error!(
+            error = %err,
+            "OTLP exporting is configured but could not be started; continuing without it"
+        );
+    } else if let Some(otlp) = prepared
+        .config_file
+        .as_ref()
+        .and_then(|f| f.openshell.gateway.otlp.as_ref())
+    {
+        info!(endpoint = %otlp.endpoint, "OTLP exporting enabled");
+    }
     if prepared.config.auth.allow_unauthenticated_users {
         warn!(
             "Unauthenticated user access enabled — only use this for trusted local development or a fully trusted fronting proxy"
@@ -487,9 +505,11 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> {
 
     info!(bind = %prepared.config.bind_address, "Starting OpenShell server");
 
-    Box::pin(run_server(prepared, tracing_log_bus))
-        .await
-        .into_diagnostic()
+    let result = Box::pin(run_server(prepared, tracing_log_bus)).await;
+
+    tracing_handle.shutdown();
+
+    result.into_diagnostic()
 }
 
 fn parse_compute_driver(value: &str) -> std::result::Result<String, String> {