From 9de27e5938d848f7c1ff89e93ade091bc98c7108 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 10:54:57 -0700 Subject: [PATCH 01/29] test(vm): add composable QEMU test guests Signed-off-by: Drew Newberry --- flake.nix | 5 + nix/vm/README.md | 155 +++++++++++ nix/vm/configuration/docker.yml | 67 +++++ nix/vm/configuration/podman.yml | 34 +++ nix/vm/configuration/selinux.yml | 62 +++++ nix/vm/default.nix | 109 ++++++++ nix/vm/distros/centos.nix | 19 ++ nix/vm/distros/rocky.nix | 19 ++ nix/vm/distros/ubuntu.nix | 22 ++ nix/vm/run.sh | 430 +++++++++++++++++++++++++++++++ tasks/scripts/package-deb.sh | 18 -- 11 files changed, 922 insertions(+), 18 deletions(-) create mode 100644 nix/vm/README.md create mode 100644 nix/vm/configuration/docker.yml create mode 100644 nix/vm/configuration/podman.yml create mode 100644 nix/vm/configuration/selinux.yml create mode 100644 nix/vm/default.nix create mode 100644 nix/vm/distros/centos.nix create mode 100644 nix/vm/distros/rocky.nix create mode 100644 nix/vm/distros/ubuntu.nix create mode 100644 nix/vm/run.sh diff --git a/flake.nix b/flake.nix index 13c4857bc6..aa4ce8829a 100644 --- a/flake.nix +++ b/flake.nix @@ -37,11 +37,16 @@ projectRootFile = "flake.nix"; programs.nixfmt.enable = true; }; + testVm = import ./nix/vm { inherit pkgs; }; in { + apps.test-vm = testVm.app; + devShells.default = pkgs.mkShell { packages = with pkgs; [ rustToolchain + # Assemble Debian artifacts on macOS and Linux. + dpkg # Required to find packages pkg-config # Required for bindgen generation. diff --git a/nix/vm/README.md b/nix/vm/README.md new file mode 100644 index 0000000000..862b2ba0e2 --- /dev/null +++ b/nix/vm/README.md @@ -0,0 +1,155 @@ + + +# Test VMs + +This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS and KVM on native-architecture Linux hosts. + +## Requirements + +- Nix with flakes enabled. +- Apple Silicon macOS with HVF, or Linux with readable and writable `/dev/kvm`. +- Enough local capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay. +- Native-architecture artifacts. The runner does not use software emulation. + +The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. + +## Supported configurations + +| Distro | Docker | Podman | SELinux | Package format | +| --- | --- | --- | --- | --- | +| Ubuntu 24.04 | Yes | Yes | No | `.deb` | +| CentOS Stream 10 | No | Yes | Yes | `.rpm` | +| Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | + +List the available distros and configurations: + +```shell +nix run .#test-vm -- --list +``` + +## Open an interactive VM + +Boot a base Ubuntu VM: + +```shell +nix run .#test-vm -- --distro ubuntu +``` + +Apply the Docker configuration before opening the SSH session: + +```shell +nix run .#test-vm -- --distro ubuntu --with docker +``` + +Other combinations use the same interface: + +```shell +nix run .#test-vm -- --distro rocky --with docker +nix run .#test-vm -- --distro centos --with podman +``` + +Configurations are repeatable: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --with docker \ + --with podman +``` + +Ensure SELinux is enforcing on CentOS or Rocky: + +```shell +nix run .#test-vm -- \ + --distro rocky \ + --with docker \ + --with selinux \ + -- getenforce +``` + +`--with selinux` installs the required tooling, persists `SELINUX=enforcing`, applies enforcing mode live, and verifies the result. It fails on Ubuntu and on guests where SELinux is fully disabled and would require a reboot to enable. + +## Ansible configurations + +Configurations are Ansible playbooks stored under `nix/vm/configuration/`. Ansible runs on the host using the VM's ephemeral SSH key and loopback port. The guest does not install Ansible. + +Configurations run in the order provided on the command line. OpenShell packages and copied binaries are installed after all configurations succeed. + +## Install an OpenShell package + +Package existing ARM64 Linux binaries with the repository's `package:deb:arm64` mise task: + +```shell +OPENSHELL_CLI_BINARY="$PWD/target/aarch64-unknown-linux-musl/release/openshell" \ +OPENSHELL_GATEWAY_BINARY="$PWD/target/aarch64-unknown-linux-gnu/release/openshell-gateway" \ +OPENSHELL_DRIVER_VM_BINARY="$PWD/target/aarch64-unknown-linux-gnu/release/openshell-driver-vm" \ +OPENSHELL_DEB_VERSION=0.0.0-local \ +OPENSHELL_OUTPUT_DIR="$PWD/artifacts" \ +nix develop --command mise run package:deb:arm64 +``` + +Install the package in an Ubuntu VM and run a command: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --with docker \ + --install artifacts/openshell_0.0.0-local_arm64.deb \ + -- openshell --version +``` + +For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the host and guest architecture. + +`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS and Rocky Linux. This prototype can install an existing RPM but does not build one. + +## Copy binaries directly + +Use `--copy SOURCE:DEST` to install an executable without creating a package: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --copy ./openshell:/usr/local/bin/openshell \ + -- openshell --version +``` + +The destination must be an absolute guest path. Copied files are installed with mode `0755`. + +## Runner options + +```text +--distro NAME Base distro: ubuntu, centos, or rocky +--with NAME Apply docker, podman, or selinux; repeatable +--install PATH Install a .deb or .rpm package; repeatable +--copy SRC:DEST Copy an executable into the guest; repeatable +--ssh-port PORT Use a specific loopback SSH forwarding port +--keep Preserve the disk overlay and logs after shutdown +--list List distros and configurations +``` + +Arguments after `--` are executed inside the guest. Without a command, the runner opens an interactive SSH session. + +## Lifecycle + +Nix downloads and caches the selected, hash-pinned cloud image. Each invocation then: + +1. Creates a temporary QCOW2 overlay. +2. Boots QEMU with HVF or KVM acceleration. +3. Creates an ephemeral `openshell` user and SSH key through cloud-init. +4. Applies the selected Ansible configurations. +5. Installs or copies the supplied artifacts. +6. Opens SSH or executes the requested guest command. +7. Powers off QEMU and deletes the overlay. + +Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log for debugging. The retained directory is printed when the runner exits. + +## Current limitations + +- Host and guest architectures must match. +- Software emulation without HVF or KVM is not supported. +- Configurations are applied fresh on every run; prepared VM caching is not implemented. +- Only loopback SSH is forwarded to the host; guest gateway ports are not exposed. +- The runner does not build OpenShell, configure a gateway, or select an E2E test suite. diff --git a/nix/vm/configuration/docker.yml b/nix/vm/configuration/docker.yml new file mode 100644 index 0000000000..1c158d3653 --- /dev/null +++ b/nix/vm/configuration/docker.yml @@ -0,0 +1,67 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure Docker in a disposable test VM. + +- name: Configure Docker + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate Docker support + ansible.builtin.assert: + that: + - ansible_facts.distribution in ["Ubuntu", "Rocky"] + fail_msg: >- + Docker is supported on Ubuntu and Rocky in this prototype, + not {{ ansible_facts.distribution }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + when: ansible_facts.distribution == "Ubuntu" + + - name: Install Docker on Ubuntu + ansible.builtin.apt: + name: docker.io + state: present + install_recommends: false + when: ansible_facts.distribution == "Ubuntu" + + - name: Configure the Docker CE repository on Rocky + ansible.builtin.get_url: + url: https://download.docker.com/linux/centos/docker-ce.repo + dest: /etc/yum.repos.d/docker-ce.repo + mode: "0644" + when: ansible_facts.distribution == "Rocky" + + - name: Install Docker on Rocky + ansible.builtin.dnf: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + when: ansible_facts.distribution == "Rocky" + + - name: Enable Docker + ansible.builtin.service: + name: docker + enabled: true + state: started + + - name: Add the test user to the Docker group + ansible.builtin.user: + name: openshell + groups: + - docker + append: true + + - name: Verify Docker + ansible.builtin.command: + cmd: docker info + changed_when: false diff --git a/nix/vm/configuration/podman.yml b/nix/vm/configuration/podman.yml new file mode 100644 index 0000000000..37ef04ea58 --- /dev/null +++ b/nix/vm/configuration/podman.yml @@ -0,0 +1,34 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure Podman in a disposable test VM. + +- name: Configure Podman + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate Podman support + ansible.builtin.assert: + that: + - ansible_facts.distribution in ["Ubuntu", "CentOS", "Rocky"] + fail_msg: >- + Podman is unsupported on {{ ansible_facts.distribution }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + when: ansible_facts.distribution == "Ubuntu" + + - name: Install Podman + ansible.builtin.package: + name: podman + state: present + + - name: Verify rootless Podman + ansible.builtin.command: + cmd: podman info + become: false + changed_when: false diff --git a/nix/vm/configuration/selinux.yml b/nix/vm/configuration/selinux.yml new file mode 100644 index 0000000000..3b4e32dd7b --- /dev/null +++ b/nix/vm/configuration/selinux.yml @@ -0,0 +1,62 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Ensure SELinux is enforcing in a disposable test VM. + +- name: Configure SELinux + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate SELinux support + ansible.builtin.assert: + that: + - ansible_facts.os_family == "RedHat" + fail_msg: >- + SELinux configuration is supported only on CentOS and Rocky, + not {{ ansible_facts.distribution }}. + + - name: Install SELinux tools + ansible.builtin.package: + name: policycoreutils + state: present + + - name: Read the current SELinux mode + ansible.builtin.command: + cmd: getenforce + register: selinux_current + changed_when: false + + - name: Reject a fully disabled SELinux system + ansible.builtin.assert: + that: + - selinux_current.stdout != "Disabled" + fail_msg: >- + SELinux is disabled and cannot be enabled live. Set SELINUX=enforcing + and reboot the guest before applying this configuration. + + - name: Persist enforcing mode + ansible.builtin.lineinfile: + path: /etc/selinux/config + regexp: ^SELINUX= + line: SELINUX=enforcing + + - name: Enable enforcing mode now + ansible.builtin.command: + cmd: setenforce 1 + when: selinux_current.stdout != "Enforcing" + changed_when: true + + - name: Read the resulting SELinux mode + ansible.builtin.command: + cmd: getenforce + register: selinux_result + changed_when: false + + - name: Verify enforcing mode + ansible.builtin.assert: + that: + - selinux_result.stdout == "Enforcing" + fail_msg: SELinux did not enter enforcing mode. diff --git a/nix/vm/default.nix b/nix/vm/default.nix new file mode 100644 index 0000000000..18fb2f0c2b --- /dev/null +++ b/nix/vm/default.nix @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Composable distro VMs for installing and exercising artifacts. + +{ pkgs }: + +let + isAarch64 = pkgs.stdenv.hostPlatform.isAarch64; + isDarwin = pkgs.stdenv.hostPlatform.isDarwin; + architecture = if isAarch64 then "aarch64" else "x86_64"; + qemu = pkgs.qemu.override { hostCpuOnly = true; }; + qemuBinary = + if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; + + ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + centos = import ./distros/centos.nix { inherit pkgs architecture; }; + rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; + + catalog = pkgs.writeText "openshell-test-vm-catalog.sh" '' + test_vm_list_distros() { + printf '%s\n' ubuntu centos rocky + } + + test_vm_load_distro() { + case "$1" in + ubuntu) + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext ubuntu.image.drvPath} + TEST_VM_OS_ID=${ubuntu.osId} + TEST_VM_OS_VERSION=${ubuntu.osVersion} + TEST_VM_PACKAGE_FAMILY=${ubuntu.packageFamily} + ;; + centos) + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext centos.image.drvPath} + TEST_VM_OS_ID=${centos.osId} + TEST_VM_OS_VERSION=${centos.osVersion} + TEST_VM_PACKAGE_FAMILY=${centos.packageFamily} + ;; + rocky) + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext rocky.image.drvPath} + TEST_VM_OS_ID=${rocky.osId} + TEST_VM_OS_VERSION=${rocky.osVersion} + TEST_VM_PACKAGE_FAMILY=${rocky.packageFamily} + ;; + *) + return 1 + ;; + esac + + TEST_VM_QEMU=${qemuBinary} + TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} + TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} + TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} + TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + TEST_VM_ARCHITECTURE=${architecture} + export TEST_VM_IMAGE_DRV TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY + export TEST_VM_QEMU TEST_VM_FIRMWARE_CODE TEST_VM_FIRMWARE_VARS + export TEST_VM_MACHINE TEST_VM_ACCELERATOR TEST_VM_ARCHITECTURE + } + + test_vm_list_configurations() { + printf '%s\n' docker podman selinux + } + + test_vm_load_configuration() { + case "$1" in + docker) + TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/docker.yml} + ;; + podman) + TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/podman.yml} + ;; + selinux) + TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/selinux.yml} + ;; + *) + return 1 + ;; + esac + export TEST_VM_CONFIGURATION_PLAYBOOK + } + ''; + + runner = pkgs.writeShellApplication { + name = "openshell-test-vm"; + runtimeInputs = [ + qemu + pkgs.python3Packages.ansible-core + pkgs.coreutils + pkgs.gnugrep + pkgs.nix + pkgs.openssh + pkgs.python3 + pkgs.xorriso + ]; + text = '' + export OPENSHELL_TEST_VM_RUNTIME=1 + export OPENSHELL_TEST_VM_CATALOG=${catalog} + exec ${pkgs.bash}/bin/bash ${./run.sh} "$@" + ''; + }; +in +{ + app = { + type = "app"; + program = "${runner}/bin/openshell-test-vm"; + meta.description = "Boot and configure a disposable distro VM"; + }; +} diff --git a/nix/vm/distros/centos.nix b/nix/vm/distros/centos.nix new file mode 100644 index 0000000000..f38ad68533 --- /dev/null +++ b/nix/vm/distros/centos.nix @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +{ + osId = "centos"; + osVersion = "10"; + packageFamily = "rpm"; + image = pkgs.fetchurl { + name = "CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; + url = "https://cloud.centos.org/centos/10-stream/${architecture}/images/CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; + hash = + if architecture == "aarch64" then + "sha256-55IuyMUvsbpvqgug2S7w6JpLCSIpR4HJVkuMch60Rag=" + else + "sha256-k3lpRd9eVJUr4hyoUGfwyfOxGi3W7iFjFU/ZdIxMQdc="; + }; +} diff --git a/nix/vm/distros/rocky.nix b/nix/vm/distros/rocky.nix new file mode 100644 index 0000000000..48fc162e89 --- /dev/null +++ b/nix/vm/distros/rocky.nix @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +{ + osId = "rocky"; + osVersion = "9"; + packageFamily = "rpm"; + image = pkgs.fetchurl { + name = "Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; + url = "https://download.rockylinux.org/pub/rocky/9/images/${architecture}/Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; + hash = + if architecture == "aarch64" then + "sha256-JGkqRE8fC4u5U3XDjItD+AmaEVNHYjaRviwzC0DIof4=" + else + "sha256-ksIGzG95DGFYMkfu/oeJD4goQgZiwXys8kfOx4q07sg="; + }; +} diff --git a/nix/vm/distros/ubuntu.nix b/nix/vm/distros/ubuntu.nix new file mode 100644 index 0000000000..36d4c75c64 --- /dev/null +++ b/nix/vm/distros/ubuntu.nix @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64"; +in +{ + osId = "ubuntu"; + osVersion = "24.04"; + packageFamily = "deb"; + image = pkgs.fetchurl { + name = "ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; + url = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; + hash = + if architecture == "aarch64" then + "sha256-meHUgrlY5r/QGDpMSM5twzTgmj4ppFYPb1/4VZPQnR0=" + else + "sha256-eqbZ9eijpVx0RbE40xpz0Rh4cSEbK32p2i4abL8WmyE="; + }; +} diff --git a/nix/vm/run.sh b/nix/vm/run.sh new file mode 100644 index 0000000000..4a8136a55e --- /dev/null +++ b/nix/vm/run.sh @@ -0,0 +1,430 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Boot and configure a disposable cloud image, then install artifacts. + +set -Eeuo pipefail + +usage() { + cat <<'EOF' +Usage: + nix run .#test-vm -- --distro DISTRO [OPTIONS] [-- COMMAND...] + +Options: + --distro NAME Base distro: ubuntu, centos, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, selinux) + --install PATH Install a .deb or .rpm package; repeatable + --copy SRC:DEST Copy an executable to an absolute guest path; repeatable + --ssh-port PORT Use a specific loopback SSH forwarding port + --keep Keep the disposable disk and logs after shutdown + --list List distros and configurations + -h, --help Show this help + +With no COMMAND, the runner opens an interactive SSH session. +EOF +} + +if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || [ ! -r "${OPENSHELL_TEST_VM_CATALOG:-}" ]; then + echo "run this script through 'nix run .#test-vm -- ...'" >&2 + exit 2 +fi + +# The generated catalog contains only trusted Nix store paths and profile data. +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_CATALOG}" + +distro= +requested_ssh_port= +keep=0 +list=0 +configurations=() +packages=() +copies=() +guest_command=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --distro) + distro=${2:-} + shift 2 + ;; + --with) + configurations+=("${2:-}") + shift 2 + ;; + --install) + packages+=("${2:-}") + shift 2 + ;; + --copy) + copies+=("${2:-}") + shift 2 + ;; + --ssh-port) + requested_ssh_port=${2:-} + shift 2 + ;; + --keep) + keep=1 + shift + ;; + --list) + list=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + --) + shift + guest_command=("$@") + break + ;; + *) + echo "unknown test VM argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ "${list}" -eq 1 ]; then + echo "Distros:" + test_vm_list_distros | sed 's/^/ /' + echo "Configurations:" + test_vm_list_configurations | sed 's/^/ /' + exit 0 +fi + +if [ -z "${distro}" ]; then + echo "--distro is required" >&2 + usage >&2 + exit 2 +fi +if ! test_vm_load_distro "${distro}"; then + echo "unknown distro: ${distro}" >&2 + exit 2 +fi + +for item in "${configurations[@]}"; do + if [ -z "${item}" ] || ! test_vm_load_configuration "${item}"; then + echo "unknown configuration: ${item:-}" >&2 + exit 2 + fi +done + +if [ -n "${requested_ssh_port}" ] && { + [[ ! ${requested_ssh_port} =~ ^[0-9]+$ ]] || + [ "${requested_ssh_port}" -lt 1024 ] || + [ "${requested_ssh_port}" -gt 65535 ] + }; then + echo "--ssh-port must be an integer between 1024 and 65535" >&2 + exit 2 +fi + +for package in "${packages[@]}"; do + if [ ! -f "${package}" ]; then + echo "package does not exist: ${package}" >&2 + exit 2 + fi + case "${TEST_VM_PACKAGE_FAMILY}:${package}" in + deb:*.deb | rpm:*.rpm) ;; + *) + echo "${package} does not match the ${TEST_VM_PACKAGE_FAMILY} package family" >&2 + exit 2 + ;; + esac +done + +for copy_spec in "${copies[@]}"; do + source_path=${copy_spec%%:*} + destination=${copy_spec#*:} + if [ "${source_path}" = "${copy_spec}" ] || [ ! -f "${source_path}" ]; then + echo "invalid --copy source: ${copy_spec}" >&2 + exit 2 + fi + case "${destination}" in + /*) + if [[ ${destination} == *"/../"* ]] || [[ ${destination} == */.. ]]; then + echo "--copy destination must not contain '..': ${destination}" >&2 + exit 2 + fi + ;; + *) + echo "--copy destination must be absolute: ${destination}" >&2 + exit 2 + ;; + esac +done + +if [ "${TEST_VM_ACCELERATOR}" = kvm ]; then + if [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + echo "KVM requires a readable and writable /dev/kvm" >&2 + exit 1 + fi +fi + +echo "==> Realizing the pinned ${distro} cloud image" +TEST_VM_IMAGE=$(nix build --no-link --print-out-paths "${TEST_VM_IMAGE_DRV}^out") + +umask 077 +run_parent=${TMPDIR:-/tmp}/openshell-test-vm +mkdir -p "${run_parent}" +run_dir=$(mktemp -d "${run_parent%/}/run.XXXXXX") +overlay=${run_dir}/disk.qcow2 +seed=${run_dir}/seed.iso +vars=${run_dir}/firmware-vars.fd +private_key=${run_dir}/id_ed25519 +serial_log=${run_dir}/serial.log +qemu_log=${run_dir}/qemu.log +qemu_pid= +ssh_port= +ssh_args=() +scp_args=() +ansible_config=${run_dir}/ansible.cfg +ansible_inventory=${run_dir}/inventory.ini + +show_logs() { + if [ -s "${qemu_log}" ]; then + echo "=== QEMU log ===" >&2 + tail -n 100 "${qemu_log}" >&2 || true + fi + if [ -s "${serial_log}" ]; then + echo "=== serial log ===" >&2 + tail -n 200 "${serial_log}" >&2 || true + fi +} + +cleanup() { + status=$? + trap - EXIT INT TERM + if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + fi + if [ "${status}" -ne 0 ]; then + show_logs + fi + if [ "${keep}" -eq 1 ]; then + echo "Kept test VM state at ${run_dir}" >&2 + else + rm -rf "${run_dir}" + fi + exit "${status}" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +pick_free_port() { + python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()' +} + +ssh-keygen -q -t ed25519 -N "" -f "${private_key}" +public_key=$(<"${private_key}.pub") + +cat >"${run_dir}/meta-data" <"${run_dir}/user-data" <"${qemu_log}" + echo "==> Booting ${distro} (${TEST_VM_ARCHITECTURE}) with QEMU/${TEST_VM_ACCELERATOR}" + "${TEST_VM_QEMU}" \ + -name "openshell-test-${distro}" \ + -machine "${TEST_VM_MACHINE},accel=${TEST_VM_ACCELERATOR}" \ + -cpu host \ + -smp 4 \ + -m 4096 \ + -drive "if=pflash,format=raw,readonly=on,file=${TEST_VM_FIRMWARE_CODE}" \ + -drive "if=pflash,format=raw,file=${vars}" \ + -drive "if=none,format=qcow2,file=${overlay},id=osdisk" \ + -device virtio-blk-pci,drive=osdisk \ + -drive "if=none,format=raw,readonly=on,file=${seed},id=seed" \ + -device virtio-blk-pci,drive=seed \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -monitor none \ + -serial "file:${serial_log}" \ + -no-reboot \ + >/dev/null 2>"${qemu_log}" & + qemu_pid=$! + + sleep 1 + if kill -0 "${qemu_pid}" 2>/dev/null; then + break + fi + wait "${qemu_pid}" || true + qemu_pid= + if ! grep -q "Could not set up host forwarding rule" "${qemu_log}" || + [ -n "${requested_ssh_port}" ]; then + echo "QEMU exited during startup" >&2 + exit 1 + fi + echo "SSH port was claimed concurrently; retrying (${attempt}/5)" >&2 +done + +if [ -z "${qemu_pid}" ]; then + echo "QEMU could not allocate an SSH forwarding port" >&2 + exit 1 +fi + +ssh_args=( + -F /dev/null + -i "${private_key}" + -p "${ssh_port}" + -o BatchMode=yes + -o ConnectTimeout=5 + -o LogLevel=ERROR + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) +scp_args=( + -F /dev/null + -i "${private_key}" + -P "${ssh_port}" + -o BatchMode=yes + -o ConnectTimeout=5 + -o LogLevel=ERROR + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null +) + +echo "==> Waiting up to 180 seconds for SSH on 127.0.0.1:${ssh_port}" +ssh_ready=0 +for _ in $(seq 1 90); do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + wait "${qemu_pid}" || true + qemu_pid= + echo "QEMU exited before SSH became ready" >&2 + exit 1 + fi + if ssh "${ssh_args[@]}" openshell@127.0.0.1 true 2>/dev/null; then + ssh_ready=1 + break + fi + sleep 2 +done +if [ "${ssh_ready}" -ne 1 ]; then + echo "SSH did not become ready within 180 seconds" >&2 + exit 1 +fi + +echo "==> Validating ${distro}" +# Profile values come from the trusted Nix-generated catalog. +# shellcheck disable=SC2029 +ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "set -eu; . /etc/os-release; test \"\${ID}\" = '${TEST_VM_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_VM_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_VM_ARCHITECTURE}'" +ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo cloud-init status --wait >/dev/null' + +cat >"${ansible_config}" <"${ansible_inventory}" < Applying configuration: ${item}" + ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ + ansible-playbook "${TEST_VM_CONFIGURATION_PLAYBOOK}" +done + +for package in "${packages[@]}"; do + package_name=$(basename "${package}") + if [[ ! ${package_name} =~ ^[A-Za-z0-9._+~-]+$ ]]; then + echo "package filename contains unsupported characters: ${package_name}" >&2 + exit 2 + fi + remote_package="/home/openshell/${package_name}" + echo "==> Installing package: ${package_name}" + scp "${scp_args[@]}" "${package}" "openshell@127.0.0.1:${remote_package}" + case "${TEST_VM_PACKAGE_FAMILY}" in + deb) + # The remote path uses a restricted package filename. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo apt-get update -qq && sudo env DEBIAN_FRONTEND=noninteractive apt-get install -qq -y '${remote_package}' && rm -f '${remote_package}'" + ;; + rpm) + # The remote path uses a restricted package filename. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo dnf install -q -y '${remote_package}' && rm -f '${remote_package}'" + ;; + esac +done + +for copy_spec in "${copies[@]}"; do + source_path=${copy_spec%%:*} + destination=${copy_spec#*:} + copy_name=$(basename "${source_path}") + if [[ ! ${copy_name} =~ ^[A-Za-z0-9._+~-]+$ ]]; then + echo "copy filename contains unsupported characters: ${copy_name}" >&2 + exit 2 + fi + remote_copy="/home/openshell/${copy_name}" + echo "==> Copying ${copy_name} to ${destination}" + scp "${scp_args[@]}" "${source_path}" "openshell@127.0.0.1:${remote_copy}" + # The destination was restricted to an absolute, traversal-free path. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "sudo install -D -m 0755 '${remote_copy}' '${destination}'; rm -f '${remote_copy}'" +done + +echo "==> Test VM ready: ${distro} (SSH port ${ssh_port})" +if [ "${#guest_command[@]}" -eq 0 ]; then + ssh -t "${ssh_args[@]}" openshell@127.0.0.1 +else + printf -v quoted_command '%q ' "${guest_command[@]}" + # quoted_command is shell-escaped locally before it reaches the guest. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 "bash -lc $(printf '%q' "${quoted_command}")" +fi + +echo "==> Shutting down ${distro}" +ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo systemctl poweroff' >/dev/null 2>&1 || true +for _ in $(seq 1 30); do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + wait "${qemu_pid}" || true + qemu_pid= + break + fi + sleep 1 +done diff --git a/tasks/scripts/package-deb.sh b/tasks/scripts/package-deb.sh index 9d7e3d3281..3e20f6256e 100755 --- a/tasks/scripts/package-deb.sh +++ b/tasks/scripts/package-deb.sh @@ -167,22 +167,4 @@ dpkg-deb --build --root-owner-group "$pkgroot" "$package_file" dpkg-deb --info "$package_file" dpkg-deb --contents "$package_file" -# --------------------------------------------------------------------------- -# Smoke tests -# --------------------------------------------------------------------------- - -extract_dir="${tmpdir}/extract" -mkdir -p "$extract_dir" -dpkg-deb -x "$package_file" "$extract_dir" -"$extract_dir/usr/bin/openshell" --version -"$extract_dir/usr/bin/openshell-gateway" --version -"$extract_dir/usr/libexec/openshell/openshell-driver-vm" --version - -if command -v systemd-analyze >/dev/null 2>&1; then - # verify --user catches user-scope-specific issues like StateDirectory= - # resolution and the %h/%S specifiers used in this unit. - systemd-analyze --user verify "$extract_dir/usr/lib/systemd/user/openshell-gateway.service" \ - || echo "warning: systemd-analyze verify failed in the build environment" >&2 -fi - echo "Wrote ${package_file}" From 2a9d51d98baf2602c34deaf01ca67233b4bb69d6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 11:05:19 -0700 Subject: [PATCH 02/29] docs(vm): describe test VM directory structure Signed-off-by: Drew Newberry --- nix/vm/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/nix/vm/README.md b/nix/vm/README.md index 862b2ba0e2..4288bc229a 100644 --- a/nix/vm/README.md +++ b/nix/vm/README.md @@ -16,6 +16,31 @@ This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linu The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. +## Directory structure + +```text +nix/vm/ +├── README.md +├── default.nix +├── run.sh +├── distros/ +│ ├── ubuntu.nix +│ ├── centos.nix +│ └── rocky.nix +└── configuration/ + ├── docker.yml + ├── podman.yml + └── selinux.yml +``` + +- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, imports the distro catalog, and maps `--with` names to playbooks. +- `run.sh` owns the disposable VM lifecycle: argument validation, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup. +- `distros/*.nix` define the immutable base-image catalog. Each record pins the image URL and hash and declares the expected OS ID, version, and package family. Distro definitions do not provision extra software. +- `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments. +- `README.md` documents the supported combinations and developer interface. + +The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-vm` app. Debian artifact creation remains outside the VM harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist. + ## Supported configurations | Distro | Docker | Podman | SELinux | Package format | From dc2a466ef4fb1c5a854e006d84398c86eed98a92 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 11:17:34 -0700 Subject: [PATCH 03/29] refactor(vm): replace shell catalog functions Signed-off-by: Drew Newberry --- nix/vm/README.md | 2 +- nix/vm/default.nix | 98 +++++++++++++++++----------------------------- nix/vm/run.sh | 28 +++++++------ 3 files changed, 53 insertions(+), 75 deletions(-) diff --git a/nix/vm/README.md b/nix/vm/README.md index 4288bc229a..918296f5f5 100644 --- a/nix/vm/README.md +++ b/nix/vm/README.md @@ -33,7 +33,7 @@ nix/vm/ └── selinux.yml ``` -- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, imports the distro catalog, and maps `--with` names to playbooks. +- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, and exposes distro profiles and configuration playbooks as Nix-store catalogs. - `run.sh` owns the disposable VM lifecycle: argument validation, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup. - `distros/*.nix` define the immutable base-image catalog. Each record pins the image URL and hash and declares the expected OS ID, version, and package family. Distro definitions do not provision extra software. - `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments. diff --git a/nix/vm/default.nix b/nix/vm/default.nix index 18fb2f0c2b..44acd01e66 100644 --- a/nix/vm/default.nix +++ b/nix/vm/default.nix @@ -13,73 +13,38 @@ let qemuBinary = if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; - ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; - centos = import ./distros/centos.nix { inherit pkgs architecture; }; - rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; - - catalog = pkgs.writeText "openshell-test-vm-catalog.sh" '' - test_vm_list_distros() { - printf '%s\n' ubuntu centos rocky - } + distros = { + ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + centos = import ./distros/centos.nix { inherit pkgs architecture; }; + rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; + }; - test_vm_load_distro() { - case "$1" in - ubuntu) - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext ubuntu.image.drvPath} - TEST_VM_OS_ID=${ubuntu.osId} - TEST_VM_OS_VERSION=${ubuntu.osVersion} - TEST_VM_PACKAGE_FAMILY=${ubuntu.packageFamily} - ;; - centos) - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext centos.image.drvPath} - TEST_VM_OS_ID=${centos.osId} - TEST_VM_OS_VERSION=${centos.osVersion} - TEST_VM_PACKAGE_FAMILY=${centos.packageFamily} - ;; - rocky) - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext rocky.image.drvPath} - TEST_VM_OS_ID=${rocky.osId} - TEST_VM_OS_VERSION=${rocky.osVersion} - TEST_VM_PACKAGE_FAMILY=${rocky.packageFamily} - ;; - *) - return 1 - ;; - esac + configurations = { + docker = ./configuration/docker.yml; + podman = ./configuration/podman.yml; + selinux = ./configuration/selinux.yml; + }; - TEST_VM_QEMU=${qemuBinary} - TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} - TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} - TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} - TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} - TEST_VM_ARCHITECTURE=${architecture} + mkDistroProfile = + name: distro: + pkgs.writeText "openshell-test-vm-${name}" '' + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath} + TEST_VM_OS_ID=${pkgs.lib.escapeShellArg distro.osId} + TEST_VM_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion} + TEST_VM_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily} export TEST_VM_IMAGE_DRV TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY - export TEST_VM_QEMU TEST_VM_FIRMWARE_CODE TEST_VM_FIRMWARE_VARS - export TEST_VM_MACHINE TEST_VM_ACCELERATOR TEST_VM_ARCHITECTURE - } + ''; - test_vm_list_configurations() { - printf '%s\n' docker podman selinux - } + distroCatalog = pkgs.linkFarm "openshell-test-vm-distros" ( + pkgs.lib.mapAttrsToList (name: distro: { + inherit name; + path = mkDistroProfile name distro; + }) distros + ); - test_vm_load_configuration() { - case "$1" in - docker) - TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/docker.yml} - ;; - podman) - TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/podman.yml} - ;; - selinux) - TEST_VM_CONFIGURATION_PLAYBOOK=${./configuration/selinux.yml} - ;; - *) - return 1 - ;; - esac - export TEST_VM_CONFIGURATION_PLAYBOOK - } - ''; + configurationCatalog = pkgs.linkFarm "openshell-test-vm-configurations" ( + pkgs.lib.mapAttrsToList (name: path: { inherit name path; }) configurations + ); runner = pkgs.writeShellApplication { name = "openshell-test-vm"; @@ -95,7 +60,14 @@ let ]; text = '' export OPENSHELL_TEST_VM_RUNTIME=1 - export OPENSHELL_TEST_VM_CATALOG=${catalog} + export OPENSHELL_TEST_VM_DISTROS=${distroCatalog} + export OPENSHELL_TEST_VM_CONFIGURATIONS=${configurationCatalog} + export TEST_VM_QEMU=${qemuBinary} + export TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} + export TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} + export TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} + export TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + export TEST_VM_ARCHITECTURE=${architecture} exec ${pkgs.bash}/bin/bash ${./run.sh} "$@" ''; }; diff --git a/nix/vm/run.sh b/nix/vm/run.sh index 4a8136a55e..ccf052ba27 100644 --- a/nix/vm/run.sh +++ b/nix/vm/run.sh @@ -25,15 +25,13 @@ With no COMMAND, the runner opens an interactive SSH session. EOF } -if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || [ ! -r "${OPENSHELL_TEST_VM_CATALOG:-}" ]; then +if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || + [ ! -d "${OPENSHELL_TEST_VM_DISTROS:-}" ] || + [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ]; then echo "run this script through 'nix run .#test-vm -- ...'" >&2 exit 2 fi -# The generated catalog contains only trusted Nix store paths and profile data. -# shellcheck disable=SC1090 -. "${OPENSHELL_TEST_VM_CATALOG}" - distro= requested_ssh_port= keep=0 @@ -92,9 +90,13 @@ done if [ "${list}" -eq 1 ]; then echo "Distros:" - test_vm_list_distros | sed 's/^/ /' + for entry in "${OPENSHELL_TEST_VM_DISTROS}"/*; do + printf ' %s\n' "${entry##*/}" + done echo "Configurations:" - test_vm_list_configurations | sed 's/^/ /' + for entry in "${OPENSHELL_TEST_VM_CONFIGURATIONS}"/*; do + printf ' %s\n' "${entry##*/}" + done exit 0 fi @@ -103,13 +105,18 @@ if [ -z "${distro}" ]; then usage >&2 exit 2 fi -if ! test_vm_load_distro "${distro}"; then +if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + [ ! -r "${OPENSHELL_TEST_VM_DISTROS}/${distro}" ]; then echo "unknown distro: ${distro}" >&2 exit 2 fi +# Distro profiles contain only trusted values generated into the Nix store. +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_DISTROS}/${distro}" for item in "${configurations[@]}"; do - if [ -z "${item}" ] || ! test_vm_load_configuration "${item}"; then + if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + [ ! -r "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" ]; then echo "unknown configuration: ${item:-}" >&2 exit 2 fi @@ -360,10 +367,9 @@ guest ansible_host=127.0.0.1 ansible_port=${ssh_port} ansible_user=openshell ans EOF for item in "${configurations[@]}"; do - test_vm_load_configuration "${item}" echo "==> Applying configuration: ${item}" ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ - ansible-playbook "${TEST_VM_CONFIGURATION_PLAYBOOK}" + ansible-playbook "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" done for package in "${packages[@]}"; do From 59aa8d984583ca87dee93bb1a6f83dc61a28e0f8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 12:50:52 -0700 Subject: [PATCH 04/29] test(vm): add Fedora release guest support Signed-off-by: Drew Newberry --- nix/vm/README.md | 19 +++++++++++-------- nix/vm/configuration/docker.yml | 13 +++++++++++++ nix/vm/configuration/podman.yml | 2 +- nix/vm/configuration/selinux.yml | 2 +- nix/vm/default.nix | 1 + nix/vm/distros/fedora.nix | 19 +++++++++++++++++++ nix/vm/run.sh | 29 ++++++++++++++++++----------- 7 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 nix/vm/distros/fedora.nix diff --git a/nix/vm/README.md b/nix/vm/README.md index 918296f5f5..444c9c5b4f 100644 --- a/nix/vm/README.md +++ b/nix/vm/README.md @@ -5,14 +5,14 @@ SPDX-License-Identifier: Apache-2.0 # Test VMs -This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS and KVM on native-architecture Linux hosts. +This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. ## Requirements - Nix with flakes enabled. -- Apple Silicon macOS with HVF, or Linux with readable and writable `/dev/kvm`. +- Apple Silicon macOS with HVF, or a native-architecture Linux host. Linux uses KVM when `/dev/kvm` is available and falls back to QEMU TCG otherwise. - Enough local capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay. -- Native-architecture artifacts. The runner does not use software emulation. +- Native-architecture artifacts. TCG emulates the guest CPU on Linux but does not enable cross-architecture guests. The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. @@ -26,6 +26,7 @@ nix/vm/ ├── distros/ │ ├── ubuntu.nix │ ├── centos.nix +│ ├── fedora.nix │ └── rocky.nix └── configuration/ ├── docker.yml @@ -47,6 +48,7 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-vm` | --- | --- | --- | --- | --- | | Ubuntu 24.04 | Yes | Yes | No | `.deb` | | CentOS Stream 10 | No | Yes | Yes | `.rpm` | +| Fedora 44 | No | Yes | Yes | `.rpm` | | Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | List the available distros and configurations: @@ -74,6 +76,7 @@ Other combinations use the same interface: ```shell nix run .#test-vm -- --distro rocky --with docker nix run .#test-vm -- --distro centos --with podman +nix run .#test-vm -- --distro fedora --with podman ``` Configurations are repeatable: @@ -85,7 +88,7 @@ nix run .#test-vm -- \ --with podman ``` -Ensure SELinux is enforcing on CentOS or Rocky: +Ensure SELinux is enforcing on CentOS, Fedora, or Rocky: ```shell nix run .#test-vm -- \ @@ -128,7 +131,7 @@ nix run .#test-vm -- \ For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the host and guest architecture. -`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS and Rocky Linux. This prototype can install an existing RPM but does not build one. +`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS, Fedora, and Rocky Linux. This prototype can install an existing RPM but does not build one. ## Copy binaries directly @@ -146,7 +149,7 @@ The destination must be an absolute guest path. Copied files are installed with ## Runner options ```text ---distro NAME Base distro: ubuntu, centos, or rocky +--distro NAME Base distro: ubuntu, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy an executable into the guest; repeatable @@ -162,7 +165,7 @@ Arguments after `--` are executed inside the guest. Without a command, the runne Nix downloads and caches the selected, hash-pinned cloud image. Each invocation then: 1. Creates a temporary QCOW2 overlay. -2. Boots QEMU with HVF or KVM acceleration. +2. Boots QEMU with HVF, KVM, or the Linux TCG fallback. 3. Creates an ephemeral `openshell` user and SSH key through cloud-init. 4. Applies the selected Ansible configurations. 5. Installs or copies the supplied artifacts. @@ -174,7 +177,7 @@ Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log f ## Current limitations - Host and guest architectures must match. -- Software emulation without HVF or KVM is not supported. +- TCG is slower than hardware virtualization and uses a longer SSH readiness timeout. - Configurations are applied fresh on every run; prepared VM caching is not implemented. - Only loopback SSH is forwarded to the host; guest gateway ports are not exposed. - The runner does not build OpenShell, configure a gateway, or select an E2E test suite. diff --git a/nix/vm/configuration/docker.yml b/nix/vm/configuration/docker.yml index 1c158d3653..1b25e365f4 100644 --- a/nix/vm/configuration/docker.yml +++ b/nix/vm/configuration/docker.yml @@ -60,6 +60,19 @@ groups: - docker append: true + register: docker_group_membership + + - name: Read the test user ID + ansible.builtin.command: + cmd: id -u openshell + register: test_user_id + changed_when: false + + - name: Refresh the test user's systemd group membership + ansible.builtin.systemd_service: + name: "user@{{ test_user_id.stdout }}.service" + state: restarted + when: docker_group_membership.changed - name: Verify Docker ansible.builtin.command: diff --git a/nix/vm/configuration/podman.yml b/nix/vm/configuration/podman.yml index 37ef04ea58..c7851ce141 100644 --- a/nix/vm/configuration/podman.yml +++ b/nix/vm/configuration/podman.yml @@ -13,7 +13,7 @@ - name: Validate Podman support ansible.builtin.assert: that: - - ansible_facts.distribution in ["Ubuntu", "CentOS", "Rocky"] + - ansible_facts.distribution in ["Ubuntu", "CentOS", "Fedora", "Rocky"] fail_msg: >- Podman is unsupported on {{ ansible_facts.distribution }}. diff --git a/nix/vm/configuration/selinux.yml b/nix/vm/configuration/selinux.yml index 3b4e32dd7b..6f34e16c67 100644 --- a/nix/vm/configuration/selinux.yml +++ b/nix/vm/configuration/selinux.yml @@ -15,7 +15,7 @@ that: - ansible_facts.os_family == "RedHat" fail_msg: >- - SELinux configuration is supported only on CentOS and Rocky, + SELinux configuration is supported only on CentOS, Fedora, and Rocky, not {{ ansible_facts.distribution }}. - name: Install SELinux tools diff --git a/nix/vm/default.nix b/nix/vm/default.nix index 44acd01e66..ec82b4b8be 100644 --- a/nix/vm/default.nix +++ b/nix/vm/default.nix @@ -16,6 +16,7 @@ let distros = { ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; centos = import ./distros/centos.nix { inherit pkgs architecture; }; + fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; }; diff --git a/nix/vm/distros/fedora.nix b/nix/vm/distros/fedora.nix new file mode 100644 index 0000000000..d8701ed166 --- /dev/null +++ b/nix/vm/distros/fedora.nix @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +{ + osId = "fedora"; + osVersion = "44"; + packageFamily = "rpm"; + image = pkgs.fetchurl { + name = "Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; + url = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/${architecture}/images/Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; + hash = + if architecture == "aarch64" then + "sha256-VcYKO4DTYWoIcFr9BFnnX+nwPFSrp6RuQAKkGnL6DVs=" + else + "sha256-KGgP5bNxpaguv0OjGSbghqFo5ZlJ0DlpxQk+cHH5C38="; + }; +} diff --git a/nix/vm/run.sh b/nix/vm/run.sh index ccf052ba27..1a54ea4a33 100644 --- a/nix/vm/run.sh +++ b/nix/vm/run.sh @@ -12,7 +12,7 @@ Usage: nix run .#test-vm -- --distro DISTRO [OPTIONS] [-- COMMAND...] Options: - --distro NAME Base distro: ubuntu, centos, or rocky + --distro NAME Base distro: ubuntu, centos, fedora, or rocky --with NAME Apply a configuration; repeatable (docker, podman, selinux) --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy an executable to an absolute guest path; repeatable @@ -166,11 +166,14 @@ for copy_spec in "${copies[@]}"; do esac done -if [ "${TEST_VM_ACCELERATOR}" = kvm ]; then - if [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then - echo "KVM requires a readable and writable /dev/kvm" >&2 - exit 1 - fi +test_vm_cpu=host +ssh_wait_seconds=180 +if [ "${TEST_VM_ACCELERATOR}" = kvm ] && + { [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; }; then + echo "==> /dev/kvm is unavailable; falling back to QEMU/TCG" + TEST_VM_ACCELERATOR=tcg + test_vm_cpu=max + ssh_wait_seconds=600 fi echo "==> Realizing the pinned ${distro} cloud image" @@ -266,7 +269,7 @@ for attempt in $(seq 1 5); do "${TEST_VM_QEMU}" \ -name "openshell-test-${distro}" \ -machine "${TEST_VM_MACHINE},accel=${TEST_VM_ACCELERATOR}" \ - -cpu host \ + -cpu "${test_vm_cpu}" \ -smp 4 \ -m 4096 \ -drive "if=pflash,format=raw,readonly=on,file=${TEST_VM_FIRMWARE_CODE}" \ @@ -324,9 +327,9 @@ scp_args=( -o UserKnownHostsFile=/dev/null ) -echo "==> Waiting up to 180 seconds for SSH on 127.0.0.1:${ssh_port}" +echo "==> Waiting up to ${ssh_wait_seconds} seconds for SSH on 127.0.0.1:${ssh_port}" ssh_ready=0 -for _ in $(seq 1 90); do +for _ in $(seq 1 "$((ssh_wait_seconds / 2))"); do if ! kill -0 "${qemu_pid}" 2>/dev/null; then wait "${qemu_pid}" || true qemu_pid= @@ -340,7 +343,7 @@ for _ in $(seq 1 90); do sleep 2 done if [ "${ssh_ready}" -ne 1 ]; then - echo "SSH did not become ready within 180 seconds" >&2 + echo "SSH did not become ready within ${ssh_wait_seconds} seconds" >&2 exit 1 fi @@ -349,7 +352,11 @@ echo "==> Validating ${distro}" # shellcheck disable=SC2029 ssh "${ssh_args[@]}" openshell@127.0.0.1 \ "set -eu; . /etc/os-release; test \"\${ID}\" = '${TEST_VM_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_VM_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_VM_ARCHITECTURE}'" -ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo cloud-init status --wait >/dev/null' +# cloud-init returns 2 when it completes with recoverable errors. Fedora can +# report that status for an initial transient-hostname warning even though the +# requested user and SSH configuration were applied successfully. +ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + 'sudo cloud-init status --wait >/dev/null; status=$?; [ "$status" -eq 0 ] || [ "$status" -eq 2 ]' cat >"${ansible_config}" < Date: Fri, 24 Jul 2026 16:00:46 -0700 Subject: [PATCH 05/29] test(vm): enable rootless Podman socket Signed-off-by: Drew Newberry --- nix/vm/configuration/podman.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nix/vm/configuration/podman.yml b/nix/vm/configuration/podman.yml index c7851ce141..3012c59f3e 100644 --- a/nix/vm/configuration/podman.yml +++ b/nix/vm/configuration/podman.yml @@ -27,6 +27,14 @@ name: podman state: present + - name: Enable the rootless Podman API socket + ansible.builtin.systemd_service: + name: podman.socket + scope: user + enabled: true + state: started + become: false + - name: Verify rootless Podman ansible.builtin.command: cmd: podman info From 52cb5e1b90ababfdf89a4ee95a37bb111509263f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 11:54:23 -0700 Subject: [PATCH 06/29] feat(test-guest): add OCI-backed image caching Signed-off-by: Drew Newberry --- architecture/build.md | 14 + flake.nix | 5 +- nix/{vm => test-guest}/README.md | 102 +++- nix/test-guest/cache-lib.sh | 155 ++++++ nix/test-guest/cache-seal.sh | 57 +++ nix/test-guest/cache.sh | 444 ++++++++++++++++++ .../configuration/docker.yml | 2 +- .../configuration/podman.yml | 2 +- .../configuration/selinux.yml | 2 +- nix/test-guest/default.nix | 113 +++++ nix/test-guest/distros/centos.nix | 24 + nix/test-guest/distros/fedora.nix | 24 + nix/test-guest/distros/rocky.nix | 24 + nix/{vm => test-guest}/distros/ubuntu.nix | 15 +- nix/{vm => test-guest}/run.sh | 68 ++- nix/vm/default.nix | 82 ---- nix/vm/distros/centos.nix | 19 - nix/vm/distros/fedora.nix | 19 - nix/vm/distros/rocky.nix | 19 - 19 files changed, 1000 insertions(+), 190 deletions(-) rename nix/{vm => test-guest}/README.md (53%) create mode 100644 nix/test-guest/cache-lib.sh create mode 100644 nix/test-guest/cache-seal.sh create mode 100644 nix/test-guest/cache.sh rename nix/{vm => test-guest}/configuration/docker.yml (97%) rename nix/{vm => test-guest}/configuration/podman.yml (94%) rename nix/{vm => test-guest}/configuration/selinux.yml (96%) create mode 100644 nix/test-guest/default.nix create mode 100644 nix/test-guest/distros/centos.nix create mode 100644 nix/test-guest/distros/fedora.nix create mode 100644 nix/test-guest/distros/rocky.nix rename nix/{vm => test-guest}/distros/ubuntu.nix (52%) rename nix/{vm => test-guest}/run.sh (84%) delete mode 100644 nix/vm/default.nix delete mode 100644 nix/vm/distros/centos.nix delete mode 100644 nix/vm/distros/fedora.nix delete mode 100644 nix/vm/distros/rocky.nix diff --git a/architecture/build.md b/architecture/build.md index eb8672f15a..6ca5c3d59e 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -135,6 +135,20 @@ contexts use `KIND_EXPERIMENTAL_PROVIDER=docker|podman` when set, and ambiguous or unknown contexts require an explicit `CONTAINER_ENGINE`. Other image builds do not infer from kube context. +## Disposable Test Guests + +The Nix test guest harness under `nix/test-guest` boots native-architecture cloud images +through QEMU for package, release, and E2E validation. A prepared cache entry is +captured after the exact ordered Ansible configuration list and before +test-specific packages, copied binaries, forwarded ports, or commands. + +Prepared disks are flattened, sanitized QCOW2 images. The local cache keeps them +read-only and each test receives a fresh writable overlay and cloud-init +identity. The optional shared cache stores the compressed standalone disk and +its compatibility metadata as a custom OCI artifact. Normal test runs consume +local entries only; the separate cache app owns OCI pulls, trusted builds, and +explicit publication. + ## Python Wheel Packaging The generated protobuf/gRPC stubs under `python/openshell/_proto/` are gitignored diff --git a/flake.nix b/flake.nix index aa4ce8829a..08943cbb60 100644 --- a/flake.nix +++ b/flake.nix @@ -37,10 +37,11 @@ projectRootFile = "flake.nix"; programs.nixfmt.enable = true; }; - testVm = import ./nix/vm { inherit pkgs; }; + testGuest = import ./nix/test-guest { inherit pkgs; }; in { - apps.test-vm = testVm.app; + apps.test-guest = testGuest.app; + apps.test-guest-cache = testGuest.cacheApp; devShells.default = pkgs.mkShell { packages = with pkgs; [ diff --git a/nix/vm/README.md b/nix/test-guest/README.md similarity index 53% rename from nix/vm/README.md rename to nix/test-guest/README.md index 444c9c5b4f..e3c7f6ef6d 100644 --- a/nix/vm/README.md +++ b/nix/test-guest/README.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. SPDX-License-Identifier: Apache-2.0 --> -# Test VMs +# Test Guests This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. @@ -19,10 +19,13 @@ The first run downloads the selected cloud image and VM runtime. Nix reuses thos ## Directory structure ```text -nix/vm/ +nix/test-guest/ ├── README.md ├── default.nix ├── run.sh +├── cache.sh +├── cache-lib.sh +├── cache-seal.sh ├── distros/ │ ├── ubuntu.nix │ ├── centos.nix @@ -34,13 +37,16 @@ nix/vm/ └── selinux.yml ``` -- `default.nix` assembles the VM flake app. It selects host architecture and acceleration, supplies QEMU and Ansible through the Nix runtime closure, and exposes distro profiles and configuration playbooks as Nix-store catalogs. -- `run.sh` owns the disposable VM lifecycle: argument validation, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup. -- `distros/*.nix` define the immutable base-image catalog. Each record pins the image URL and hash and declares the expected OS ID, version, and package family. Distro definitions do not provision extra software. +- `default.nix` assembles the guest and cache flake apps. It selects host architecture and acceleration, supplies the runtime tools, and exposes distro profiles and configuration playbooks as Nix-store catalogs. +- `run.sh` owns the disposable guest lifecycle: cache lookup, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup. +- `cache.sh` ensures an exact prepared disk exists locally. It can pull or explicitly push the disk as an OCI artifact. +- `cache-lib.sh` defines deterministic cache identity and validation helpers shared by the runner and cache command. +- `cache-seal.sh` removes per-instance state and zeroes free space inside a prepared guest before capture. +- `distros/*.nix` define the immutable base-image catalog. Each record pins and exports the image URL and hash and declares the expected OS ID, version, and package family. - `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments. - `README.md` documents the supported combinations and developer interface. -The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-vm` app. Debian artifact creation remains outside the VM harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist. +The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-guest` and `test-guest-cache` apps. Debian artifact creation remains outside the guest harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist. ## Supported configurations @@ -54,7 +60,7 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-vm` List the available distros and configurations: ```shell -nix run .#test-vm -- --list +nix run .#test-guest -- --list ``` ## Open an interactive VM @@ -62,27 +68,27 @@ nix run .#test-vm -- --list Boot a base Ubuntu VM: ```shell -nix run .#test-vm -- --distro ubuntu +nix run .#test-guest -- --distro ubuntu ``` Apply the Docker configuration before opening the SSH session: ```shell -nix run .#test-vm -- --distro ubuntu --with docker +nix run .#test-guest -- --distro ubuntu --with docker ``` Other combinations use the same interface: ```shell -nix run .#test-vm -- --distro rocky --with docker -nix run .#test-vm -- --distro centos --with podman -nix run .#test-vm -- --distro fedora --with podman +nix run .#test-guest -- --distro rocky --with docker +nix run .#test-guest -- --distro centos --with podman +nix run .#test-guest -- --distro fedora --with podman ``` Configurations are repeatable: ```shell -nix run .#test-vm -- \ +nix run .#test-guest -- \ --distro ubuntu \ --with docker \ --with podman @@ -91,7 +97,7 @@ nix run .#test-vm -- \ Ensure SELinux is enforcing on CentOS, Fedora, or Rocky: ```shell -nix run .#test-vm -- \ +nix run .#test-guest -- \ --distro rocky \ --with docker \ --with selinux \ @@ -102,10 +108,57 @@ nix run .#test-vm -- \ ## Ansible configurations -Configurations are Ansible playbooks stored under `nix/vm/configuration/`. Ansible runs on the host using the VM's ephemeral SSH key and loopback port. The guest does not install Ansible. +Configurations are Ansible playbooks stored under `nix/test-guest/configuration/`. Ansible runs on the host using the VM's ephemeral SSH key and loopback port. The guest does not install Ansible. Configurations run in the order provided on the command line. OpenShell packages and copied binaries are installed after all configurations succeed. +## Prepared VM cache + +The `test-guest-cache` app ensures a prepared disk exists for one exact distro, host architecture, and ordered configuration list. It checks the local cache first, optionally pulls a matching OCI artifact, or builds and validates a new local entry on a miss: + +```shell +nix run .#test-guest-cache -- \ + --distro ubuntu \ + --with docker +``` + +Configure an OCI repository to use it as a shared backing cache: + +```shell +nix run .#test-guest-cache -- \ + --distro ubuntu \ + --with docker \ + --repository ghcr.io/nvidia/openshell/test-guest-cache +``` + +The command never publishes implicitly. Add `--push` after authenticating ORAS through its Docker-compatible credential configuration: + +```shell +nix run .#test-guest-cache -- \ + --distro ubuntu \ + --with docker \ + --repository ghcr.io/nvidia/openshell/test-guest-cache \ + --push +``` + +A cache build boots and configures a disposable VM, runs the internal sealing script, flattens the overlay into a standalone QCOW2 disk, and validates a fresh boot before committing the entry. The OCI artifact contains metadata and a `disk.qcow2.zst` layer. + +The key includes the pinned base-image identity, guest architecture, ordered configuration file digests, Ansible version, cache generation, and sealing script digest. Installed packages, copied binaries, forwarded ports, and guest commands are never cached. + +Normal `test-guest` runs automatically use an exact valid local entry and create a fresh writable overlay, cloud-init instance, machine ID, and SSH identity. On a local miss, the runner uses the pinned cloud image and applies configurations normally. Set `OPENSHELL_TEST_VM_CACHE_DISABLE=1` to bypass local lookup. + +The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-guest`. Override it with `--cache-dir` on the cache command or `OPENSHELL_TEST_VM_CACHE_DIR` for either app. + +Cache command options: + +```text +--distro NAME Base distro: ubuntu, centos, fedora, or rocky +--with NAME Apply docker, podman, or selinux; repeatable +--repository REF OCI repository without a tag +--cache-dir PATH Override the local prepared-disk cache directory +--push Publish the ensured entry to the repository +``` + ## Install an OpenShell package Package existing ARM64 Linux binaries with the repository's `package:deb:arm64` mise task: @@ -122,7 +175,7 @@ nix develop --command mise run package:deb:arm64 Install the package in an Ubuntu VM and run a command: ```shell -nix run .#test-vm -- \ +nix run .#test-guest -- \ --distro ubuntu \ --with docker \ --install artifacts/openshell_0.0.0-local_arm64.deb \ @@ -138,7 +191,7 @@ For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. T Use `--copy SOURCE:DEST` to install an executable without creating a package: ```shell -nix run .#test-vm -- \ +nix run .#test-guest -- \ --distro ubuntu \ --copy ./openshell:/usr/local/bin/openshell \ -- openshell --version @@ -162,15 +215,17 @@ Arguments after `--` are executed inside the guest. Without a command, the runne ## Lifecycle -Nix downloads and caches the selected, hash-pinned cloud image. Each invocation then: +Each invocation checks for an exact prepared local cache entry. On a miss, Nix realizes the hash-pinned cloud image and the runner applies the selected configurations. It then: -1. Creates a temporary QCOW2 overlay. +1. Creates a temporary QCOW2 overlay backed by the prepared cache disk or pinned cloud image. 2. Boots QEMU with HVF, KVM, or the Linux TCG fallback. -3. Creates an ephemeral `openshell` user and SSH key through cloud-init. -4. Applies the selected Ansible configurations. +3. Creates a fresh cloud-init instance and ephemeral SSH key. +4. Applies the selected Ansible configurations only when the base is not prepared. 5. Installs or copies the supplied artifacts. 6. Opens SSH or executes the requested guest command. -7. Powers off QEMU and deletes the overlay. +7. Powers off QEMU and deletes the writable overlay. + +Prepared cache disks remain read-only. Test-specific state exists only in the disposable overlay. Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log for debugging. The retained directory is printed when the runner exits. @@ -178,6 +233,7 @@ Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log f - Host and guest architectures must match. - TCG is slower than hardware virtualization and uses a longer SSH readiness timeout. -- Configurations are applied fresh on every run; prepared VM caching is not implemented. +- Prepared cache entries are architecture-specific and match the exact ordered configuration list. +- OCI pulls transfer a complete compressed standalone disk; incremental disk layers are not implemented. - Only loopback SSH is forwarded to the host; guest gateway ports are not exposed. - The runner does not build OpenShell, configure a gateway, or select an E2E test suite. diff --git a/nix/test-guest/cache-lib.sh b/nix/test-guest/cache-lib.sh new file mode 100644 index 0000000000..5b37f91893 --- /dev/null +++ b/nix/test-guest/cache-lib.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared cache identity and validation helpers for the test guest runner. + +TEST_VM_CACHE_SCHEMA_VERSION=1 +TEST_VM_CACHE_DISK_LAYOUT=standalone-qcow2-zstd-v1 +TEST_VM_CACHE_ARTIFACT_TYPE=application/vnd.nvidia.openshell.test-guest.cache.v1 +TEST_VM_CACHE_METADATA_TYPE=application/vnd.nvidia.openshell.test-guest.cache.metadata.v1+json +TEST_VM_CACHE_DISK_TYPE=application/vnd.nvidia.openshell.test-guest.cache.disk.qcow2.v1+zstd + +test_vm_cache_root() { + if [ -n "${OPENSHELL_TEST_VM_CACHE_DIR:-}" ]; then + printf '%s\n' "${OPENSHELL_TEST_VM_CACHE_DIR}" + else + printf '%s\n' "${XDG_CACHE_HOME:-${HOME}/.cache}/openshell/test-guest" + fi +} + +test_vm_cache_oci_architecture() { + case "${TEST_VM_ARCHITECTURE}" in + x86_64) printf '%s\n' amd64 ;; + aarch64) printf '%s\n' arm64 ;; + *) + echo "unsupported cache architecture: ${TEST_VM_ARCHITECTURE}" >&2 + return 1 + ;; + esac +} + +test_vm_cache_sha256() { + if [ ! -r "$1" ]; then + echo "cache identity input is not readable: $1" >&2 + return 1 + fi + sha256sum "$1" | cut -d ' ' -f 1 +} + +test_vm_cache_key() { + local distro_name=$1 + shift + local configuration + local configuration_hash + local architecture + local seal_hash + local material + local configuration_line + + architecture=$(test_vm_cache_oci_architecture) || return 1 + seal_hash=$(test_vm_cache_sha256 "${OPENSHELL_TEST_VM_CACHE_SEAL}") || return 1 + printf -v material \ + 'schema=%s\ngeneration=%s\ndisk_layout=%s\ndistro=%s\nos_version=%s\narchitecture=%s\nbase_url=%s\nbase_hash=%s\noverlay_growth=%s\nansible_version=%s\nseal_sha256=%s\n' \ + "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + "${TEST_VM_CACHE_GENERATION}" \ + "${TEST_VM_CACHE_DISK_LAYOUT}" \ + "${distro_name}" \ + "${TEST_VM_OS_VERSION}" \ + "${architecture}" \ + "${TEST_VM_IMAGE_URL}" \ + "${TEST_VM_IMAGE_HASH}" \ + 16G \ + "${TEST_VM_ANSIBLE_VERSION}" \ + "${seal_hash}" + + for configuration in "$@"; do + configuration_hash=$( + test_vm_cache_sha256 \ + "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${configuration}" + ) || return 1 + printf -v configuration_line \ + 'configuration=%s:%s\n' \ + "${configuration}" \ + "${configuration_hash}" + material+=${configuration_line} + done + + printf '%s' "${material}" | sha256sum | cut -d ' ' -f 1 +} + +test_vm_cache_tag() { + local distro_name=$1 + local key=$2 + printf 'v%s-%s-%s-%s\n' \ + "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + "${distro_name}" \ + "$(test_vm_cache_oci_architecture)" \ + "${key}" +} + +test_vm_cache_entry_dir() { + local root=$1 + local key=$2 + printf '%s/entries/%s\n' "${root}" "${key}" +} + +test_vm_cache_metadata_matches() { + local metadata=$1 + local key=$2 + local distro_name=$3 + shift 3 + local architecture + local expected_configurations + architecture=$(test_vm_cache_oci_architecture) || return 1 + expected_configurations=$(jq -cn --args '$ARGS.positional' "$@") || return 1 + + jq -e \ + --argjson schema "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + --arg key "${key}" \ + --arg distro "${distro_name}" \ + --arg os_version "${TEST_VM_OS_VERSION}" \ + --arg architecture "$(test_vm_cache_oci_architecture)" \ + --arg base_hash "${TEST_VM_IMAGE_HASH}" \ + --argjson configurations "${expected_configurations}" \ + ' + .schema == $schema and + .key == $key and + .distro == $distro and + .os_version == $os_version and + .architecture == $architecture and + .base_image_hash == $base_hash and + .configurations == $configurations + ' "${metadata}" >/dev/null +} + +test_vm_cache_local_entry_valid() { + local root=$1 + local key=$2 + local distro_name=$3 + shift 3 + local entry + entry=$(test_vm_cache_entry_dir "${root}" "${key}") + + [ -f "${entry}/complete" ] && + [ -f "${entry}/disk.qcow2" ] && + [ -f "${entry}/metadata.json" ] && + test_vm_cache_metadata_matches \ + "${entry}/metadata.json" "${key}" "${distro_name}" "$@" +} + +test_vm_cache_validate_disk() { + local disk=$1 + local info + info=$(qemu-img info --output=json "${disk}") + + jq -e ' + .format == "qcow2" and + (.["backing-filename"]? == null) and + (.["data-file"]? == null) and + ((.snapshots? // []) | length == 0) and + (.["virtual-size"] > 0) and + (.["virtual-size"] <= 68719476736) + ' <<<"${info}" >/dev/null + qemu-img check "${disk}" >/dev/null +} diff --git a/nix/test-guest/cache-seal.sh b/nix/test-guest/cache-seal.sh new file mode 100644 index 0000000000..9f0da8564d --- /dev/null +++ b/nix/test-guest/cache-seal.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sanitize a configured test guest before its disk is published as a cache base. + +set -Eeuo pipefail + +if [ "$(id -u)" -ne 0 ]; then + echo "cache sealing must run as root" >&2 + exit 1 +fi + +rm -f /home/openshell/.ssh/authorized_keys +rm -f /root/.ssh/authorized_keys +rm -f /etc/ssh/ssh_host_*_key /etc/ssh/ssh_host_*_key.pub + +if command -v cloud-init >/dev/null 2>&1; then + cloud-init clean --logs --machine-id || true +fi +rm -rf /var/lib/cloud/instance /var/lib/cloud/instances /var/lib/cloud/seed +mkdir -p /var/lib/cloud/instances + +: >/etc/machine-id +rm -f /var/lib/dbus/machine-id +rm -f /var/lib/systemd/random-seed +rm -f /var/lib/NetworkManager/*lease* /var/lib/dhcp/*lease* 2>/dev/null || true + +rm -f /root/.bash_history /home/openshell/.bash_history +rm -f /root/.docker/config.json /home/openshell/.docker/config.json +rm -f /root/.config/containers/auth.json +rm -f /home/openshell/.config/containers/auth.json + +if command -v apt-get >/dev/null 2>&1; then + apt-get clean +fi +if command -v dnf >/dev/null 2>&1; then + dnf clean all +fi + +journalctl --rotate >/dev/null 2>&1 || true +journalctl --vacuum-time=1s >/dev/null 2>&1 || true +find /var/log -type f -exec truncate -s 0 {} + 2>/dev/null || true +find /tmp /var/tmp -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + 2>/dev/null || true + +rm -f -- "$0" +sync + +# Deleted credentials can remain in allocated blocks. Fill free space with +# zeroes so qemu-img convert can safely omit those blocks from the cache disk. +zero_file=/var/tmp/openshell-cache-zero +dd if=/dev/zero of="${zero_file}" bs=64M status=none 2>/dev/null || true +rm -f "${zero_file}" +sync + +# The authorized key is gone, so arrange shutdown before returning to the host. +nohup /bin/sh -c 'sleep 2; systemctl poweroff' /dev/null 2>&1 & diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh new file mode 100644 index 0000000000..b986077e80 --- /dev/null +++ b/nix/test-guest/cache.sh @@ -0,0 +1,444 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Ensure a prepared test guest disk exists locally, optionally backed by OCI. + +set -Eeuo pipefail + +usage() { + cat <<'EOF' +Usage: + nix run .#test-guest-cache -- --distro DISTRO [OPTIONS] + +Options: + --distro NAME Base distro: ubuntu, centos, fedora, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, selinux) + --repository REF OCI repository without a tag + --cache-dir PATH Override the local prepared-disk cache directory + --push Publish a newly built or local entry to the repository + -h, --help Show this help + +The command ensures one exact distro, architecture, and ordered configuration +combination exists locally. It checks the local cache, pulls from OCI when a +repository is configured, or builds and validates a prepared disk on a miss. +Publishing is explicit and requires both --repository and --push. +EOF +} + +if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || + [ ! -d "${OPENSHELL_TEST_VM_DISTROS:-}" ] || + [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ] || + [ ! -r "${OPENSHELL_TEST_VM_CACHE_LIB:-}" ] || + [ ! -r "${OPENSHELL_TEST_VM_CACHE_SEAL:-}" ] || + [ ! -r "${OPENSHELL_TEST_VM_RUNNER:-}" ]; then + echo "run this script through 'nix run .#test-guest-cache -- ...'" >&2 + exit 2 +fi + +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_CACHE_LIB}" + +require_value() { + if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then + echo "$1 requires a value" >&2 + exit 2 + fi +} + +distro= +repository=${OPENSHELL_TEST_VM_CACHE_REPOSITORY:-} +cache_dir= +push=0 +configurations=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --distro) + require_value "$@" + distro=$2 + shift 2 + ;; + --with) + require_value "$@" + configurations+=("$2") + shift 2 + ;; + --repository) + require_value "$@" + repository=$2 + shift 2 + ;; + --cache-dir) + require_value "$@" + cache_dir=$2 + shift 2 + ;; + --push) + push=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "unknown test guest cache argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ -z "${distro}" ]; then + echo "--distro is required" >&2 + usage >&2 + exit 2 +fi +if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + [ ! -r "${OPENSHELL_TEST_VM_DISTROS}/${distro}" ]; then + echo "unknown distro: ${distro}" >&2 + exit 2 +fi + +# Distro profiles contain only trusted values generated into the Nix store. +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_DISTROS}/${distro}" + +for item in "${configurations[@]}"; do + if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] || + [ ! -r "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" ]; then + echo "unknown configuration: ${item:-}" >&2 + exit 2 + fi +done + +if [ "${push}" -eq 1 ] && [ -z "${repository}" ]; then + echo "--push requires --repository" >&2 + exit 2 +fi +if [[ ${repository} == *[[:space:]]* ]] || [[ ${repository} == *@* ]]; then + echo "--repository must be an untagged OCI repository reference" >&2 + exit 2 +fi + +if [ -n "${cache_dir}" ]; then + OPENSHELL_TEST_VM_CACHE_DIR=${cache_dir} + export OPENSHELL_TEST_VM_CACHE_DIR +fi + +umask 077 +cache_root=$(test_vm_cache_root) +cache_key=$(test_vm_cache_key "${distro}" "${configurations[@]}") +cache_tag=$(test_vm_cache_tag "${distro}" "${cache_key}") +entry_dir=$(test_vm_cache_entry_dir "${cache_root}" "${cache_key}") +remote_ref= +if [ -n "${repository}" ]; then + remote_ref="${repository}:${cache_tag}" +fi + +mkdir -p "${cache_root}/entries" "${cache_root}/locks" "${cache_root}/staging" + +lock_dir= +build_stage= +preserve_build_stage=0 + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ -n "${lock_dir}" ]; then + rmdir "${lock_dir}" 2>/dev/null || true + fi + if [ -n "${build_stage}" ] && [ -d "${build_stage}" ]; then + if [ "${status}" -ne 0 ] && [ "${preserve_build_stage}" -eq 1 ]; then + echo "Kept failed cache build state at ${build_stage}" >&2 + else + rm -rf "${build_stage}" + fi + fi + exit "${status}" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +for _ in $(seq 1 120); do + if mkdir "${cache_root}/locks/${cache_key}.lock" 2>/dev/null; then + lock_dir="${cache_root}/locks/${cache_key}.lock" + break + fi + sleep 1 +done +if [ -z "${lock_dir}" ]; then + echo "timed out waiting for cache lock: ${cache_key}" >&2 + exit 1 +fi + +install_entry() { + local disk=$1 + local metadata=$2 + local temporary_entry + temporary_entry=$(mktemp -d "${cache_root}/entries/.${cache_key}.XXXXXX") + cp --sparse=always "${disk}" "${temporary_entry}/disk.qcow2" + chmod 0444 "${temporary_entry}/disk.qcow2" + install -m 0444 "${metadata}" "${temporary_entry}/metadata.json" + : >"${temporary_entry}/complete" + chmod 0444 "${temporary_entry}/complete" + + if [ -e "${entry_dir}" ]; then + if test_vm_cache_local_entry_valid \ + "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then + rm -rf "${temporary_entry}" + return + fi + echo "cache entry appeared concurrently but is invalid: ${entry_dir}" >&2 + rm -rf "${temporary_entry}" + return 1 + fi + mv "${temporary_entry}" "${entry_dir}" +} + +is_remote_miss() { + grep -Eqi '404|MANIFEST_UNKNOWN|manifest unknown|not found' "$1" +} + +pull_remote() { + local pull_dir + local pull_log + local compressed_size + local expected_sha + local actual_sha + pull_dir=$(mktemp -d "${cache_root}/staging/pull.${cache_key}.XXXXXX") + pull_log="${pull_dir}/oras.log" + + if ! oras pull --output "${pull_dir}" "${remote_ref}" >"${pull_log}" 2>&1; then + if is_remote_miss "${pull_log}"; then + rm -rf "${pull_dir}" + return 1 + fi + cat "${pull_log}" >&2 + rm -rf "${pull_dir}" + return 2 + fi + + if [ ! -f "${pull_dir}/metadata.json" ] || + [ ! -f "${pull_dir}/disk.qcow2.zst" ]; then + echo "OCI cache artifact is missing metadata.json or disk.qcow2.zst" >&2 + rm -rf "${pull_dir}" + return 2 + fi + if ! test_vm_cache_metadata_matches \ + "${pull_dir}/metadata.json" "${cache_key}" "${distro}" "${configurations[@]}"; then + echo "OCI cache metadata does not match the requested VM" >&2 + rm -rf "${pull_dir}" + return 2 + fi + + compressed_size=$(wc -c <"${pull_dir}/disk.qcow2.zst") + if [ "${compressed_size}" -gt 17179869184 ]; then + echo "OCI cache disk exceeds the 16 GiB compressed size limit" >&2 + rm -rf "${pull_dir}" + return 2 + fi + + ( + # Limit the decompressed file to 32 GiB before qemu-img parses it. + ulimit -f 67108864 + zstd -d --sparse -f \ + "${pull_dir}/disk.qcow2.zst" \ + -o "${pull_dir}/disk.qcow2" + ) + expected_sha=$(jq -r '.disk_sha256' "${pull_dir}/metadata.json") + actual_sha=$(test_vm_cache_sha256 "${pull_dir}/disk.qcow2") + if [ "${expected_sha}" != "${actual_sha}" ]; then + echo "OCI cache disk checksum does not match metadata" >&2 + rm -rf "${pull_dir}" + return 2 + fi + if ! test_vm_cache_validate_disk "${pull_dir}/disk.qcow2"; then + echo "OCI cache disk failed QCOW2 validation" >&2 + rm -rf "${pull_dir}" + return 2 + fi + + install_entry "${pull_dir}/disk.qcow2" "${pull_dir}/metadata.json" + rm -rf "${pull_dir}" + echo "==> Cache remote hit: ${remote_ref}" +} + +build_local() { + local -a prepare_args + local -a validate_args + local -a run_dirs + local configuration + local prepared_disk + local validation + local configuration_json + local disk_sha + local virtual_size + local created + + build_stage=$(mktemp -d "${cache_root}/staging/build.${cache_key}.XXXXXX") + preserve_build_stage=1 + mkdir -p "${build_stage}/prepare-tmp" "${build_stage}/validate-tmp" + + prepare_args=(--distro "${distro}" --keep) + for configuration in "${configurations[@]}"; do + prepare_args+=(--with "${configuration}") + done + prepare_args+=( + --copy + "${OPENSHELL_TEST_VM_CACHE_SEAL}:/usr/local/sbin/openshell-test-guest-cache-seal" + -- + sudo + /usr/local/sbin/openshell-test-guest-cache-seal + ) + + echo "==> Cache miss: preparing ${distro} ($(test_vm_cache_oci_architecture))" + TMPDIR="${build_stage}/prepare-tmp" \ + "${TEST_VM_BASH}" "${OPENSHELL_TEST_VM_RUNNER}" "${prepare_args[@]}" + + shopt -s nullglob + run_dirs=("${build_stage}/prepare-tmp/openshell-test-guest"/run.*) + shopt -u nullglob + if [ "${#run_dirs[@]}" -ne 1 ] || + [ ! -f "${run_dirs[0]}/disk.qcow2" ]; then + echo "cache preparation did not retain exactly one guest disk" >&2 + return 1 + fi + + prepared_disk="${build_stage}/disk.qcow2" + qemu-img convert -q -f qcow2 -O qcow2 \ + "${run_dirs[0]}/disk.qcow2" "${prepared_disk}" + test_vm_cache_validate_disk "${prepared_disk}" + + validation='test -s /etc/machine-id; test -n "$(find /etc/ssh -name "ssh_host_*_key" -type f -print -quit)"' + for configuration in "${configurations[@]}"; do + case "${configuration}" in + docker) validation+='; docker info >/dev/null' ;; + podman) validation+='; podman info >/dev/null' ;; + selinux) validation+='; test "$(getenforce)" = Enforcing' ;; + esac + done + + validate_args=(--distro "${distro}") + for configuration in "${configurations[@]}"; do + validate_args+=(--with "${configuration}") + done + validate_args+=(-- bash -lc "${validation}") + + echo "==> Validating fresh boot from prepared cache disk" + TMPDIR="${build_stage}/validate-tmp" \ + OPENSHELL_TEST_VM_IMAGE_OVERRIDE="${prepared_disk}" \ + "${TEST_VM_BASH}" "${OPENSHELL_TEST_VM_RUNNER}" "${validate_args[@]}" + + configuration_json=$(jq -cn --args '$ARGS.positional' "${configurations[@]}") + disk_sha=$(test_vm_cache_sha256 "${prepared_disk}") + virtual_size=$( + qemu-img info --output=json "${prepared_disk}" | + jq -r '.["virtual-size"]' + ) + created=$(date -u +%Y-%m-%dT%H:%M:%SZ) + + jq -n \ + --argjson schema "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + --arg key "${cache_key}" \ + --arg distro "${distro}" \ + --arg os_version "${TEST_VM_OS_VERSION}" \ + --arg architecture "$(test_vm_cache_oci_architecture)" \ + --arg base_image_url "${TEST_VM_IMAGE_URL}" \ + --arg base_image_hash "${TEST_VM_IMAGE_HASH}" \ + --arg disk_layout "${TEST_VM_CACHE_DISK_LAYOUT}" \ + --arg disk_sha256 "${disk_sha}" \ + --arg created "${created}" \ + --argjson virtual_size "${virtual_size}" \ + --argjson configurations "${configuration_json}" \ + '{ + schema: $schema, + key: $key, + distro: $distro, + os_version: $os_version, + architecture: $architecture, + base_image_url: $base_image_url, + base_image_hash: $base_image_hash, + configurations: $configurations, + disk_layout: $disk_layout, + disk_sha256: $disk_sha256, + virtual_size: $virtual_size, + created: $created + }' >"${build_stage}/metadata.json" + + install_entry "${prepared_disk}" "${build_stage}/metadata.json" + preserve_build_stage=0 + rm -rf "${build_stage}" + build_stage= + echo "==> Cache build complete: ${entry_dir}" +} + +push_remote() { + local push_dir + local manifest_log + push_dir=$(mktemp -d "${cache_root}/staging/push.${cache_key}.XXXXXX") + manifest_log="${push_dir}/manifest.log" + + if oras manifest fetch "${remote_ref}" >"${manifest_log}" 2>&1; then + echo "==> Cache already published: ${remote_ref}" + rm -rf "${push_dir}" + return + fi + if ! is_remote_miss "${manifest_log}"; then + cat "${manifest_log}" >&2 + rm -rf "${push_dir}" + return 1 + fi + + install -m 0644 "${entry_dir}/metadata.json" "${push_dir}/metadata.json" + zstd -T0 -3 -f \ + "${entry_dir}/disk.qcow2" \ + -o "${push_dir}/disk.qcow2.zst" + + ( + cd "${push_dir}" + oras push \ + --artifact-type "${TEST_VM_CACHE_ARTIFACT_TYPE}" \ + "${remote_ref}" \ + "metadata.json:${TEST_VM_CACHE_METADATA_TYPE}" \ + "disk.qcow2.zst:${TEST_VM_CACHE_DISK_TYPE}" + ) + rm -rf "${push_dir}" + echo "==> Cache push complete: ${remote_ref}" +} + +echo "==> Cache key: ${cache_key}" +if test_vm_cache_local_entry_valid \ + "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then + echo "==> Cache local hit: ${entry_dir}" +else + if [ -e "${entry_dir}" ]; then + rejected="${cache_root}/staging/rejected.${cache_key}.$(date +%s)" + mv "${entry_dir}" "${rejected}" + echo "Moved invalid cache entry to ${rejected}" >&2 + fi + + pulled=0 + if [ -n "${remote_ref}" ]; then + if pull_remote; then + pulled=1 + else + pull_status=$? + if [ "${pull_status}" -ne 1 ]; then + exit "${pull_status}" + fi + echo "==> Cache remote miss: ${remote_ref}" + fi + fi + if [ "${pulled}" -eq 0 ]; then + build_local + fi +fi + +if [ "${push}" -eq 1 ]; then + push_remote +fi + +echo "==> Cache ready: ${entry_dir}" diff --git a/nix/vm/configuration/docker.yml b/nix/test-guest/configuration/docker.yml similarity index 97% rename from nix/vm/configuration/docker.yml rename to nix/test-guest/configuration/docker.yml index 1b25e365f4..612c9e1c81 100644 --- a/nix/vm/configuration/docker.yml +++ b/nix/test-guest/configuration/docker.yml @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# PROTOTYPE: Configure Docker in a disposable test VM. +# PROTOTYPE: Configure Docker in a disposable test guest. - name: Configure Docker hosts: test_vm diff --git a/nix/vm/configuration/podman.yml b/nix/test-guest/configuration/podman.yml similarity index 94% rename from nix/vm/configuration/podman.yml rename to nix/test-guest/configuration/podman.yml index 3012c59f3e..2872f5cbfe 100644 --- a/nix/vm/configuration/podman.yml +++ b/nix/test-guest/configuration/podman.yml @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# PROTOTYPE: Configure Podman in a disposable test VM. +# PROTOTYPE: Configure Podman in a disposable test guest. - name: Configure Podman hosts: test_vm diff --git a/nix/vm/configuration/selinux.yml b/nix/test-guest/configuration/selinux.yml similarity index 96% rename from nix/vm/configuration/selinux.yml rename to nix/test-guest/configuration/selinux.yml index 6f34e16c67..958feb4172 100644 --- a/nix/vm/configuration/selinux.yml +++ b/nix/test-guest/configuration/selinux.yml @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# PROTOTYPE: Ensure SELinux is enforcing in a disposable test VM. +# PROTOTYPE: Ensure SELinux is enforcing in a disposable test guest. - name: Configure SELinux hosts: test_vm diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix new file mode 100644 index 0000000000..9863f6d767 --- /dev/null +++ b/nix/test-guest/default.nix @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Composable distro VMs for installing and exercising artifacts. + +{ pkgs }: + +let + isAarch64 = pkgs.stdenv.hostPlatform.isAarch64; + isDarwin = pkgs.stdenv.hostPlatform.isDarwin; + architecture = if isAarch64 then "aarch64" else "x86_64"; + qemu = pkgs.qemu.override { hostCpuOnly = true; }; + qemuBinary = + if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; + + distros = { + ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + centos = import ./distros/centos.nix { inherit pkgs architecture; }; + fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; + rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; + }; + + configurations = { + docker = ./configuration/docker.yml; + podman = ./configuration/podman.yml; + selinux = ./configuration/selinux.yml; + }; + + mkDistroProfile = + name: distro: + pkgs.writeText "openshell-test-guest-${name}" '' + TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath} + TEST_VM_IMAGE_URL=${pkgs.lib.escapeShellArg distro.imageUrl} + TEST_VM_IMAGE_HASH=${pkgs.lib.escapeShellArg distro.imageHash} + TEST_VM_OS_ID=${pkgs.lib.escapeShellArg distro.osId} + TEST_VM_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion} + TEST_VM_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily} + export TEST_VM_IMAGE_DRV TEST_VM_IMAGE_URL TEST_VM_IMAGE_HASH + export TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY + ''; + + distroCatalog = pkgs.linkFarm "openshell-test-guest-distros" ( + pkgs.lib.mapAttrsToList (name: distro: { + inherit name; + path = mkDistroProfile name distro; + }) distros + ); + + configurationCatalog = pkgs.linkFarm "openshell-test-guest-configurations" ( + pkgs.lib.mapAttrsToList (name: path: { inherit name path; }) configurations + ); + + runtimeInputs = [ + qemu + pkgs.python3Packages.ansible-core + pkgs.coreutils + pkgs.gnugrep + pkgs.jq + pkgs.nix + pkgs.openssh + pkgs.oras + pkgs.python3 + pkgs.xorriso + pkgs.zstd + ]; + + runtimeEnvironment = '' + export OPENSHELL_TEST_VM_RUNTIME=1 + export OPENSHELL_TEST_VM_DISTROS=${distroCatalog} + export OPENSHELL_TEST_VM_CONFIGURATIONS=${configurationCatalog} + export OPENSHELL_TEST_VM_CACHE_LIB=${./cache-lib.sh} + export OPENSHELL_TEST_VM_CACHE_SEAL=${./cache-seal.sh} + export OPENSHELL_TEST_VM_RUNNER=${./run.sh} + export TEST_VM_BASH=${pkgs.bash}/bin/bash + export TEST_VM_QEMU=${qemuBinary} + export TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} + export TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} + export TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} + export TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + export TEST_VM_ARCHITECTURE=${architecture} + export TEST_VM_ANSIBLE_VERSION=${pkgs.python3Packages.ansible-core.version} + export TEST_VM_CACHE_GENERATION=1 + ''; + + runner = pkgs.writeShellApplication { + name = "openshell-test-guest"; + inherit runtimeInputs; + text = runtimeEnvironment + '' + exec ${pkgs.bash}/bin/bash ${./run.sh} "$@" + ''; + }; + + cacheRunner = pkgs.writeShellApplication { + name = "openshell-test-guest-cache"; + inherit runtimeInputs; + text = runtimeEnvironment + '' + exec ${pkgs.bash}/bin/bash ${./cache.sh} "$@" + ''; + }; +in +{ + app = { + type = "app"; + program = "${runner}/bin/openshell-test-guest"; + meta.description = "Boot and configure a disposable distro guest"; + }; + + cacheApp = { + type = "app"; + program = "${cacheRunner}/bin/openshell-test-guest-cache"; + meta.description = "Ensure a prepared test guest disk is available locally or in OCI"; + }; +} diff --git a/nix/test-guest/distros/centos.nix b/nix/test-guest/distros/centos.nix new file mode 100644 index 0000000000..5dcce2d755 --- /dev/null +++ b/nix/test-guest/distros/centos.nix @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageUrl = "https://cloud.centos.org/centos/10-stream/${architecture}/images/CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; + imageHash = + if architecture == "aarch64" then + "sha256-55IuyMUvsbpvqgug2S7w6JpLCSIpR4HJVkuMch60Rag=" + else + "sha256-k3lpRd9eVJUr4hyoUGfwyfOxGi3W7iFjFU/ZdIxMQdc="; +in +{ + osId = "centos"; + osVersion = "10"; + packageFamily = "rpm"; + inherit imageUrl imageHash; + image = pkgs.fetchurl { + name = "CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; + url = imageUrl; + hash = imageHash; + }; +} diff --git a/nix/test-guest/distros/fedora.nix b/nix/test-guest/distros/fedora.nix new file mode 100644 index 0000000000..f7784d046b --- /dev/null +++ b/nix/test-guest/distros/fedora.nix @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageUrl = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/${architecture}/images/Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; + imageHash = + if architecture == "aarch64" then + "sha256-VcYKO4DTYWoIcFr9BFnnX+nwPFSrp6RuQAKkGnL6DVs=" + else + "sha256-KGgP5bNxpaguv0OjGSbghqFo5ZlJ0DlpxQk+cHH5C38="; +in +{ + osId = "fedora"; + osVersion = "44"; + packageFamily = "rpm"; + inherit imageUrl imageHash; + image = pkgs.fetchurl { + name = "Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; + url = imageUrl; + hash = imageHash; + }; +} diff --git a/nix/test-guest/distros/rocky.nix b/nix/test-guest/distros/rocky.nix new file mode 100644 index 0000000000..5ff610dfde --- /dev/null +++ b/nix/test-guest/distros/rocky.nix @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageUrl = "https://download.rockylinux.org/pub/rocky/9/images/${architecture}/Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; + imageHash = + if architecture == "aarch64" then + "sha256-JGkqRE8fC4u5U3XDjItD+AmaEVNHYjaRviwzC0DIof4=" + else + "sha256-ksIGzG95DGFYMkfu/oeJD4goQgZiwXys8kfOx4q07sg="; +in +{ + osId = "rocky"; + osVersion = "9"; + packageFamily = "rpm"; + inherit imageUrl imageHash; + image = pkgs.fetchurl { + name = "Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; + url = imageUrl; + hash = imageHash; + }; +} diff --git a/nix/vm/distros/ubuntu.nix b/nix/test-guest/distros/ubuntu.nix similarity index 52% rename from nix/vm/distros/ubuntu.nix rename to nix/test-guest/distros/ubuntu.nix index 36d4c75c64..3f59e92f8c 100644 --- a/nix/vm/distros/ubuntu.nix +++ b/nix/test-guest/distros/ubuntu.nix @@ -5,18 +5,21 @@ let imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64"; + imageUrl = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; + imageHash = + if architecture == "aarch64" then + "sha256-meHUgrlY5r/QGDpMSM5twzTgmj4ppFYPb1/4VZPQnR0=" + else + "sha256-eqbZ9eijpVx0RbE40xpz0Rh4cSEbK32p2i4abL8WmyE="; in { osId = "ubuntu"; osVersion = "24.04"; packageFamily = "deb"; + inherit imageUrl imageHash; image = pkgs.fetchurl { name = "ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; - url = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-${imageArchitecture}.img"; - hash = - if architecture == "aarch64" then - "sha256-meHUgrlY5r/QGDpMSM5twzTgmj4ppFYPb1/4VZPQnR0=" - else - "sha256-eqbZ9eijpVx0RbE40xpz0Rh4cSEbK32p2i4abL8WmyE="; + url = imageUrl; + hash = imageHash; }; } diff --git a/nix/vm/run.sh b/nix/test-guest/run.sh similarity index 84% rename from nix/vm/run.sh rename to nix/test-guest/run.sh index 1a54ea4a33..bf92e639ac 100644 --- a/nix/vm/run.sh +++ b/nix/test-guest/run.sh @@ -9,7 +9,7 @@ set -Eeuo pipefail usage() { cat <<'EOF' Usage: - nix run .#test-vm -- --distro DISTRO [OPTIONS] [-- COMMAND...] + nix run .#test-guest -- --distro DISTRO [OPTIONS] [-- COMMAND...] Options: --distro NAME Base distro: ubuntu, centos, fedora, or rocky @@ -27,8 +27,9 @@ EOF if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || [ ! -d "${OPENSHELL_TEST_VM_DISTROS:-}" ] || - [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ]; then - echo "run this script through 'nix run .#test-vm -- ...'" >&2 + [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ] || + [ ! -r "${OPENSHELL_TEST_VM_CACHE_LIB:-}" ]; then + echo "run this script through 'nix run .#test-guest -- ...'" >&2 exit 2 fi @@ -81,7 +82,7 @@ while [ "$#" -gt 0 ]; do break ;; *) - echo "unknown test VM argument: $1" >&2 + echo "unknown test guest argument: $1" >&2 usage >&2 exit 2 ;; @@ -176,11 +177,38 @@ if [ "${TEST_VM_ACCELERATOR}" = kvm ] && ssh_wait_seconds=600 fi -echo "==> Realizing the pinned ${distro} cloud image" -TEST_VM_IMAGE=$(nix build --no-link --print-out-paths "${TEST_VM_IMAGE_DRV}^out") +# shellcheck disable=SC1090 +. "${OPENSHELL_TEST_VM_CACHE_LIB}" + +prepared_image=0 +TEST_VM_IMAGE= +if [ -n "${OPENSHELL_TEST_VM_IMAGE_OVERRIDE:-}" ]; then + if [ ! -f "${OPENSHELL_TEST_VM_IMAGE_OVERRIDE}" ]; then + echo "prepared guest image does not exist: ${OPENSHELL_TEST_VM_IMAGE_OVERRIDE}" >&2 + exit 2 + fi + TEST_VM_IMAGE=${OPENSHELL_TEST_VM_IMAGE_OVERRIDE} + prepared_image=1 + echo "==> Using explicit prepared guest image" +elif [ "${OPENSHELL_TEST_VM_CACHE_DISABLE:-0}" -ne 1 ]; then + cache_root=$(test_vm_cache_root) + cache_key=$(test_vm_cache_key "${distro}" "${configurations[@]}") + if test_vm_cache_local_entry_valid \ + "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then + cache_entry=$(test_vm_cache_entry_dir "${cache_root}" "${cache_key}") + TEST_VM_IMAGE="${cache_entry}/disk.qcow2" + prepared_image=1 + echo "==> Cache local hit: ${cache_entry}" + fi +fi + +if [ "${prepared_image}" -eq 0 ]; then + echo "==> Realizing the pinned ${distro} cloud image" + TEST_VM_IMAGE=$(nix build --no-link --print-out-paths "${TEST_VM_IMAGE_DRV}^out") +fi umask 077 -run_parent=${TMPDIR:-/tmp}/openshell-test-vm +run_parent=${TMPDIR:-/tmp}/openshell-test-guest mkdir -p "${run_parent}" run_dir=$(mktemp -d "${run_parent%/}/run.XXXXXX") overlay=${run_dir}/disk.qcow2 @@ -218,7 +246,7 @@ cleanup() { show_logs fi if [ "${keep}" -eq 1 ]; then - echo "Kept test VM state at ${run_dir}" >&2 + echo "Kept test guest state at ${run_dir}" >&2 else rm -rf "${run_dir}" fi @@ -237,8 +265,8 @@ ssh-keygen -q -t ed25519 -N "" -f "${private_key}" public_key=$(<"${private_key}.pub") cat >"${run_dir}/meta-data" <"${run_dir}/user-data" <"${ansible_inventory}" < Applying configuration: ${item}" - ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ - ansible-playbook "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" -done +if [ "${prepared_image}" -eq 0 ]; then + for item in "${configurations[@]}"; do + echo "==> Applying configuration: ${item}" + ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ + ansible-playbook "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" + done +else + echo "==> Reusing cached configuration: ${configurations[*]:-base image}" +fi for package in "${packages[@]}"; do package_name=$(basename "${package}") @@ -421,7 +455,7 @@ for copy_spec in "${copies[@]}"; do "sudo install -D -m 0755 '${remote_copy}' '${destination}'; rm -f '${remote_copy}'" done -echo "==> Test VM ready: ${distro} (SSH port ${ssh_port})" +echo "==> Test guest ready: ${distro} (SSH port ${ssh_port})" if [ "${#guest_command[@]}" -eq 0 ]; then ssh -t "${ssh_args[@]}" openshell@127.0.0.1 else diff --git a/nix/vm/default.nix b/nix/vm/default.nix deleted file mode 100644 index ec82b4b8be..0000000000 --- a/nix/vm/default.nix +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# PROTOTYPE: Composable distro VMs for installing and exercising artifacts. - -{ pkgs }: - -let - isAarch64 = pkgs.stdenv.hostPlatform.isAarch64; - isDarwin = pkgs.stdenv.hostPlatform.isDarwin; - architecture = if isAarch64 then "aarch64" else "x86_64"; - qemu = pkgs.qemu.override { hostCpuOnly = true; }; - qemuBinary = - if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; - - distros = { - ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; - centos = import ./distros/centos.nix { inherit pkgs architecture; }; - fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; - rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; - }; - - configurations = { - docker = ./configuration/docker.yml; - podman = ./configuration/podman.yml; - selinux = ./configuration/selinux.yml; - }; - - mkDistroProfile = - name: distro: - pkgs.writeText "openshell-test-vm-${name}" '' - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath} - TEST_VM_OS_ID=${pkgs.lib.escapeShellArg distro.osId} - TEST_VM_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion} - TEST_VM_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily} - export TEST_VM_IMAGE_DRV TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY - ''; - - distroCatalog = pkgs.linkFarm "openshell-test-vm-distros" ( - pkgs.lib.mapAttrsToList (name: distro: { - inherit name; - path = mkDistroProfile name distro; - }) distros - ); - - configurationCatalog = pkgs.linkFarm "openshell-test-vm-configurations" ( - pkgs.lib.mapAttrsToList (name: path: { inherit name path; }) configurations - ); - - runner = pkgs.writeShellApplication { - name = "openshell-test-vm"; - runtimeInputs = [ - qemu - pkgs.python3Packages.ansible-core - pkgs.coreutils - pkgs.gnugrep - pkgs.nix - pkgs.openssh - pkgs.python3 - pkgs.xorriso - ]; - text = '' - export OPENSHELL_TEST_VM_RUNTIME=1 - export OPENSHELL_TEST_VM_DISTROS=${distroCatalog} - export OPENSHELL_TEST_VM_CONFIGURATIONS=${configurationCatalog} - export TEST_VM_QEMU=${qemuBinary} - export TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} - export TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} - export TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} - export TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} - export TEST_VM_ARCHITECTURE=${architecture} - exec ${pkgs.bash}/bin/bash ${./run.sh} "$@" - ''; - }; -in -{ - app = { - type = "app"; - program = "${runner}/bin/openshell-test-vm"; - meta.description = "Boot and configure a disposable distro VM"; - }; -} diff --git a/nix/vm/distros/centos.nix b/nix/vm/distros/centos.nix deleted file mode 100644 index f38ad68533..0000000000 --- a/nix/vm/distros/centos.nix +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -{ pkgs, architecture }: - -{ - osId = "centos"; - osVersion = "10"; - packageFamily = "rpm"; - image = pkgs.fetchurl { - name = "CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; - url = "https://cloud.centos.org/centos/10-stream/${architecture}/images/CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2"; - hash = - if architecture == "aarch64" then - "sha256-55IuyMUvsbpvqgug2S7w6JpLCSIpR4HJVkuMch60Rag=" - else - "sha256-k3lpRd9eVJUr4hyoUGfwyfOxGi3W7iFjFU/ZdIxMQdc="; - }; -} diff --git a/nix/vm/distros/fedora.nix b/nix/vm/distros/fedora.nix deleted file mode 100644 index d8701ed166..0000000000 --- a/nix/vm/distros/fedora.nix +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -{ pkgs, architecture }: - -{ - osId = "fedora"; - osVersion = "44"; - packageFamily = "rpm"; - image = pkgs.fetchurl { - name = "Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; - url = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/${architecture}/images/Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2"; - hash = - if architecture == "aarch64" then - "sha256-VcYKO4DTYWoIcFr9BFnnX+nwPFSrp6RuQAKkGnL6DVs=" - else - "sha256-KGgP5bNxpaguv0OjGSbghqFo5ZlJ0DlpxQk+cHH5C38="; - }; -} diff --git a/nix/vm/distros/rocky.nix b/nix/vm/distros/rocky.nix deleted file mode 100644 index 48fc162e89..0000000000 --- a/nix/vm/distros/rocky.nix +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -{ pkgs, architecture }: - -{ - osId = "rocky"; - osVersion = "9"; - packageFamily = "rpm"; - image = pkgs.fetchurl { - name = "Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; - url = "https://download.rockylinux.org/pub/rocky/9/images/${architecture}/Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2"; - hash = - if architecture == "aarch64" then - "sha256-JGkqRE8fC4u5U3XDjItD+AmaEVNHYjaRviwzC0DIof4=" - else - "sha256-ksIGzG95DGFYMkfu/oeJD4goQgZiwXys8kfOx4q07sg="; - }; -} From eebc077adde381b6a1dd451f95cc6f29bc643c4f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 12:54:43 -0700 Subject: [PATCH 07/29] perf(test-guest): accelerate cached guest startup Signed-off-by: Drew Newberry --- nix/test-guest/README.md | 4 +- nix/test-guest/cache-lib.sh | 44 +++++++-------- nix/test-guest/cache.sh | 50 ++++++++--------- nix/test-guest/default.nix | 46 ++++++++-------- nix/test-guest/run.sh | 106 ++++++++++++++++++++---------------- 5 files changed, 131 insertions(+), 119 deletions(-) diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index e3c7f6ef6d..f6de6d8492 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -145,9 +145,9 @@ A cache build boots and configures a disposable VM, runs the internal sealing sc The key includes the pinned base-image identity, guest architecture, ordered configuration file digests, Ansible version, cache generation, and sealing script digest. Installed packages, copied binaries, forwarded ports, and guest commands are never cached. -Normal `test-guest` runs automatically use an exact valid local entry and create a fresh writable overlay, cloud-init instance, machine ID, and SSH identity. On a local miss, the runner uses the pinned cloud image and applies configurations normally. Set `OPENSHELL_TEST_VM_CACHE_DISABLE=1` to bypass local lookup. +Normal `test-guest` runs automatically use an exact valid local entry and create a fresh writable overlay, cloud-init instance, machine ID, and SSH identity. On a local miss, the runner uses the pinned cloud image and applies configurations normally. Set `OPENSHELL_TEST_GUEST_CACHE_DISABLE=1` to bypass local lookup. -The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-guest`. Override it with `--cache-dir` on the cache command or `OPENSHELL_TEST_VM_CACHE_DIR` for either app. +The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-guest`. Override it with `--cache-dir` on the cache command or `OPENSHELL_TEST_GUEST_CACHE_DIR` for either app. Cache command options: diff --git a/nix/test-guest/cache-lib.sh b/nix/test-guest/cache-lib.sh index 5b37f91893..1e6f76b191 100644 --- a/nix/test-guest/cache-lib.sh +++ b/nix/test-guest/cache-lib.sh @@ -4,26 +4,26 @@ # Shared cache identity and validation helpers for the test guest runner. -TEST_VM_CACHE_SCHEMA_VERSION=1 -TEST_VM_CACHE_DISK_LAYOUT=standalone-qcow2-zstd-v1 -TEST_VM_CACHE_ARTIFACT_TYPE=application/vnd.nvidia.openshell.test-guest.cache.v1 -TEST_VM_CACHE_METADATA_TYPE=application/vnd.nvidia.openshell.test-guest.cache.metadata.v1+json -TEST_VM_CACHE_DISK_TYPE=application/vnd.nvidia.openshell.test-guest.cache.disk.qcow2.v1+zstd +TEST_GUEST_CACHE_SCHEMA_VERSION=1 +TEST_GUEST_CACHE_DISK_LAYOUT=standalone-qcow2-zstd-v1 +TEST_GUEST_CACHE_ARTIFACT_TYPE=application/vnd.nvidia.openshell.test-guest.cache.v1 +TEST_GUEST_CACHE_METADATA_TYPE=application/vnd.nvidia.openshell.test-guest.cache.metadata.v1+json +TEST_GUEST_CACHE_DISK_TYPE=application/vnd.nvidia.openshell.test-guest.cache.disk.qcow2.v1+zstd test_vm_cache_root() { - if [ -n "${OPENSHELL_TEST_VM_CACHE_DIR:-}" ]; then - printf '%s\n' "${OPENSHELL_TEST_VM_CACHE_DIR}" + if [ -n "${OPENSHELL_TEST_GUEST_CACHE_DIR:-}" ]; then + printf '%s\n' "${OPENSHELL_TEST_GUEST_CACHE_DIR}" else printf '%s\n' "${XDG_CACHE_HOME:-${HOME}/.cache}/openshell/test-guest" fi } test_vm_cache_oci_architecture() { - case "${TEST_VM_ARCHITECTURE}" in + case "${TEST_GUEST_ARCHITECTURE}" in x86_64) printf '%s\n' amd64 ;; aarch64) printf '%s\n' arm64 ;; *) - echo "unsupported cache architecture: ${TEST_VM_ARCHITECTURE}" >&2 + echo "unsupported cache architecture: ${TEST_GUEST_ARCHITECTURE}" >&2 return 1 ;; esac @@ -48,25 +48,25 @@ test_vm_cache_key() { local configuration_line architecture=$(test_vm_cache_oci_architecture) || return 1 - seal_hash=$(test_vm_cache_sha256 "${OPENSHELL_TEST_VM_CACHE_SEAL}") || return 1 + seal_hash=$(test_vm_cache_sha256 "${OPENSHELL_TEST_GUEST_CACHE_SEAL}") || return 1 printf -v material \ 'schema=%s\ngeneration=%s\ndisk_layout=%s\ndistro=%s\nos_version=%s\narchitecture=%s\nbase_url=%s\nbase_hash=%s\noverlay_growth=%s\nansible_version=%s\nseal_sha256=%s\n' \ - "${TEST_VM_CACHE_SCHEMA_VERSION}" \ - "${TEST_VM_CACHE_GENERATION}" \ - "${TEST_VM_CACHE_DISK_LAYOUT}" \ + "${TEST_GUEST_CACHE_SCHEMA_VERSION}" \ + "${TEST_GUEST_CACHE_GENERATION}" \ + "${TEST_GUEST_CACHE_DISK_LAYOUT}" \ "${distro_name}" \ - "${TEST_VM_OS_VERSION}" \ + "${TEST_GUEST_OS_VERSION}" \ "${architecture}" \ - "${TEST_VM_IMAGE_URL}" \ - "${TEST_VM_IMAGE_HASH}" \ + "${TEST_GUEST_IMAGE_URL}" \ + "${TEST_GUEST_IMAGE_HASH}" \ 16G \ - "${TEST_VM_ANSIBLE_VERSION}" \ + "${TEST_GUEST_ANSIBLE_VERSION}" \ "${seal_hash}" for configuration in "$@"; do configuration_hash=$( test_vm_cache_sha256 \ - "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${configuration}" + "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${configuration}" ) || return 1 printf -v configuration_line \ 'configuration=%s:%s\n' \ @@ -82,7 +82,7 @@ test_vm_cache_tag() { local distro_name=$1 local key=$2 printf 'v%s-%s-%s-%s\n' \ - "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + "${TEST_GUEST_CACHE_SCHEMA_VERSION}" \ "${distro_name}" \ "$(test_vm_cache_oci_architecture)" \ "${key}" @@ -105,12 +105,12 @@ test_vm_cache_metadata_matches() { expected_configurations=$(jq -cn --args '$ARGS.positional' "$@") || return 1 jq -e \ - --argjson schema "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + --argjson schema "${TEST_GUEST_CACHE_SCHEMA_VERSION}" \ --arg key "${key}" \ --arg distro "${distro_name}" \ - --arg os_version "${TEST_VM_OS_VERSION}" \ + --arg os_version "${TEST_GUEST_OS_VERSION}" \ --arg architecture "$(test_vm_cache_oci_architecture)" \ - --arg base_hash "${TEST_VM_IMAGE_HASH}" \ + --arg base_hash "${TEST_GUEST_IMAGE_HASH}" \ --argjson configurations "${expected_configurations}" \ ' .schema == $schema and diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index b986077e80..b94ecd7bd1 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -26,18 +26,18 @@ Publishing is explicit and requires both --repository and --push. EOF } -if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || - [ ! -d "${OPENSHELL_TEST_VM_DISTROS:-}" ] || - [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ] || - [ ! -r "${OPENSHELL_TEST_VM_CACHE_LIB:-}" ] || - [ ! -r "${OPENSHELL_TEST_VM_CACHE_SEAL:-}" ] || - [ ! -r "${OPENSHELL_TEST_VM_RUNNER:-}" ]; then +if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ] || + [ ! -d "${OPENSHELL_TEST_GUEST_DISTROS:-}" ] || + [ ! -d "${OPENSHELL_TEST_GUEST_CONFIGURATIONS:-}" ] || + [ ! -r "${OPENSHELL_TEST_GUEST_CACHE_LIB:-}" ] || + [ ! -r "${OPENSHELL_TEST_GUEST_CACHE_SEAL:-}" ] || + [ ! -r "${OPENSHELL_TEST_GUEST_RUNNER:-}" ]; then echo "run this script through 'nix run .#test-guest-cache -- ...'" >&2 exit 2 fi # shellcheck disable=SC1090 -. "${OPENSHELL_TEST_VM_CACHE_LIB}" +. "${OPENSHELL_TEST_GUEST_CACHE_LIB}" require_value() { if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then @@ -47,7 +47,7 @@ require_value() { } distro= -repository=${OPENSHELL_TEST_VM_CACHE_REPOSITORY:-} +repository=${OPENSHELL_TEST_GUEST_CACHE_REPOSITORY:-} cache_dir= push=0 configurations=() @@ -96,18 +96,18 @@ if [ -z "${distro}" ]; then exit 2 fi if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || - [ ! -r "${OPENSHELL_TEST_VM_DISTROS}/${distro}" ]; then + [ ! -r "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" ]; then echo "unknown distro: ${distro}" >&2 exit 2 fi # Distro profiles contain only trusted values generated into the Nix store. # shellcheck disable=SC1090 -. "${OPENSHELL_TEST_VM_DISTROS}/${distro}" +. "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" for item in "${configurations[@]}"; do if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] || - [ ! -r "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" ]; then + [ ! -r "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${item}" ]; then echo "unknown configuration: ${item:-}" >&2 exit 2 fi @@ -123,8 +123,8 @@ if [[ ${repository} == *[[:space:]]* ]] || [[ ${repository} == *@* ]]; then fi if [ -n "${cache_dir}" ]; then - OPENSHELL_TEST_VM_CACHE_DIR=${cache_dir} - export OPENSHELL_TEST_VM_CACHE_DIR + OPENSHELL_TEST_GUEST_CACHE_DIR=${cache_dir} + export OPENSHELL_TEST_GUEST_CACHE_DIR fi umask 077 @@ -288,7 +288,7 @@ build_local() { done prepare_args+=( --copy - "${OPENSHELL_TEST_VM_CACHE_SEAL}:/usr/local/sbin/openshell-test-guest-cache-seal" + "${OPENSHELL_TEST_GUEST_CACHE_SEAL}:/usr/local/sbin/openshell-test-guest-cache-seal" -- sudo /usr/local/sbin/openshell-test-guest-cache-seal @@ -296,7 +296,7 @@ build_local() { echo "==> Cache miss: preparing ${distro} ($(test_vm_cache_oci_architecture))" TMPDIR="${build_stage}/prepare-tmp" \ - "${TEST_VM_BASH}" "${OPENSHELL_TEST_VM_RUNNER}" "${prepare_args[@]}" + "${TEST_GUEST_BASH}" "${OPENSHELL_TEST_GUEST_RUNNER}" "${prepare_args[@]}" shopt -s nullglob run_dirs=("${build_stage}/prepare-tmp/openshell-test-guest"/run.*) @@ -329,8 +329,8 @@ build_local() { echo "==> Validating fresh boot from prepared cache disk" TMPDIR="${build_stage}/validate-tmp" \ - OPENSHELL_TEST_VM_IMAGE_OVERRIDE="${prepared_disk}" \ - "${TEST_VM_BASH}" "${OPENSHELL_TEST_VM_RUNNER}" "${validate_args[@]}" + OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE="${prepared_disk}" \ + "${TEST_GUEST_BASH}" "${OPENSHELL_TEST_GUEST_RUNNER}" "${validate_args[@]}" configuration_json=$(jq -cn --args '$ARGS.positional' "${configurations[@]}") disk_sha=$(test_vm_cache_sha256 "${prepared_disk}") @@ -341,14 +341,14 @@ build_local() { created=$(date -u +%Y-%m-%dT%H:%M:%SZ) jq -n \ - --argjson schema "${TEST_VM_CACHE_SCHEMA_VERSION}" \ + --argjson schema "${TEST_GUEST_CACHE_SCHEMA_VERSION}" \ --arg key "${cache_key}" \ --arg distro "${distro}" \ - --arg os_version "${TEST_VM_OS_VERSION}" \ + --arg os_version "${TEST_GUEST_OS_VERSION}" \ --arg architecture "$(test_vm_cache_oci_architecture)" \ - --arg base_image_url "${TEST_VM_IMAGE_URL}" \ - --arg base_image_hash "${TEST_VM_IMAGE_HASH}" \ - --arg disk_layout "${TEST_VM_CACHE_DISK_LAYOUT}" \ + --arg base_image_url "${TEST_GUEST_IMAGE_URL}" \ + --arg base_image_hash "${TEST_GUEST_IMAGE_HASH}" \ + --arg disk_layout "${TEST_GUEST_CACHE_DISK_LAYOUT}" \ --arg disk_sha256 "${disk_sha}" \ --arg created "${created}" \ --argjson virtual_size "${virtual_size}" \ @@ -400,10 +400,10 @@ push_remote() { ( cd "${push_dir}" oras push \ - --artifact-type "${TEST_VM_CACHE_ARTIFACT_TYPE}" \ + --artifact-type "${TEST_GUEST_CACHE_ARTIFACT_TYPE}" \ "${remote_ref}" \ - "metadata.json:${TEST_VM_CACHE_METADATA_TYPE}" \ - "disk.qcow2.zst:${TEST_VM_CACHE_DISK_TYPE}" + "metadata.json:${TEST_GUEST_CACHE_METADATA_TYPE}" \ + "disk.qcow2.zst:${TEST_GUEST_CACHE_DISK_TYPE}" ) rm -rf "${push_dir}" echo "==> Cache push complete: ${remote_ref}" diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index 9863f6d767..f34fb062aa 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -29,14 +29,14 @@ let mkDistroProfile = name: distro: pkgs.writeText "openshell-test-guest-${name}" '' - TEST_VM_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath} - TEST_VM_IMAGE_URL=${pkgs.lib.escapeShellArg distro.imageUrl} - TEST_VM_IMAGE_HASH=${pkgs.lib.escapeShellArg distro.imageHash} - TEST_VM_OS_ID=${pkgs.lib.escapeShellArg distro.osId} - TEST_VM_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion} - TEST_VM_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily} - export TEST_VM_IMAGE_DRV TEST_VM_IMAGE_URL TEST_VM_IMAGE_HASH - export TEST_VM_OS_ID TEST_VM_OS_VERSION TEST_VM_PACKAGE_FAMILY + TEST_GUEST_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath} + TEST_GUEST_IMAGE_URL=${pkgs.lib.escapeShellArg distro.imageUrl} + TEST_GUEST_IMAGE_HASH=${pkgs.lib.escapeShellArg distro.imageHash} + TEST_GUEST_OS_ID=${pkgs.lib.escapeShellArg distro.osId} + TEST_GUEST_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion} + TEST_GUEST_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily} + export TEST_GUEST_IMAGE_DRV TEST_GUEST_IMAGE_URL TEST_GUEST_IMAGE_HASH + export TEST_GUEST_OS_ID TEST_GUEST_OS_VERSION TEST_GUEST_PACKAGE_FAMILY ''; distroCatalog = pkgs.linkFarm "openshell-test-guest-distros" ( @@ -65,21 +65,21 @@ let ]; runtimeEnvironment = '' - export OPENSHELL_TEST_VM_RUNTIME=1 - export OPENSHELL_TEST_VM_DISTROS=${distroCatalog} - export OPENSHELL_TEST_VM_CONFIGURATIONS=${configurationCatalog} - export OPENSHELL_TEST_VM_CACHE_LIB=${./cache-lib.sh} - export OPENSHELL_TEST_VM_CACHE_SEAL=${./cache-seal.sh} - export OPENSHELL_TEST_VM_RUNNER=${./run.sh} - export TEST_VM_BASH=${pkgs.bash}/bin/bash - export TEST_VM_QEMU=${qemuBinary} - export TEST_VM_FIRMWARE_CODE=${pkgs.OVMF.firmware} - export TEST_VM_FIRMWARE_VARS=${pkgs.OVMF.variables} - export TEST_VM_MACHINE=${if isAarch64 then "virt" else "q35"} - export TEST_VM_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} - export TEST_VM_ARCHITECTURE=${architecture} - export TEST_VM_ANSIBLE_VERSION=${pkgs.python3Packages.ansible-core.version} - export TEST_VM_CACHE_GENERATION=1 + export OPENSHELL_TEST_GUEST_RUNTIME=1 + export OPENSHELL_TEST_GUEST_DISTROS=${distroCatalog} + export OPENSHELL_TEST_GUEST_CONFIGURATIONS=${configurationCatalog} + export OPENSHELL_TEST_GUEST_CACHE_LIB=${./cache-lib.sh} + export OPENSHELL_TEST_GUEST_CACHE_SEAL=${./cache-seal.sh} + export OPENSHELL_TEST_GUEST_RUNNER=${./run.sh} + export TEST_GUEST_BASH=${pkgs.bash}/bin/bash + export TEST_GUEST_QEMU=${qemuBinary} + export TEST_GUEST_FIRMWARE_CODE=${pkgs.OVMF.firmware} + export TEST_GUEST_FIRMWARE_VARS=${pkgs.OVMF.variables} + export TEST_GUEST_MACHINE=${if isAarch64 then "virt" else "q35"} + export TEST_GUEST_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + export TEST_GUEST_ARCHITECTURE=${architecture} + export TEST_GUEST_ANSIBLE_VERSION=${pkgs.python3Packages.ansible-core.version} + export TEST_GUEST_CACHE_GENERATION=1 ''; runner = pkgs.writeShellApplication { diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index bf92e639ac..07d27abe8a 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -25,10 +25,10 @@ With no COMMAND, the runner opens an interactive SSH session. EOF } -if [ "${OPENSHELL_TEST_VM_RUNTIME:-}" != 1 ] || - [ ! -d "${OPENSHELL_TEST_VM_DISTROS:-}" ] || - [ ! -d "${OPENSHELL_TEST_VM_CONFIGURATIONS:-}" ] || - [ ! -r "${OPENSHELL_TEST_VM_CACHE_LIB:-}" ]; then +if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ] || + [ ! -d "${OPENSHELL_TEST_GUEST_DISTROS:-}" ] || + [ ! -d "${OPENSHELL_TEST_GUEST_CONFIGURATIONS:-}" ] || + [ ! -r "${OPENSHELL_TEST_GUEST_CACHE_LIB:-}" ]; then echo "run this script through 'nix run .#test-guest -- ...'" >&2 exit 2 fi @@ -91,11 +91,11 @@ done if [ "${list}" -eq 1 ]; then echo "Distros:" - for entry in "${OPENSHELL_TEST_VM_DISTROS}"/*; do + for entry in "${OPENSHELL_TEST_GUEST_DISTROS}"/*; do printf ' %s\n' "${entry##*/}" done echo "Configurations:" - for entry in "${OPENSHELL_TEST_VM_CONFIGURATIONS}"/*; do + for entry in "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}"/*; do printf ' %s\n' "${entry##*/}" done exit 0 @@ -107,17 +107,17 @@ if [ -z "${distro}" ]; then exit 2 fi if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] || - [ ! -r "${OPENSHELL_TEST_VM_DISTROS}/${distro}" ]; then + [ ! -r "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" ]; then echo "unknown distro: ${distro}" >&2 exit 2 fi # Distro profiles contain only trusted values generated into the Nix store. # shellcheck disable=SC1090 -. "${OPENSHELL_TEST_VM_DISTROS}/${distro}" +. "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" for item in "${configurations[@]}"; do if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] || - [ ! -r "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" ]; then + [ ! -r "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${item}" ]; then echo "unknown configuration: ${item:-}" >&2 exit 2 fi @@ -137,10 +137,10 @@ for package in "${packages[@]}"; do echo "package does not exist: ${package}" >&2 exit 2 fi - case "${TEST_VM_PACKAGE_FAMILY}:${package}" in + case "${TEST_GUEST_PACKAGE_FAMILY}:${package}" in deb:*.deb | rpm:*.rpm) ;; *) - echo "${package} does not match the ${TEST_VM_PACKAGE_FAMILY} package family" >&2 + echo "${package} does not match the ${TEST_GUEST_PACKAGE_FAMILY} package family" >&2 exit 2 ;; esac @@ -169,34 +169,34 @@ done test_vm_cpu=host ssh_wait_seconds=180 -if [ "${TEST_VM_ACCELERATOR}" = kvm ] && +if [ "${TEST_GUEST_ACCELERATOR}" = kvm ] && { [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; }; then echo "==> /dev/kvm is unavailable; falling back to QEMU/TCG" - TEST_VM_ACCELERATOR=tcg + TEST_GUEST_ACCELERATOR=tcg test_vm_cpu=max ssh_wait_seconds=600 fi # shellcheck disable=SC1090 -. "${OPENSHELL_TEST_VM_CACHE_LIB}" +. "${OPENSHELL_TEST_GUEST_CACHE_LIB}" prepared_image=0 -TEST_VM_IMAGE= -if [ -n "${OPENSHELL_TEST_VM_IMAGE_OVERRIDE:-}" ]; then - if [ ! -f "${OPENSHELL_TEST_VM_IMAGE_OVERRIDE}" ]; then - echo "prepared guest image does not exist: ${OPENSHELL_TEST_VM_IMAGE_OVERRIDE}" >&2 +TEST_GUEST_IMAGE= +if [ -n "${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE:-}" ]; then + if [ ! -f "${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE}" ]; then + echo "prepared guest image does not exist: ${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE}" >&2 exit 2 fi - TEST_VM_IMAGE=${OPENSHELL_TEST_VM_IMAGE_OVERRIDE} + TEST_GUEST_IMAGE=${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE} prepared_image=1 echo "==> Using explicit prepared guest image" -elif [ "${OPENSHELL_TEST_VM_CACHE_DISABLE:-0}" -ne 1 ]; then +elif [ "${OPENSHELL_TEST_GUEST_CACHE_DISABLE:-0}" -ne 1 ]; then cache_root=$(test_vm_cache_root) cache_key=$(test_vm_cache_key "${distro}" "${configurations[@]}") if test_vm_cache_local_entry_valid \ "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then cache_entry=$(test_vm_cache_entry_dir "${cache_root}" "${cache_key}") - TEST_VM_IMAGE="${cache_entry}/disk.qcow2" + TEST_GUEST_IMAGE="${cache_entry}/disk.qcow2" prepared_image=1 echo "==> Cache local hit: ${cache_entry}" fi @@ -204,7 +204,7 @@ fi if [ "${prepared_image}" -eq 0 ]; then echo "==> Realizing the pinned ${distro} cloud image" - TEST_VM_IMAGE=$(nix build --no-link --print-out-paths "${TEST_VM_IMAGE_DRV}^out") + TEST_GUEST_IMAGE=$(nix build --no-link --print-out-paths "${TEST_GUEST_IMAGE_DRV}^out") fi umask 077 @@ -215,6 +215,7 @@ overlay=${run_dir}/disk.qcow2 seed=${run_dir}/seed.iso vars=${run_dir}/firmware-vars.fd private_key=${run_dir}/id_ed25519 +ssh_control_path=${run_dir}/ssh-control serial_log=${run_dir}/serial.log qemu_log=${run_dir}/qemu.log qemu_pid= @@ -285,29 +286,30 @@ EOF xorriso -as mkisofs -quiet -output "${seed}" -volid cidata -joliet -rock user-data meta-data ) -qemu-img create -q -f qcow2 -F qcow2 -b "${TEST_VM_IMAGE}" "${overlay}" +qemu-img create -q -f qcow2 -F qcow2 -b "${TEST_GUEST_IMAGE}" "${overlay}" if [ "${prepared_image}" -eq 0 ]; then qemu-img resize -q "${overlay}" +16G fi -cp "${TEST_VM_FIRMWARE_VARS}" "${vars}" +cp "${TEST_GUEST_FIRMWARE_VARS}" "${vars}" chmod 0600 "${vars}" for attempt in $(seq 1 5); do ssh_port=${requested_ssh_port:-$(pick_free_port)} : >"${qemu_log}" - echo "==> Booting ${distro} (${TEST_VM_ARCHITECTURE}) with QEMU/${TEST_VM_ACCELERATOR}" - "${TEST_VM_QEMU}" \ + echo "==> Booting ${distro} (${TEST_GUEST_ARCHITECTURE}) with QEMU/${TEST_GUEST_ACCELERATOR}" + "${TEST_GUEST_QEMU}" \ -name "openshell-test-${distro}" \ - -machine "${TEST_VM_MACHINE},accel=${TEST_VM_ACCELERATOR}" \ + -machine "${TEST_GUEST_MACHINE},accel=${TEST_GUEST_ACCELERATOR}" \ -cpu "${test_vm_cpu}" \ -smp 4 \ -m 4096 \ - -drive "if=pflash,format=raw,readonly=on,file=${TEST_VM_FIRMWARE_CODE}" \ + -drive "if=pflash,format=raw,readonly=on,file=${TEST_GUEST_FIRMWARE_CODE}" \ -drive "if=pflash,format=raw,file=${vars}" \ -drive "if=none,format=qcow2,file=${overlay},id=osdisk" \ - -device virtio-blk-pci,drive=osdisk \ + -device virtio-blk-pci,drive=osdisk,bootindex=1 \ -drive "if=none,format=raw,readonly=on,file=${seed},id=seed" \ -device virtio-blk-pci,drive=seed \ + -boot strict=on \ -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ -device virtio-net-pci,netdev=net0 \ -display none \ @@ -317,7 +319,7 @@ for attempt in $(seq 1 5); do >/dev/null 2>"${qemu_log}" & qemu_pid=$! - sleep 1 + sleep 0.25 if kill -0 "${qemu_pid}" 2>/dev/null; then break fi @@ -342,6 +344,9 @@ ssh_args=( -p "${ssh_port}" -o BatchMode=yes -o ConnectTimeout=5 + -o ControlMaster=auto + -o ControlPersist=60 + -o "ControlPath=${ssh_control_path}" -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @@ -352,6 +357,9 @@ scp_args=( -P "${ssh_port}" -o BatchMode=yes -o ConnectTimeout=5 + -o ControlMaster=auto + -o ControlPersist=60 + -o "ControlPath=${ssh_control_path}" -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @@ -359,18 +367,18 @@ scp_args=( echo "==> Waiting up to ${ssh_wait_seconds} seconds for SSH on 127.0.0.1:${ssh_port}" ssh_ready=0 -for _ in $(seq 1 "$((ssh_wait_seconds / 2))"); do +for _ in $(seq 1 "$((ssh_wait_seconds * 4))"); do if ! kill -0 "${qemu_pid}" 2>/dev/null; then wait "${qemu_pid}" || true qemu_pid= echo "QEMU exited before SSH became ready" >&2 exit 1 fi - if ssh "${ssh_args[@]}" openshell@127.0.0.1 true 2>/dev/null; then + if ssh "${ssh_args[@]}" -o ConnectTimeout=1 openshell@127.0.0.1 true 2>/dev/null; then ssh_ready=1 break fi - sleep 2 + sleep 0.25 done if [ "${ssh_ready}" -ne 1 ]; then echo "SSH did not become ready within ${ssh_wait_seconds} seconds" >&2 @@ -381,12 +389,10 @@ echo "==> Validating ${distro}" # Profile values come from the trusted Nix-generated catalog. # shellcheck disable=SC2029 ssh "${ssh_args[@]}" openshell@127.0.0.1 \ - "set -eu; . /etc/os-release; test \"\${ID}\" = '${TEST_VM_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_VM_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_VM_ARCHITECTURE}'" + "set -eu; set +e; sudo cloud-init status --wait >/dev/null; status=\$?; set -e; [ \"\${status}\" -eq 0 ] || [ \"\${status}\" -eq 2 ]; . /etc/os-release; test \"\${ID}\" = '${TEST_GUEST_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_GUEST_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_GUEST_ARCHITECTURE}'" # cloud-init returns 2 when it completes with recoverable errors. Fedora can # report that status for an initial transient-hostname warning even though the # requested user and SSH configuration were applied successfully. -ssh "${ssh_args[@]}" openshell@127.0.0.1 \ - 'sudo cloud-init status --wait >/dev/null; status=$?; [ "$status" -eq 0 ] || [ "$status" -eq 2 ]' cat >"${ansible_config}" < Applying configuration: ${item}" ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \ - ansible-playbook "${OPENSHELL_TEST_VM_CONFIGURATIONS}/${item}" + ansible-playbook "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${item}" done else echo "==> Reusing cached configuration: ${configurations[*]:-base image}" @@ -422,7 +428,7 @@ for package in "${packages[@]}"; do remote_package="/home/openshell/${package_name}" echo "==> Installing package: ${package_name}" scp "${scp_args[@]}" "${package}" "openshell@127.0.0.1:${remote_package}" - case "${TEST_VM_PACKAGE_FAMILY}" in + case "${TEST_GUEST_PACKAGE_FAMILY}" in deb) # The remote path uses a restricted package filename. # shellcheck disable=SC2029 @@ -466,12 +472,18 @@ else fi echo "==> Shutting down ${distro}" -ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo systemctl poweroff' >/dev/null 2>&1 || true -for _ in $(seq 1 30); do - if ! kill -0 "${qemu_pid}" 2>/dev/null; then - wait "${qemu_pid}" || true - qemu_pid= - break - fi - sleep 1 -done +if [ "${keep}" -eq 1 ]; then + ssh "${ssh_args[@]}" openshell@127.0.0.1 'sudo systemctl poweroff' >/dev/null 2>&1 || true + for _ in $(seq 1 120); do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + wait "${qemu_pid}" || true + qemu_pid= + break + fi + sleep 0.25 + done +else + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + qemu_pid= +fi From 49ae902c28f3c514535434e65bc4dc2c640be5da Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 13:08:20 -0700 Subject: [PATCH 08/29] fix(test-guest): address review feedback Signed-off-by: Drew Newberry --- nix/test-guest/configuration/docker.yml | 13 +++---------- nix/test-guest/run.sh | 6 +++++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/nix/test-guest/configuration/docker.yml b/nix/test-guest/configuration/docker.yml index 612c9e1c81..3a4ae53b07 100644 --- a/nix/test-guest/configuration/docker.yml +++ b/nix/test-guest/configuration/docker.yml @@ -62,19 +62,12 @@ append: true register: docker_group_membership - - name: Read the test user ID - ansible.builtin.command: - cmd: id -u openshell - register: test_user_id - changed_when: false - - - name: Refresh the test user's systemd group membership - ansible.builtin.systemd_service: - name: "user@{{ test_user_id.stdout }}.service" - state: restarted + - name: Refresh the test user's group membership + ansible.builtin.meta: reset_connection when: docker_group_membership.changed - name: Verify Docker ansible.builtin.command: cmd: docker info + become: false changed_when: false diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 07d27abe8a..677f54ae34 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -159,6 +159,10 @@ for copy_spec in "${copies[@]}"; do echo "--copy destination must not contain '..': ${destination}" >&2 exit 2 fi + if [[ ! ${destination} =~ ^/[A-Za-z0-9._+~/-]+$ ]]; then + echo "--copy destination contains unsupported characters: ${destination}" >&2 + exit 2 + fi ;; *) echo "--copy destination must be absolute: ${destination}" >&2 @@ -455,7 +459,7 @@ for copy_spec in "${copies[@]}"; do remote_copy="/home/openshell/${copy_name}" echo "==> Copying ${copy_name} to ${destination}" scp "${scp_args[@]}" "${source_path}" "openshell@127.0.0.1:${remote_copy}" - # The destination was restricted to an absolute, traversal-free path. + # The remote path and destination use restricted path characters. # shellcheck disable=SC2029 ssh "${ssh_args[@]}" openshell@127.0.0.1 \ "sudo install -D -m 0755 '${remote_copy}' '${destination}'; rm -f '${remote_copy}'" From 9f240044655387561c88b4b305e8770f88042765 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 13:31:10 -0700 Subject: [PATCH 09/29] fix(test-guest): verify OCI cache provenance Signed-off-by: Drew Newberry --- architecture/build.md | 4 ++- nix/test-guest/README.md | 12 +++++-- nix/test-guest/cache.sh | 76 ++++++++++++++++++++++++++++++++++------ 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/architecture/build.md b/architecture/build.md index 6ca5c3d59e..c64ecf7c39 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -147,7 +147,9 @@ read-only and each test receives a fresh writable overlay and cloud-init identity. The optional shared cache stores the compressed standalone disk and its compatibility metadata as a custom OCI artifact. Normal test runs consume local entries only; the separate cache app owns OCI pulls, trusted builds, and -explicit publication. +explicit publication. OCI pulls require a trusted manifest digest and retain +that provenance with the local entry; mutable tags are used only for explicit +publication. ## Python Wheel Packaging diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index f6de6d8492..20cdc1a8be 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -122,13 +122,15 @@ nix run .#test-guest-cache -- \ --with docker ``` -Configure an OCI repository to use it as a shared backing cache: +Configure an OCI repository and a trusted manifest digest to use it as a shared +backing cache: ```shell nix run .#test-guest-cache -- \ --distro ubuntu \ --with docker \ - --repository ghcr.io/nvidia/openshell/test-guest-cache + --repository ghcr.io/nvidia/openshell/test-guest-cache \ + --digest sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef ``` The command never publishes implicitly. Add `--push` after authenticating ORAS through its Docker-compatible credential configuration: @@ -141,6 +143,11 @@ nix run .#test-guest-cache -- \ --push ``` +A successful push prints the immutable `repository@sha256:...` reference. Supply +that digest to consumers through trusted CI configuration. Pulls by mutable tag +are not allowed. A pulled local entry records its manifest digest and is reused +only when it matches the requested trusted digest. + A cache build boots and configures a disposable VM, runs the internal sealing script, flattens the overlay into a standalone QCOW2 disk, and validates a fresh boot before committing the entry. The OCI artifact contains metadata and a `disk.qcow2.zst` layer. The key includes the pinned base-image identity, guest architecture, ordered configuration file digests, Ansible version, cache generation, and sealing script digest. Installed packages, copied binaries, forwarded ports, and guest commands are never cached. @@ -155,6 +162,7 @@ Cache command options: --distro NAME Base distro: ubuntu, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --repository REF OCI repository without a tag +--digest DIGEST Trusted OCI manifest digest required for pulls --cache-dir PATH Override the local prepared-disk cache directory --push Publish the ensured entry to the repository ``` diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index b94ecd7bd1..4943a028c8 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -15,13 +15,14 @@ Options: --distro NAME Base distro: ubuntu, centos, fedora, or rocky --with NAME Apply a configuration; repeatable (docker, podman, selinux) --repository REF OCI repository without a tag + --digest DIGEST Trusted OCI manifest digest required for pulls --cache-dir PATH Override the local prepared-disk cache directory --push Publish a newly built or local entry to the repository -h, --help Show this help The command ensures one exact distro, architecture, and ordered configuration -combination exists locally. It checks the local cache, pulls from OCI when a -repository is configured, or builds and validates a prepared disk on a miss. +combination exists locally. OCI pulls require a trusted manifest digest. +Otherwise, the command builds and validates a prepared disk on a miss. Publishing is explicit and requires both --repository and --push. EOF } @@ -48,6 +49,7 @@ require_value() { distro= repository=${OPENSHELL_TEST_GUEST_CACHE_REPOSITORY:-} +pull_digest=${OPENSHELL_TEST_GUEST_CACHE_DIGEST:-} cache_dir= push=0 configurations=() @@ -69,6 +71,11 @@ while [ "$#" -gt 0 ]; do repository=$2 shift 2 ;; + --digest) + require_value "$@" + pull_digest=$2 + shift 2 + ;; --cache-dir) require_value "$@" cache_dir=$2 @@ -121,6 +128,19 @@ if [[ ${repository} == *[[:space:]]* ]] || [[ ${repository} == *@* ]]; then echo "--repository must be an untagged OCI repository reference" >&2 exit 2 fi +if [ -n "${repository}" ] && [ "${push}" -eq 0 ] && [ -z "${pull_digest}" ]; then + echo "--repository requires --digest for OCI pulls" >&2 + exit 2 +fi +if [ -n "${pull_digest}" ] && [ -z "${repository}" ]; then + echo "--digest requires --repository" >&2 + exit 2 +fi +if [ -n "${pull_digest}" ] && + [[ ! ${pull_digest} =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "--digest must be a lowercase sha256 OCI manifest digest" >&2 + exit 2 +fi if [ -n "${cache_dir}" ]; then OPENSHELL_TEST_GUEST_CACHE_DIR=${cache_dir} @@ -133,9 +153,13 @@ cache_key=$(test_vm_cache_key "${distro}" "${configurations[@]}") cache_tag=$(test_vm_cache_tag "${distro}" "${cache_key}") entry_dir=$(test_vm_cache_entry_dir "${cache_root}" "${cache_key}") remote_ref= +pull_ref= if [ -n "${repository}" ]; then remote_ref="${repository}:${cache_tag}" fi +if [ -n "${pull_digest}" ]; then + pull_ref="${repository}@${pull_digest}" +fi mkdir -p "${cache_root}/entries" "${cache_root}/locks" "${cache_root}/staging" @@ -177,17 +201,21 @@ fi install_entry() { local disk=$1 local metadata=$2 + local manifest_digest=${3:-} local temporary_entry temporary_entry=$(mktemp -d "${cache_root}/entries/.${cache_key}.XXXXXX") cp --sparse=always "${disk}" "${temporary_entry}/disk.qcow2" chmod 0444 "${temporary_entry}/disk.qcow2" install -m 0444 "${metadata}" "${temporary_entry}/metadata.json" + if [ -n "${manifest_digest}" ]; then + printf '%s\n' "${manifest_digest}" >"${temporary_entry}/manifest.digest" + chmod 0444 "${temporary_entry}/manifest.digest" + fi : >"${temporary_entry}/complete" chmod 0444 "${temporary_entry}/complete" if [ -e "${entry_dir}" ]; then - if test_vm_cache_local_entry_valid \ - "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then + if local_entry_valid; then rm -rf "${temporary_entry}" return fi @@ -198,6 +226,16 @@ install_entry() { mv "${temporary_entry}" "${entry_dir}" } +local_entry_valid() { + test_vm_cache_local_entry_valid \ + "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}" || + return 1 + if [ -n "${pull_digest}" ]; then + [ -f "${entry_dir}/manifest.digest" ] && + [ "$(<"${entry_dir}/manifest.digest")" = "${pull_digest}" ] + fi +} + is_remote_miss() { grep -Eqi '404|MANIFEST_UNKNOWN|manifest unknown|not found' "$1" } @@ -208,10 +246,21 @@ pull_remote() { local compressed_size local expected_sha local actual_sha + local resolved_digest pull_dir=$(mktemp -d "${cache_root}/staging/pull.${cache_key}.XXXXXX") pull_log="${pull_dir}/oras.log" - if ! oras pull --output "${pull_dir}" "${remote_ref}" >"${pull_log}" 2>&1; then + resolved_digest=$(oras resolve "${pull_ref}") || { + rm -rf "${pull_dir}" + return 2 + } + if [ "${resolved_digest}" != "${pull_digest}" ]; then + echo "OCI cache manifest digest does not match trusted digest" >&2 + rm -rf "${pull_dir}" + return 2 + fi + + if ! oras pull --output "${pull_dir}" "${pull_ref}" >"${pull_log}" 2>&1; then if is_remote_miss "${pull_log}"; then rm -rf "${pull_dir}" return 1 @@ -261,9 +310,10 @@ pull_remote() { return 2 fi - install_entry "${pull_dir}/disk.qcow2" "${pull_dir}/metadata.json" + install_entry \ + "${pull_dir}/disk.qcow2" "${pull_dir}/metadata.json" "${resolved_digest}" rm -rf "${pull_dir}" - echo "==> Cache remote hit: ${remote_ref}" + echo "==> Cache remote hit: ${pull_ref}" } build_local() { @@ -378,11 +428,14 @@ build_local() { push_remote() { local push_dir local manifest_log + local published_digest push_dir=$(mktemp -d "${cache_root}/staging/push.${cache_key}.XXXXXX") manifest_log="${push_dir}/manifest.log" if oras manifest fetch "${remote_ref}" >"${manifest_log}" 2>&1; then echo "==> Cache already published: ${remote_ref}" + published_digest=$(oras resolve "${remote_ref}") + echo "==> Cache immutable reference: ${repository}@${published_digest}" rm -rf "${push_dir}" return fi @@ -405,13 +458,14 @@ push_remote() { "metadata.json:${TEST_GUEST_CACHE_METADATA_TYPE}" \ "disk.qcow2.zst:${TEST_GUEST_CACHE_DISK_TYPE}" ) + published_digest=$(oras resolve "${remote_ref}") rm -rf "${push_dir}" echo "==> Cache push complete: ${remote_ref}" + echo "==> Cache immutable reference: ${repository}@${published_digest}" } echo "==> Cache key: ${cache_key}" -if test_vm_cache_local_entry_valid \ - "${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then +if local_entry_valid; then echo "==> Cache local hit: ${entry_dir}" else if [ -e "${entry_dir}" ]; then @@ -421,7 +475,7 @@ else fi pulled=0 - if [ -n "${remote_ref}" ]; then + if [ -n "${pull_ref}" ]; then if pull_remote; then pulled=1 else @@ -429,7 +483,7 @@ else if [ "${pull_status}" -ne 1 ]; then exit "${pull_status}" fi - echo "==> Cache remote miss: ${remote_ref}" + echo "==> Cache remote miss: ${pull_ref}" fi fi if [ "${pulled}" -eq 0 ]; then From 7f365efb813777575f291f6e48230913aa287303 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 14:35:18 -0700 Subject: [PATCH 10/29] fix(test-guest): harden cached guest reuse Signed-off-by: Drew Newberry --- nix/test-guest/README.md | 2 +- nix/test-guest/cache-lib.sh | 16 +++++++++++++++- nix/test-guest/run.sh | 4 +++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 20cdc1a8be..ebb0083442 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -152,7 +152,7 @@ A cache build boots and configures a disposable VM, runs the internal sealing sc The key includes the pinned base-image identity, guest architecture, ordered configuration file digests, Ansible version, cache generation, and sealing script digest. Installed packages, copied binaries, forwarded ports, and guest commands are never cached. -Normal `test-guest` runs automatically use an exact valid local entry and create a fresh writable overlay, cloud-init instance, machine ID, and SSH identity. On a local miss, the runner uses the pinned cloud image and applies configurations normally. Set `OPENSHELL_TEST_GUEST_CACHE_DISABLE=1` to bypass local lookup. +Normal `test-guest` runs automatically use an exact valid local entry after rechecking its disk checksum and QCOW2 structure. They create a fresh writable overlay, cloud-init instance, machine ID, and SSH identity. On a local miss, the runner uses the pinned cloud image and applies configurations normally. Set `OPENSHELL_TEST_GUEST_CACHE_DISABLE=1` to bypass local lookup. The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-guest`. Override it with `--cache-dir` on the cache command or `OPENSHELL_TEST_GUEST_CACHE_DIR` for either app. diff --git a/nix/test-guest/cache-lib.sh b/nix/test-guest/cache-lib.sh index 1e6f76b191..e5d5f0a7b1 100644 --- a/nix/test-guest/cache-lib.sh +++ b/nix/test-guest/cache-lib.sh @@ -129,13 +129,27 @@ test_vm_cache_local_entry_valid() { local distro_name=$3 shift 3 local entry + local expected_sha + local actual_sha entry=$(test_vm_cache_entry_dir "${root}" "${key}") [ -f "${entry}/complete" ] && [ -f "${entry}/disk.qcow2" ] && [ -f "${entry}/metadata.json" ] && test_vm_cache_metadata_matches \ - "${entry}/metadata.json" "${key}" "${distro_name}" "$@" + "${entry}/metadata.json" "${key}" "${distro_name}" "$@" || + return 1 + + expected_sha=$(jq -er ' + .disk_sha256 | + select(type == "string" and test("^[a-f0-9]{64}$")) + ' "${entry}/metadata.json") || + return 1 + actual_sha=$(test_vm_cache_sha256 "${entry}/disk.qcow2") || + return 1 + + [ "${actual_sha}" = "${expected_sha}" ] && + test_vm_cache_validate_disk "${entry}/disk.qcow2" } test_vm_cache_validate_disk() { diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 677f54ae34..4dffd530d2 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -215,11 +215,12 @@ umask 077 run_parent=${TMPDIR:-/tmp}/openshell-test-guest mkdir -p "${run_parent}" run_dir=$(mktemp -d "${run_parent%/}/run.XXXXXX") +ssh_control_dir=$(mktemp -d /tmp/openshell-test-guest-ssh.XXXXXX) overlay=${run_dir}/disk.qcow2 seed=${run_dir}/seed.iso vars=${run_dir}/firmware-vars.fd private_key=${run_dir}/id_ed25519 -ssh_control_path=${run_dir}/ssh-control +ssh_control_path=${ssh_control_dir}/ctl serial_log=${run_dir}/serial.log qemu_log=${run_dir}/qemu.log qemu_pid= @@ -255,6 +256,7 @@ cleanup() { else rm -rf "${run_dir}" fi + rm -rf "${ssh_control_dir}" exit "${status}" } From 7bd0790d6dd2aef7cae891da0e8a8132e86aba89 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 14:54:09 -0700 Subject: [PATCH 11/29] fix(test-guest): refresh runtime setup state Signed-off-by: Drew Newberry --- nix/test-guest/configuration/podman.yml | 6 ++++-- nix/test-guest/run.sh | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/nix/test-guest/configuration/podman.yml b/nix/test-guest/configuration/podman.yml index 2872f5cbfe..a1139b4f6d 100644 --- a/nix/test-guest/configuration/podman.yml +++ b/nix/test-guest/configuration/podman.yml @@ -22,9 +22,11 @@ update_cache: true when: ansible_facts.distribution == "Ubuntu" - - name: Install Podman + - name: Install Podman dependencies ansible.builtin.package: - name: podman + name: + - podman + - socat state: present - name: Enable the rootless Podman API socket diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 4dffd530d2..095d854d8b 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -425,6 +425,11 @@ else echo "==> Reusing cached configuration: ${configurations[*]:-base image}" fi +# Configuration may change the test user's groups. Close the SSH control +# connection established before provisioning so subsequent commands start with +# the guest's current credentials. +ssh "${ssh_args[@]}" -O exit openshell@127.0.0.1 >/dev/null 2>&1 || true + for package in "${packages[@]}"; do package_name=$(basename "${package}") if [[ ! ${package_name} =~ ^[A-Za-z0-9._+~-]+$ ]]; then From 418b2605f9404612b71674c2fd225d2312f5a879 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 14:55:21 -0700 Subject: [PATCH 12/29] feat(test-guest): support E2E runner inputs Signed-off-by: Drew Newberry --- nix/test-guest/README.md | 83 ++++++++++++++++++++++++++++++++++- nix/test-guest/run.sh | 93 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 168 insertions(+), 8 deletions(-) diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index ebb0083442..630e04fabe 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -63,6 +63,81 @@ List the available distros and configurations: nix run .#test-guest -- --list ``` +## Run a host or test-guest E2E suite + +[`e2e/run.sh`](../../e2e/run.sh) builds OpenShell from the current checkout, starts a gateway from a supplied TOML file, and runs one named host-side suite against it. This opt-in path is separate from `mise run e2e:vm`: `e2e/run.sh` selects where the gateway process runs, while the supplied gateway config selects the compute driver. It is not part of the default E2E task or CI graph. + +Omit both `--vm` and `--with` to run the gateway on the host: + +```shell +e2e/run.sh \ + --gateway-config e2e/configs/gateway/docker.toml \ + --suite smoke +``` + +Pass `--vm` or at least one `--with` to run the gateway in a disposable test guest: + +```shell +e2e/run.sh \ + --vm ubuntu \ + --with docker \ + --gateway-config e2e/configs/gateway/docker.toml \ + --suite smoke +``` + +Use `--guest-setup` when a driver needs checkout-specific guest preparation +before the gateway starts. For example, the Podman shutdown suite builds a +supervisor image around the staged `openshell-sandbox` binary: + +```shell +e2e/run.sh \ + --vm ubuntu \ + --with podman \ + --guest-setup e2e/guest-setup/podman-supervisor.sh \ + --gateway-config e2e/configs/gateway/podman.toml \ + --suite podman-shutdown +``` + +Set `OPENSHELL_E2E_EXPECT_SHUTDOWN_CLOSES=1` when a stacked change must +classify supervisor and relay transport closures as expected teardown events. + +Supplying `--with` without `--vm` selects Ubuntu. `--with` is repeatable and configurations run in input order. Supplying `--vm` without `--with` boots the selected base image without adding software. The wrapper validates both values against the test-guest catalogs: + +```shell +e2e/run.sh \ + --vm rocky \ + --with docker \ + --with selinux \ + --gateway-config e2e/configs/gateway/docker.toml \ + --suite smoke +``` + +The equivalent passthrough task accepts the same arguments: + +```shell +mise run e2e:test -- \ + --with docker \ + --gateway-config e2e/configs/gateway/docker.toml \ + --suite smoke +``` + +The wrapper always builds the tested binaries; it does not accept packages or use release artifacts. Both modes build a native host `openshell` CLI and a static Linux `openshell-sandbox` for the host architecture. Host mode also builds a native `openshell-gateway` with bundled Z3. VM mode builds a static Linux `openshell` and a Linux GNU `openshell-gateway` with bundled Z3 and a glibc 2.28 floor, then stages all three guest binaries through the VM runner's `--copy` interface. + +The Linux targets match the host and guest architecture: + +| Host architecture | CLI and sandbox | Gateway | +| --- | --- | --- | +| x86_64 | `x86_64-unknown-linux-musl` | `x86_64-unknown-linux-gnu.2.28` | +| aarch64 | `aarch64-unknown-linux-musl` | `aarch64-unknown-linux-gnu.2.28` | + +These cross-builds use the Zig and `cargo-zigbuild` versions pinned in [`mise.toml`](../../mise.toml). Install the repository's mise tools before running the wrapper. The wrapper also requires Python 3 and OpenSSL; VM mode requires `base64`. Cargo stores outputs in its normal target directory, so later runs reuse current artifacts. VM mode additionally requires the Nix, QEMU, and host-capacity prerequisites above; host mode does not invoke Nix, QEMU, or Ansible. + +Gateway configs are ordinary, fully resolved TOML files under `e2e/configs/gateway/`. The wrapper does not inspect or rewrite them. It supplies only the bind address, gateway port, plaintext transport, and isolated XDG directories at launch. Use portable relative paths for host files. The initial Docker config points `supervisor_bin` at `.cache/openshell-e2e/bin/openshell-sandbox`; the wrapper stages that path under the repository in host mode and under `/home/openshell` in VM mode. It also generates and stages the local sandbox JWT material referenced by the initial config. + +Suites are executable scripts under `e2e/suites/` and use lowercase letters, digits, and hyphens for their names. The wrapper exports the selected endpoint, isolated gateway registration, native host CLI path, mode metadata, source config path, and forwarded ports before executing a suite. Add new configs and suites without adding scenario branches to `e2e/run.sh`. + +Set `OPENSHELL_E2E_KEEP=1` to retain wrapper state after a host-mode run. In VM mode the wrapper also passes `--keep` to the test-guest runner so its overlay and logs remain available. + ## Open an interactive VM Boot a base Ubuntu VM: @@ -215,10 +290,14 @@ The destination must be an absolute guest path. Copied files are installed with --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy an executable into the guest; repeatable --ssh-port PORT Use a specific loopback SSH forwarding port +--forward-port HOST_PORT:GUEST_PORT + Forward a loopback host port to a guest port; repeatable --keep Preserve the disk overlay and logs after shutdown --list List distros and configurations ``` +Each `--forward-port` binds only `127.0.0.1` on the host. Both ports must be unprivileged values from 1024 through 65535, and each host port may appear only once. + Arguments after `--` are executed inside the guest. Without a command, the runner opens an interactive SSH session. ## Lifecycle @@ -243,5 +322,5 @@ Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log f - TCG is slower than hardware virtualization and uses a longer SSH readiness timeout. - Prepared cache entries are architecture-specific and match the exact ordered configuration list. - OCI pulls transfer a complete compressed standalone disk; incremental disk layers are not implemented. -- Only loopback SSH is forwarded to the host; guest gateway ports are not exposed. -- The runner does not build OpenShell, configure a gateway, or select an E2E test suite. +- Guest ports are reachable from the host only when explicitly exposed with loopback-only `--forward-port`. +- The low-level VM runner does not build OpenShell, configure a gateway, or select an E2E test suite. Use `e2e/run.sh` for that orchestration. diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 095d854d8b..309b71c05a 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -17,6 +17,8 @@ Options: --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy an executable to an absolute guest path; repeatable --ssh-port PORT Use a specific loopback SSH forwarding port + --forward-port HOST_PORT:GUEST_PORT + Forward a loopback host port to a guest port; repeatable --keep Keep the disposable disk and logs after shutdown --list List distros and configurations -h, --help Show this help @@ -40,6 +42,7 @@ list=0 configurations=() packages=() copies=() +forward_ports=() guest_command=() while [ "$#" -gt 0 ]; do @@ -64,6 +67,14 @@ while [ "$#" -gt 0 ]; do requested_ssh_port=${2:-} shift 2 ;; + --forward-port) + if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then + echo "--forward-port requires HOST_PORT:GUEST_PORT" >&2 + exit 2 + fi + forward_ports+=("${2:-}") + shift 2 + ;; --keep) keep=1 shift @@ -124,14 +135,57 @@ for item in "${configurations[@]}"; do done if [ -n "${requested_ssh_port}" ] && { - [[ ! ${requested_ssh_port} =~ ^[0-9]+$ ]] || - [ "${requested_ssh_port}" -lt 1024 ] || - [ "${requested_ssh_port}" -gt 65535 ] + [[ ! ${requested_ssh_port} =~ ^[0-9]+$ ]] || + [ "${requested_ssh_port}" -lt 1024 ] || + [ "${requested_ssh_port}" -gt 65535 ] }; then echo "--ssh-port must be an integer between 1024 and 65535" >&2 exit 2 fi +forward_host_ports=() +for forward_spec in "${forward_ports[@]}"; do + host_port=${forward_spec%%:*} + guest_port=${forward_spec#*:} + if [ "${host_port}" = "${forward_spec}" ] || + [[ ! ${host_port} =~ ^[1-9][0-9]*$ ]] || + [[ ! ${guest_port} =~ ^[1-9][0-9]*$ ]] || + [ "${host_port}" -lt 1024 ] || + [ "${host_port}" -gt 65535 ] || + [ "${guest_port}" -lt 1024 ] || + [ "${guest_port}" -gt 65535 ]; then + echo "--forward-port must be HOST_PORT:GUEST_PORT with both ports between 1024 and 65535: ${forward_spec:-}" >&2 + exit 2 + fi + for existing_host_port in "${forward_host_ports[@]}"; do + if [ "${host_port}" = "${existing_host_port}" ]; then + echo "duplicate --forward-port host port: ${host_port}" >&2 + exit 2 + fi + done + if [ -n "${requested_ssh_port}" ] && [ "${host_port}" = "${requested_ssh_port}" ]; then + echo "--forward-port host port conflicts with --ssh-port: ${host_port}" >&2 + exit 2 + fi + if ! python3 - "${host_port}" <<'PY' +import socket +import sys + +sock = socket.socket() +try: + sock.bind(("127.0.0.1", int(sys.argv[1]))) +except OSError: + raise SystemExit(1) +finally: + sock.close() +PY + then + echo "--forward-port host port is unavailable: ${host_port}" >&2 + exit 2 + fi + forward_host_ports+=("${host_port}") +done + for package in "${packages[@]}"; do if [ ! -f "${package}" ]; then echo "package does not exist: ${package}" >&2 @@ -268,6 +322,17 @@ pick_free_port() { python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()' } +port_is_forwarded() { + local candidate=$1 + local forwarded + for forwarded in "${forward_host_ports[@]}"; do + if [ "${candidate}" = "${forwarded}" ]; then + return 0 + fi + done + return 1 +} + ssh-keygen -q -t ed25519 -N "" -f "${private_key}" public_key=$(<"${private_key}.pub") @@ -300,7 +365,22 @@ cp "${TEST_GUEST_FIRMWARE_VARS}" "${vars}" chmod 0600 "${vars}" for attempt in $(seq 1 5); do - ssh_port=${requested_ssh_port:-$(pick_free_port)} + if [ -n "${requested_ssh_port}" ]; then + ssh_port=${requested_ssh_port} + else + while :; do + ssh_port=$(pick_free_port) + if ! port_is_forwarded "${ssh_port}"; then + break + fi + done + fi + netdev_arg="user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" + for forward_spec in "${forward_ports[@]}"; do + host_port=${forward_spec%%:*} + guest_port=${forward_spec#*:} + netdev_arg+=",hostfwd=tcp:127.0.0.1:${host_port}-:${guest_port}" + done : >"${qemu_log}" echo "==> Booting ${distro} (${TEST_GUEST_ARCHITECTURE}) with QEMU/${TEST_GUEST_ACCELERATOR}" "${TEST_GUEST_QEMU}" \ @@ -316,7 +396,7 @@ for attempt in $(seq 1 5); do -drive "if=none,format=raw,readonly=on,file=${seed},id=seed" \ -device virtio-blk-pci,drive=seed \ -boot strict=on \ - -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -netdev "${netdev_arg}" \ -device virtio-net-pci,netdev=net0 \ -display none \ -monitor none \ @@ -332,7 +412,8 @@ for attempt in $(seq 1 5); do wait "${qemu_pid}" || true qemu_pid= if ! grep -q "Could not set up host forwarding rule" "${qemu_log}" || - [ -n "${requested_ssh_port}" ]; then + [ -n "${requested_ssh_port}" ] || + [ "${#forward_ports[@]}" -gt 0 ]; then echo "QEMU exited during startup" >&2 exit 1 fi From 86cf640591154b8934db8e85483f20dd3c623c37 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 16:06:19 -0700 Subject: [PATCH 13/29] fix(test-guest): harden runner and OCI reuse Signed-off-by: Drew Newberry --- nix/test-guest/README.md | 77 +--------------------------------------- nix/test-guest/cache.sh | 36 +++++++++++++++---- nix/test-guest/run.sh | 24 ++++++++++--- 3 files changed, 49 insertions(+), 88 deletions(-) diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 630e04fabe..9b147e2ced 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -63,81 +63,6 @@ List the available distros and configurations: nix run .#test-guest -- --list ``` -## Run a host or test-guest E2E suite - -[`e2e/run.sh`](../../e2e/run.sh) builds OpenShell from the current checkout, starts a gateway from a supplied TOML file, and runs one named host-side suite against it. This opt-in path is separate from `mise run e2e:vm`: `e2e/run.sh` selects where the gateway process runs, while the supplied gateway config selects the compute driver. It is not part of the default E2E task or CI graph. - -Omit both `--vm` and `--with` to run the gateway on the host: - -```shell -e2e/run.sh \ - --gateway-config e2e/configs/gateway/docker.toml \ - --suite smoke -``` - -Pass `--vm` or at least one `--with` to run the gateway in a disposable test guest: - -```shell -e2e/run.sh \ - --vm ubuntu \ - --with docker \ - --gateway-config e2e/configs/gateway/docker.toml \ - --suite smoke -``` - -Use `--guest-setup` when a driver needs checkout-specific guest preparation -before the gateway starts. For example, the Podman shutdown suite builds a -supervisor image around the staged `openshell-sandbox` binary: - -```shell -e2e/run.sh \ - --vm ubuntu \ - --with podman \ - --guest-setup e2e/guest-setup/podman-supervisor.sh \ - --gateway-config e2e/configs/gateway/podman.toml \ - --suite podman-shutdown -``` - -Set `OPENSHELL_E2E_EXPECT_SHUTDOWN_CLOSES=1` when a stacked change must -classify supervisor and relay transport closures as expected teardown events. - -Supplying `--with` without `--vm` selects Ubuntu. `--with` is repeatable and configurations run in input order. Supplying `--vm` without `--with` boots the selected base image without adding software. The wrapper validates both values against the test-guest catalogs: - -```shell -e2e/run.sh \ - --vm rocky \ - --with docker \ - --with selinux \ - --gateway-config e2e/configs/gateway/docker.toml \ - --suite smoke -``` - -The equivalent passthrough task accepts the same arguments: - -```shell -mise run e2e:test -- \ - --with docker \ - --gateway-config e2e/configs/gateway/docker.toml \ - --suite smoke -``` - -The wrapper always builds the tested binaries; it does not accept packages or use release artifacts. Both modes build a native host `openshell` CLI and a static Linux `openshell-sandbox` for the host architecture. Host mode also builds a native `openshell-gateway` with bundled Z3. VM mode builds a static Linux `openshell` and a Linux GNU `openshell-gateway` with bundled Z3 and a glibc 2.28 floor, then stages all three guest binaries through the VM runner's `--copy` interface. - -The Linux targets match the host and guest architecture: - -| Host architecture | CLI and sandbox | Gateway | -| --- | --- | --- | -| x86_64 | `x86_64-unknown-linux-musl` | `x86_64-unknown-linux-gnu.2.28` | -| aarch64 | `aarch64-unknown-linux-musl` | `aarch64-unknown-linux-gnu.2.28` | - -These cross-builds use the Zig and `cargo-zigbuild` versions pinned in [`mise.toml`](../../mise.toml). Install the repository's mise tools before running the wrapper. The wrapper also requires Python 3 and OpenSSL; VM mode requires `base64`. Cargo stores outputs in its normal target directory, so later runs reuse current artifacts. VM mode additionally requires the Nix, QEMU, and host-capacity prerequisites above; host mode does not invoke Nix, QEMU, or Ansible. - -Gateway configs are ordinary, fully resolved TOML files under `e2e/configs/gateway/`. The wrapper does not inspect or rewrite them. It supplies only the bind address, gateway port, plaintext transport, and isolated XDG directories at launch. Use portable relative paths for host files. The initial Docker config points `supervisor_bin` at `.cache/openshell-e2e/bin/openshell-sandbox`; the wrapper stages that path under the repository in host mode and under `/home/openshell` in VM mode. It also generates and stages the local sandbox JWT material referenced by the initial config. - -Suites are executable scripts under `e2e/suites/` and use lowercase letters, digits, and hyphens for their names. The wrapper exports the selected endpoint, isolated gateway registration, native host CLI path, mode metadata, source config path, and forwarded ports before executing a suite. Add new configs and suites without adding scenario branches to `e2e/run.sh`. - -Set `OPENSHELL_E2E_KEEP=1` to retain wrapper state after a host-mode run. In VM mode the wrapper also passes `--keep` to the test-guest runner so its overlay and logs remain available. - ## Open an interactive VM Boot a base Ubuntu VM: @@ -323,4 +248,4 @@ Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log f - Prepared cache entries are architecture-specific and match the exact ordered configuration list. - OCI pulls transfer a complete compressed standalone disk; incremental disk layers are not implemented. - Guest ports are reachable from the host only when explicitly exposed with loopback-only `--forward-port`. -- The low-level VM runner does not build OpenShell, configure a gateway, or select an E2E test suite. Use `e2e/run.sh` for that orchestration. +- The runner does not build OpenShell, configure a gateway, or select an E2E test suite. diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index 4943a028c8..24c9a04a23 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -241,6 +241,10 @@ is_remote_miss() { } pull_remote() { + local requested_ref=${1:-${pull_ref}} + local trusted_digest=${2:-${pull_digest}} + local install_local=${3:-1} + local expected_local_sha=${4:-} local pull_dir local pull_log local compressed_size @@ -250,17 +254,17 @@ pull_remote() { pull_dir=$(mktemp -d "${cache_root}/staging/pull.${cache_key}.XXXXXX") pull_log="${pull_dir}/oras.log" - resolved_digest=$(oras resolve "${pull_ref}") || { + resolved_digest=$(oras resolve "${requested_ref}") || { rm -rf "${pull_dir}" return 2 } - if [ "${resolved_digest}" != "${pull_digest}" ]; then + if [ "${resolved_digest}" != "${trusted_digest}" ]; then echo "OCI cache manifest digest does not match trusted digest" >&2 rm -rf "${pull_dir}" return 2 fi - if ! oras pull --output "${pull_dir}" "${pull_ref}" >"${pull_log}" 2>&1; then + if ! oras pull --output "${pull_dir}" "${requested_ref}" >"${pull_log}" 2>&1; then if is_remote_miss "${pull_log}"; then rm -rf "${pull_dir}" return 1 @@ -298,6 +302,11 @@ pull_remote() { -o "${pull_dir}/disk.qcow2" ) expected_sha=$(jq -r '.disk_sha256' "${pull_dir}/metadata.json") + if [ -n "${expected_local_sha}" ] && [ "${expected_sha}" != "${expected_local_sha}" ]; then + echo "OCI cache disk checksum does not match the local cache entry" >&2 + rm -rf "${pull_dir}" + return 2 + fi actual_sha=$(test_vm_cache_sha256 "${pull_dir}/disk.qcow2") if [ "${expected_sha}" != "${actual_sha}" ]; then echo "OCI cache disk checksum does not match metadata" >&2 @@ -310,10 +319,14 @@ pull_remote() { return 2 fi - install_entry \ - "${pull_dir}/disk.qcow2" "${pull_dir}/metadata.json" "${resolved_digest}" + if [ "${install_local}" -eq 1 ]; then + install_entry \ + "${pull_dir}/disk.qcow2" "${pull_dir}/metadata.json" "${resolved_digest}" + fi rm -rf "${pull_dir}" - echo "==> Cache remote hit: ${pull_ref}" + if [ "${install_local}" -eq 1 ]; then + echo "==> Cache remote hit: ${requested_ref}" + fi } build_local() { @@ -426,6 +439,8 @@ build_local() { } push_remote() { + local existing_ref + local local_disk_sha local push_dir local manifest_log local published_digest @@ -433,8 +448,15 @@ push_remote() { manifest_log="${push_dir}/manifest.log" if oras manifest fetch "${remote_ref}" >"${manifest_log}" 2>&1; then - echo "==> Cache already published: ${remote_ref}" published_digest=$(oras resolve "${remote_ref}") + existing_ref="${repository}@${published_digest}" + local_disk_sha=$(jq -r '.disk_sha256' "${entry_dir}/metadata.json") + if ! pull_remote "${existing_ref}" "${published_digest}" 0 "${local_disk_sha}"; then + echo "existing OCI cache artifact failed validation: ${remote_ref}" >&2 + rm -rf "${push_dir}" + return 1 + fi + echo "==> Cache already published and validated: ${remote_ref}" echo "==> Cache immutable reference: ${repository}@${published_digest}" rm -rf "${push_dir}" return diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 309b71c05a..e5478fb97d 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -35,6 +35,13 @@ if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ] || exit 2 fi +require_value() { + if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then + echo "$1 requires a value" >&2 + exit 2 + fi +} + distro= requested_ssh_port= keep=0 @@ -48,23 +55,28 @@ guest_command=() while [ "$#" -gt 0 ]; do case "$1" in --distro) - distro=${2:-} + require_value "$@" + distro=$2 shift 2 ;; --with) - configurations+=("${2:-}") + require_value "$@" + configurations+=("$2") shift 2 ;; --install) - packages+=("${2:-}") + require_value "$@" + packages+=("$2") shift 2 ;; --copy) - copies+=("${2:-}") + require_value "$@" + copies+=("$2") shift 2 ;; --ssh-port) - requested_ssh_port=${2:-} + require_value "$@" + requested_ssh_port=$2 shift 2 ;; --forward-port) @@ -434,6 +446,7 @@ ssh_args=( -o ControlMaster=auto -o ControlPersist=60 -o "ControlPath=${ssh_control_path}" + -o IdentitiesOnly=yes -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @@ -447,6 +460,7 @@ scp_args=( -o ControlMaster=auto -o ControlPersist=60 -o "ControlPath=${ssh_control_path}" + -o IdentitiesOnly=yes -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null From 3d48a73ab33a8f9554e13d8b320bfe79088d17ea Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 17:09:44 -0700 Subject: [PATCH 14/29] feat(test-guest): prepare Podman E2E artifacts Signed-off-by: Drew Newberry --- nix/test-guest/README.md | 7 ++ nix/test-guest/configuration/podman.yml | 111 ++++++++++++++++++++++++ nix/test-guest/run.sh | 10 +++ 3 files changed, 128 insertions(+) diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 9b147e2ced..c52cc0b28e 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -112,6 +112,13 @@ Configurations are Ansible playbooks stored under `nix/test-guest/configuration/ Configurations run in the order provided on the command line. OpenShell packages and copied binaries are installed after all configurations succeed. +A configuration can install executable hooks under +`/usr/local/libexec/openshell-test-guest/post-copy.d`. The runner invokes them +in lexical order after all `--install` and `--copy` artifacts are present and +before the requested guest command starts. For example, the Podman +configuration uses a hook to prepare checkout-built E2E artifacts in the +guest's rootless Podman store. + ## Prepared VM cache The `test-guest-cache` app ensures a prepared disk exists for one exact distro, host architecture, and ordered configuration list. It checks the local cache first, optionally pulls a matching OCI artifact, or builds and validates a new local entry on a miss: diff --git a/nix/test-guest/configuration/podman.yml b/nix/test-guest/configuration/podman.yml index a1139b4f6d..400d3c3462 100644 --- a/nix/test-guest/configuration/podman.yml +++ b/nix/test-guest/configuration/podman.yml @@ -37,6 +37,117 @@ state: started become: false + - name: Install the Podman post-copy hook directory + ansible.builtin.file: + path: /usr/local/libexec/openshell-test-guest/post-copy.d + state: directory + owner: root + group: root + mode: "0755" + + - name: Install the Podman E2E artifact hook + ansible.builtin.copy: + dest: /usr/local/libexec/openshell-test-guest/post-copy.d/50-podman-e2e + owner: root + group: root + mode: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + supervisor_bin=/home/openshell/.cache/openshell-e2e/bin/openshell-sandbox + supervisor_image=localhost/openshell/supervisor:e2e-vm + relay_socket=/home/openshell/.cache/openshell-e2e/podman-gateway.sock + host_relay_unit=openshell-e2e-podman-host-relay + network_relay_unit=openshell-e2e-podman-network-relay + + # A normal interactive test guest has no checkout artifact to prepare. + if [ ! -e "${supervisor_bin}" ]; then + exit 0 + fi + if [ ! -x "${supervisor_bin}" ]; then + echo "ERROR: staged supervisor is not executable: ${supervisor_bin}" >&2 + exit 1 + fi + + # test-guest installs copied artifacts as root. Hand the E2E state tree + # back to the test user before Podman and the relay services use it. + sudo chown -R "$(id -u):$(id -g)" /home/openshell/.cache + chmod 0700 /home/openshell/.cache/openshell-e2e + + build_context="$(mktemp -d)" + cleanup() { + rm -rf "${build_context}" + } + trap cleanup EXIT + + install -m 0555 "${supervisor_bin}" "${build_context}/openshell-sandbox" + cat >"${build_context}/Containerfile" <<'EOF' + FROM alpine:3.22 + RUN apk add --no-cache nftables iptables iptables-legacy + COPY --chmod=0555 openshell-sandbox /openshell-sandbox + ENTRYPOINT ["/openshell-sandbox"] + EOF + + echo "==> Building checkout supervisor image: ${supervisor_image}" + podman build \ + --pull=missing \ + --tag "${supervisor_image}" \ + --file "${build_context}/Containerfile" \ + "${build_context}" + podman image exists "${supervisor_image}" + + mkdir -p "$(dirname "${relay_socket}")" + systemctl --user stop \ + "${network_relay_unit}.service" \ + "${host_relay_unit}.service" \ + >/dev/null 2>&1 || true + systemctl --user reset-failed \ + "${network_relay_unit}.service" \ + "${host_relay_unit}.service" \ + >/dev/null 2>&1 || true + rm -f "${relay_socket}" + + systemd-run --user \ + --unit="${host_relay_unit}" \ + --collect \ + --property=Restart=on-failure \ + --property=RestartSec=1 \ + -- \ + /usr/bin/socat \ + "UNIX-LISTEN:${relay_socket},fork" \ + TCP:127.0.0.1:8080 + for _ in $(seq 1 50); do + [ -S "${relay_socket}" ] && break + sleep 0.1 + done + if [ ! -S "${relay_socket}" ] || + ! systemctl --user is-active --quiet "${host_relay_unit}.service"; then + echo "ERROR: failed to start the host-side Podman gateway relay" >&2 + systemctl --user status "${host_relay_unit}.service" --no-pager >&2 || true + exit 1 + fi + + systemd-run --user \ + --unit="${network_relay_unit}" \ + --collect \ + --property=Restart=on-failure \ + --property=RestartSec=1 \ + -- \ + /usr/bin/podman unshare --rootless-netns \ + /usr/bin/socat \ + TCP-LISTEN:8080,bind=0.0.0.0,reuseaddr,fork \ + "UNIX-CONNECT:${relay_socket}" + for _ in $(seq 1 50); do + if systemctl --user is-active --quiet "${network_relay_unit}.service"; then + exit 0 + fi + sleep 0.1 + done + echo "ERROR: failed to start the rootless-network Podman gateway relay" >&2 + systemctl --user status "${network_relay_unit}.service" --no-pager >&2 || true + exit 1 + - name: Verify rootless Podman ansible.builtin.command: cmd: podman info diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index e5478fb97d..cee6afbe4f 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -567,6 +567,16 @@ for copy_spec in "${copies[@]}"; do "sudo install -D -m 0755 '${remote_copy}' '${destination}'; rm -f '${remote_copy}'" done +post_copy_hook_dir=/usr/local/libexec/openshell-test-guest/post-copy.d +if ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "test -d '${post_copy_hook_dir}'"; then + echo "==> Running guest post-copy hooks" + # The hook directory is installed by trusted test-guest configurations. + # shellcheck disable=SC2029 + ssh "${ssh_args[@]}" openshell@127.0.0.1 \ + "set -eu; for hook in '${post_copy_hook_dir}'/*; do [ -e \"\${hook}\" ] || continue; [ -x \"\${hook}\" ] || { echo \"hook is not executable: \${hook}\" >&2; exit 1; }; \"\${hook}\"; done" +fi + echo "==> Test guest ready: ${distro} (SSH port ${ssh_port})" if [ "${#guest_command[@]}" -eq 0 ]; then ssh -t "${ssh_args[@]}" openshell@127.0.0.1 From e031d6e02a3ab93b913eace05c9756e143d5bb68 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 14:18:55 -0700 Subject: [PATCH 15/29] feat(e2e): add host and test VM runner Signed-off-by: Drew Newberry --- e2e/configs/gateway/docker.toml | 27 ++ e2e/run.sh | 555 ++++++++++++++++++++++++++++++++ e2e/suites/smoke.sh | 11 + tasks/test.toml | 4 + 4 files changed, 597 insertions(+) create mode 100644 e2e/configs/gateway/docker.toml create mode 100755 e2e/run.sh create mode 100755 e2e/suites/smoke.sh diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml new file mode 100644 index 0000000000..008ed65316 --- /dev/null +++ b/e2e/configs/gateway/docker.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "127.0.0.1:8080" +log_level = "info" +compute_drivers = ["docker"] +disable_tls = true + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.gateway.gateway_jwt] +signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" +public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" +kid_path = ".cache/openshell-e2e/gateway-jwt/kid" +gateway_id = "openshell-e2e" +ttl_secs = 0 + +[openshell.drivers.docker] +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "IfNotPresent" +sandbox_namespace = "openshell-e2e" +supervisor_bin = ".cache/openshell-e2e/bin/openshell-sandbox" diff --git a/e2e/run.sh b/e2e/run.sh new file mode 100755 index 0000000000..10155e1a77 --- /dev/null +++ b/e2e/run.sh @@ -0,0 +1,555 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build the current checkout, run its gateway on the host or in a disposable +# Nix test VM, and execute one named host-side E2E suite against that gateway. + +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# shellcheck disable=SC1091 +source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck disable=SC1091 +source "${ROOT}/tasks/scripts/build-env.sh" + +e2e_preserve_mise_dirs + +usage() { + cat <<'EOF' +Usage: + e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ + --gateway-config PATH --suite NAME + +Options: + --vm DISTRO Run the gateway in a Nix test VM + --with CONFIG Apply a Nix test-VM configuration; repeatable + --gateway-config PATH + Fully resolved gateway TOML + --suite NAME Host-side suite at e2e/suites/NAME.sh + -h, --help Show this help + +Omit --vm and --with to run the gateway on the host. Supplying --with without +--vm selects the Ubuntu test VM. Set OPENSHELL_E2E_KEEP=1 to retain state. +EOF +} + +die() { + echo "ERROR: $*" >&2 + exit 2 +} + +require_value() { + local option=$1 + local count=$2 + local value=${3:-} + + if [ "${count}" -lt 2 ] || [ -z "${value}" ]; then + die "${option} requires a value" + fi + case "${value}" in + --*) die "${option} requires a value" ;; + esac +} + +resolve_file() { + local path=$1 + + if [ ! -f "${path}" ]; then + return 1 + fi + python3 - "${path}" <<'PY' +import os +import sys + +print(os.path.realpath(sys.argv[1])) +PY +} + +catalog_has_entry() { + local catalog=$1 + local section=$2 + local name=$3 + + printf '%s\n' "${catalog}" | awk -v wanted_section="${section}:" -v wanted_name="${name}" ' + $0 == wanted_section { + in_section = 1 + next + } + /^[^[:space:]]/ { + in_section = 0 + } + in_section && $0 == " " wanted_name { + found = 1 + } + END { + exit(found ? 0 : 1) + } + ' +} + +vm= +vm_set=0 +gateway_config= +gateway_config_set=0 +suite_name= +suite_set=0 +with_configurations=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --vm) + require_value "$1" "$#" "${2:-}" + if [ "${vm_set}" -eq 1 ]; then + die "--vm may be supplied only once" + fi + vm=$2 + vm_set=1 + shift 2 + ;; + --with) + require_value "$1" "$#" "${2:-}" + with_configurations+=("$2") + shift 2 + ;; + --gateway-config) + require_value "$1" "$#" "${2:-}" + if [ "${gateway_config_set}" -eq 1 ]; then + die "--gateway-config may be supplied only once" + fi + gateway_config=$2 + gateway_config_set=1 + shift 2 + ;; + --suite) + require_value "$1" "$#" "${2:-}" + if [ "${suite_set}" -eq 1 ]; then + die "--suite may be supplied only once" + fi + suite_name=$2 + suite_set=1 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +if [ "${gateway_config_set}" -ne 1 ]; then + die "--gateway-config is required" +fi +if [ "${suite_set}" -ne 1 ]; then + die "--suite is required" +fi +if ! command -v python3 >/dev/null 2>&1; then + die "python3 is required" +fi +gateway_config_source=${gateway_config} +if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then + die "gateway config does not exist: ${gateway_config_source}" +fi +if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" +fi +suite_path="${ROOT}/e2e/suites/${suite_name}.sh" +if [ ! -f "${suite_path}" ]; then + die "unknown suite: ${suite_name}" +fi +if [ ! -x "${suite_path}" ]; then + die "suite is not executable: ${suite_path}" +fi +suite_path="$(resolve_file "${suite_path}")" + +mode=host +if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then + mode=vm + if [ "${vm_set}" -eq 0 ]; then + vm=ubuntu + fi +fi + +if [ "${mode}" = vm ]; then + if [[ ! ${vm} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + die "invalid VM distro name: ${vm}" + fi + for configuration in "${with_configurations[@]}"; do + if [[ ! ${configuration} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + die "invalid VM configuration name: ${configuration}" + fi + done + if ! command -v nix >/dev/null 2>&1; then + die "Nix is required for VM mode" + fi + if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-vm -- --list)"; then + die "failed to read the Nix test-VM catalog" + fi + if ! catalog_has_entry "${vm_catalog}" Distros "${vm}"; then + die "unknown VM distro in the Nix test-VM catalog: ${vm}" + fi + for configuration in "${with_configurations[@]}"; do + if ! catalog_has_entry "${vm_catalog}" Configurations "${configuration}"; then + die "unknown VM configuration in the Nix test-VM catalog: ${configuration}" + fi + done +fi + +gateway_ready_timeout=${OPENSHELL_E2E_GATEWAY_READY_TIMEOUT:-600} +if [[ ! ${gateway_ready_timeout} =~ ^[1-9][0-9]*$ ]]; then + die "OPENSHELL_E2E_GATEWAY_READY_TIMEOUT must be a positive integer" +fi +if ! command -v mise >/dev/null 2>&1; then + die "mise is required to build OpenShell" +fi +if ! command -v openssl >/dev/null 2>&1; then + die "OpenSSL is required to generate sandbox JWT keys" +fi + +case "$(uname -m)" in +x86_64 | amd64) + linux_musl_target=x86_64-unknown-linux-musl + linux_gateway_rust_target=x86_64-unknown-linux-gnu + linux_gateway_zig_target=x86_64-unknown-linux-gnu.2.28 + ;; +aarch64 | arm64) + linux_musl_target=aarch64-unknown-linux-musl + linux_gateway_rust_target=aarch64-unknown-linux-gnu + linux_gateway_zig_target=aarch64-unknown-linux-gnu.2.28 + ;; +*) + die "unsupported host architecture: $(uname -m)" + ;; +esac + +cargo_jobs=() +if [ -n "${CARGO_BUILD_JOBS:-}" ]; then + cargo_jobs=(-j "${CARGO_BUILD_JOBS}") +fi + +cd "${ROOT}" +target_dir="$( + mise x -- cargo metadata --format-version=1 --no-deps | + python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])' +)" + +echo "==> Building native host openshell CLI" +mise x -- cargo build "${cargo_jobs[@]}" -p openshell-cli --bin openshell +host_cli_bin="${target_dir}/debug/openshell" + +echo "==> Preparing ${linux_musl_target} build target" +mise x -- rustup target add "${linux_musl_target}" >/dev/null +ensure_build_nofile_limit + +echo "==> Building Linux openshell-sandbox (${linux_musl_target})" +mise x -- cargo zigbuild "${cargo_jobs[@]}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-sandbox \ + --bin openshell-sandbox +linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" + +host_gateway_bin= +guest_cli_bin= +guest_gateway_bin= +if [ "${mode}" = host ]; then + echo "==> Building native host openshell-gateway" + mise x -- cargo build "${cargo_jobs[@]}" \ + -p openshell-server \ + --bin openshell-gateway \ + --features bundled-z3 + host_gateway_bin="${target_dir}/debug/openshell-gateway" +else + if ! command -v base64 >/dev/null 2>&1; then + die "base64 is required for VM mode" + fi + echo "==> Building Linux openshell CLI (${linux_musl_target})" + CXXSTDLIB=c++ mise x -- cargo zigbuild "${cargo_jobs[@]}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-cli \ + --bin openshell + guest_cli_bin="${target_dir}/${linux_musl_target}/release/openshell" + + echo "==> Preparing ${linux_gateway_rust_target} build target" + mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null + echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" + ( + eval "$( + "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ + "${linux_gateway_zig_target}" \ + "${linux_gateway_zig_target}" \ + "${target_dir}/zig-gnu-wrapper/e2e" + )" + mise x -- cargo zigbuild "${cargo_jobs[@]}" \ + --release \ + --target "${linux_gateway_zig_target}" \ + -p openshell-server \ + --bin openshell-gateway \ + --features bundled-z3 + ) + guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" +fi + +for binary in "${host_cli_bin}" "${linux_sandbox_bin}"; do + if [ ! -x "${binary}" ]; then + echo "ERROR: expected built binary at ${binary}" >&2 + exit 1 + fi +done +if [ "${mode}" = host ] && [ ! -x "${host_gateway_bin}" ]; then + echo "ERROR: expected built gateway at ${host_gateway_bin}" >&2 + exit 1 +fi +if [ "${mode}" = vm ]; then + for binary in "${guest_cli_bin}" "${guest_gateway_bin}"; do + if [ ! -x "${binary}" ]; then + echo "ERROR: expected built guest binary at ${binary}" >&2 + exit 1 + fi + done +fi + +run_parent="${ROOT}/.cache/openshell-e2e/runs" +mkdir -p "${run_parent}" +run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")" +child_pid= +child_process_group=0 +runtime_log= +portable_jwt_dir="${ROOT}/.cache/openshell-e2e/gateway-jwt" +keep=0 +if [ "${OPENSHELL_E2E_KEEP:-0}" = 1 ]; then + keep=1 +fi + +# Invoked by the EXIT trap through cleanup. +# shellcheck disable=SC2329 +stop_child() { + local pid=$1 + local process_group=$2 + local signal_target=${pid} + + if [ -z "${pid}" ] || ! kill -0 "${pid}" 2>/dev/null; then + return + fi + if [ "${process_group}" -eq 1 ]; then + signal_target="-${pid}" + fi + kill -TERM -- "${signal_target}" 2>/dev/null || true + for _ in $(seq 1 30); do + if ! kill -0 "${pid}" 2>/dev/null; then + break + fi + sleep 1 + done + if kill -0 "${pid}" 2>/dev/null; then + kill -KILL -- "${signal_target}" 2>/dev/null || true + fi + wait "${pid}" 2>/dev/null || true +} + +# Invoked by EXIT, INT, and TERM traps. +# shellcheck disable=SC2329 +cleanup() { + local incoming_status=$? + local status=${1:-${incoming_status}} + + trap - EXIT INT TERM + stop_child "${child_pid}" "${child_process_group}" + if [ "${status}" -ne 0 ] && [ -n "${runtime_log}" ] && [ -f "${runtime_log}" ]; then + echo "=== ${mode} gateway log ===" >&2 + cat "${runtime_log}" >&2 + echo "=== end ${mode} gateway log ===" >&2 + fi + if [ "${keep}" -eq 1 ]; then + echo "Kept E2E wrapper state at ${run_dir}" >&2 + else + rm -f \ + "${portable_jwt_dir}/signing.pem" \ + "${portable_jwt_dir}/public.pem" \ + "${portable_jwt_dir}/kid" + rmdir "${portable_jwt_dir}" 2>/dev/null || true + rm -rf "${run_dir}" + fi + exit "${status}" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +jwt_source_dir="${run_dir}/gateway-jwt" +e2e_generate_gateway_jwt "${jwt_source_dir}" +mkdir -p "${portable_jwt_dir}" +install -m 0600 "${jwt_source_dir}/signing.pem" "${portable_jwt_dir}/signing.pem" +install -m 0600 "${jwt_source_dir}/public.pem" "${portable_jwt_dir}/public.pem" +install -m 0600 "${jwt_source_dir}/kid" "${portable_jwt_dir}/kid" + +host_port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')" +guest_port= +if [ "${mode}" = vm ]; then + guest_port=8080 +fi + +export XDG_CONFIG_HOME="${run_dir}/host/config" +export XDG_DATA_HOME="${run_dir}/host/data" +export XDG_STATE_HOME="${run_dir}/host/state" +mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_STATE_HOME}" + +gateway_name="openshell-e2e-${mode}-${host_port}" +gateway_endpoint="http://127.0.0.1:${host_port}" +e2e_register_plaintext_gateway \ + "${XDG_CONFIG_HOME}" \ + "${gateway_name}" \ + "${gateway_endpoint}" \ + "${host_port}" + +with_contract="${with_configurations[*]}" +export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" +export OPENSHELL_GATEWAY="${gateway_name}" +export OPENSHELL_BIN="${host_cli_bin}" +export OPENSHELL_E2E_MODE="${mode}" +export OPENSHELL_E2E_VM="${vm}" +export OPENSHELL_E2E_WITH="${with_contract}" +export OPENSHELL_E2E_GATEWAY_CONFIG="${gateway_config}" +export OPENSHELL_E2E_HOST_PORT="${host_port}" +export OPENSHELL_E2E_GUEST_PORT="${guest_port}" + +staged_supervisor="${ROOT}/.cache/openshell-e2e/bin/openshell-sandbox" +mkdir -p "$(dirname "${staged_supervisor}")" +ln -sfn "${linux_sandbox_bin}" "${staged_supervisor}" + +if [ "${mode}" = host ]; then + e2e_align_docker_host_with_cli_context + runtime_log="${run_dir}/gateway.log" + echo "==> Starting host gateway at ${gateway_endpoint}" + ( + cd "${ROOT}" + exec "${host_gateway_bin}" \ + --config "${gateway_config}" \ + --bind-address 127.0.0.1 \ + --port "${host_port}" \ + --disable-tls + ) >"${runtime_log}" 2>&1 & + child_pid=$! + child_process_group=0 +else + runtime_log="${run_dir}/vm.log" + guest_launcher="${run_dir}/launch-gateway.sh" + guest_launcher_path="/home/openshell/.cache/openshell-e2e/bin/launch-gateway" + config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" + jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" + jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" + jwt_kid_payload="$(base64 <"${jwt_source_dir}/kid" | tr -d '\r\n')" + cat >"${guest_launcher}" <"\${config_path}" +printf '%s' '${jwt_signing_payload}' | base64 --decode >"\${jwt_root}/signing.pem" +printf '%s' '${jwt_public_payload}' | base64 --decode >"\${jwt_root}/public.pem" +printf '%s' '${jwt_kid_payload}' | base64 --decode >"\${jwt_root}/kid" +chmod 0600 "\${config_path}" +chmod 0600 "\${jwt_root}/signing.pem" "\${jwt_root}/public.pem" "\${jwt_root}/kid" +export XDG_CONFIG_HOME=\${state_root}/xdg/config +export XDG_DATA_HOME=\${state_root}/xdg/data +export XDG_STATE_HOME=\${state_root}/xdg/state +cd /home/openshell +exec /usr/local/bin/openshell-gateway \ + --config "\${config_path}" \ + --bind-address 0.0.0.0 \ + --port ${guest_port} \ + --disable-tls +EOF + chmod 0700 "${guest_launcher}" + + vm_args=( + nix run .#test-vm -- + --distro "${vm}" + ) + for configuration in "${with_configurations[@]}"; do + vm_args+=(--with "${configuration}") + done + vm_args+=( + --copy "${guest_cli_bin}:/usr/local/bin/openshell" + --copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway" + --copy "${linux_sandbox_bin}:/home/openshell/.cache/openshell-e2e/bin/openshell-sandbox" + --copy "${guest_launcher}:${guest_launcher_path}" + --forward-port "${host_port}:${guest_port}" + ) + if [ "${keep}" -eq 1 ]; then + vm_args+=(--keep) + fi + vm_args+=(-- "${guest_launcher_path}") + + echo "==> Starting ${vm} test VM gateway at ${gateway_endpoint}" + ( + cd "${ROOT}" + exec python3 -c \ + 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ + "${vm_args[@]}" + ) >"${runtime_log}" 2>&1 & + child_pid=$! + child_process_group=1 +fi + +wait_for_gateway() { + local elapsed=0 + local process_status + local probe_log="${run_dir}/gateway-probe.log" + + echo "==> Waiting up to ${gateway_ready_timeout}s for gateway readiness" + while [ "${elapsed}" -lt "${gateway_ready_timeout}" ]; do + if ! kill -0 "${child_pid}" 2>/dev/null; then + wait "${child_pid}" + process_status=$? + child_pid= + echo "ERROR: ${mode} gateway process exited before becoming ready" >&2 + if [ "${process_status}" -eq 0 ]; then + return 1 + fi + return "${process_status}" + fi + if NO_COLOR=1 "${OPENSHELL_BIN}" status >"${probe_log}" 2>&1 && + grep -q "Connected" "${probe_log}"; then + echo "==> Gateway ready after ${elapsed}s" + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + echo "ERROR: gateway did not become ready within ${gateway_ready_timeout}s" >&2 + if [ -s "${probe_log}" ]; then + echo "=== last gateway probe ===" >&2 + cat "${probe_log}" >&2 + echo "=== end last gateway probe ===" >&2 + fi + return 1 +} + +set +e +wait_for_gateway +readiness_status=$? +set -e +if [ "${readiness_status}" -ne 0 ]; then + cleanup "${readiness_status}" +fi + +echo "==> Running E2E suite: ${suite_name}" +set +e +"${suite_path}" +suite_status=$? +set -e +cleanup "${suite_status}" diff --git a/e2e/suites/smoke.sh b/e2e/suites/smoke.sh new file mode 100755 index 0000000000..84757e70d5 --- /dev/null +++ b/e2e/suites/smoke.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +exec cargo test \ + --manifest-path e2e/rust/Cargo.toml \ + --features e2e \ + --test smoke \ + -- --nocapture diff --git a/tasks/test.toml b/tasks/test.toml index 96dde276ce..bb13690934 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -37,6 +37,10 @@ hide = true description = "Run all end-to-end tests (Rust + Python + MCP)" depends = ["e2e:rust", "e2e:python", "e2e:mcp"] +["e2e:run"] +description = "Build the current checkout and run a named host or Nix test-VM E2E suite" +run = "e2e/run.sh" + ["e2e:gpu"] description = "Run Docker GPU end-to-end tests" depends = ["e2e:docker:gpu"] From 1d066ad8b7ca9b97e976c342e972ab3e8a8f98f9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 27 Jul 2026 14:16:54 -0700 Subject: [PATCH 16/29] test(e2e): add VM-backed Podman shutdown suite Signed-off-by: Drew Newberry --- e2e/configs/gateway/podman.toml | 29 +++++++++++ e2e/guest-setup/podman-supervisor.sh | 78 ++++++++++++++++++++++++++++ e2e/run.sh | 36 ++++++++++++- e2e/rust/Cargo.toml | 5 ++ e2e/rust/tests/podman_shutdown.rs | 70 +++++++++++++++++++++++++ e2e/suites/podman-shutdown.sh | 38 ++++++++++++++ 6 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 e2e/configs/gateway/podman.toml create mode 100755 e2e/guest-setup/podman-supervisor.sh create mode 100644 e2e/rust/tests/podman_shutdown.rs create mode 100755 e2e/suites/podman-shutdown.sh diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml new file mode 100644 index 0000000000..2ad86099fb --- /dev/null +++ b/e2e/configs/gateway/podman.toml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "127.0.0.1:8080" +log_level = "info" +compute_drivers = ["podman"] +disable_tls = true + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.gateway.gateway_jwt] +signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" +public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" +kid_path = ".cache/openshell-e2e/gateway-jwt/kid" +gateway_id = "openshell-e2e" +ttl_secs = 0 + +[openshell.drivers.podman] +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "missing" +network_name = "openshell-e2e" +grpc_endpoint = "http://10.89.0.1:8080" +host_gateway_ip = "10.89.0.1" +supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/guest-setup/podman-supervisor.sh b/e2e/guest-setup/podman-supervisor.sh new file mode 100755 index 0000000000..303c1f029e --- /dev/null +++ b/e2e/guest-setup/podman-supervisor.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build the supervisor image inside the disposable VM so the Podman driver uses +# the openshell-sandbox binary built from the checkout under test. + +set -euo pipefail + +supervisor_bin=/home/openshell/.cache/openshell-e2e/bin/openshell-sandbox +supervisor_image=localhost/openshell/supervisor:e2e-vm +build_context="$(mktemp -d)" + +cleanup() { + rm -rf "${build_context}" +} +trap cleanup EXIT + +if [ ! -x "${supervisor_bin}" ]; then + echo "ERROR: staged supervisor binary is missing: ${supervisor_bin}" >&2 + exit 1 +fi +if ! command -v podman >/dev/null 2>&1; then + echo "ERROR: Podman is required by the supervisor guest setup" >&2 + exit 1 +fi +if [ -z "${XDG_RUNTIME_DIR:-}" ]; then + echo "ERROR: XDG_RUNTIME_DIR is required to locate the rootless Podman socket" >&2 + exit 1 +fi + +# Build through the user API service so the gateway driver sees the image in +# the same Podman store even though the E2E launcher overrides XDG_DATA_HOME. +podman_remote=(podman --url "unix://${XDG_RUNTIME_DIR}/podman/podman.sock") + +# Rootless netavark isolates its bridge from the VM host network namespace. +# Relay the bridge gateway through a Unix socket so supervisors can call the +# host gateway while the host-side E2E process still reaches its forwarded port. +if ! command -v socat >/dev/null 2>&1; then + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -qq -y socat +fi +relay_socket=/home/openshell/.cache/openshell-e2e/podman-gateway.sock +rm -f "${relay_socket}" +socat "UNIX-LISTEN:${relay_socket},fork" TCP:127.0.0.1:8080 & +host_relay_pid=$! +for _ in $(seq 1 50); do + [ -S "${relay_socket}" ] && break + sleep 0.1 +done +if [ ! -S "${relay_socket}" ] || ! kill -0 "${host_relay_pid}" 2>/dev/null; then + echo "ERROR: failed to start the host-side Podman gateway relay" >&2 + exit 1 +fi +env -u XDG_CONFIG_HOME -u XDG_DATA_HOME -u XDG_STATE_HOME \ + podman unshare --rootless-netns \ + socat TCP-LISTEN:8080,bind=0.0.0.0,reuseaddr,fork "UNIX-CONNECT:${relay_socket}" & +network_relay_pid=$! +sleep 1 +if ! kill -0 "${network_relay_pid}" 2>/dev/null; then + echo "ERROR: failed to start the rootless-network Podman gateway relay" >&2 + exit 1 +fi + +install -m 0555 "${supervisor_bin}" "${build_context}/openshell-sandbox" +cat >"${build_context}/Containerfile" <<'EOF' +FROM alpine:3.22 +RUN apk add --no-cache nftables iptables iptables-legacy +COPY --chmod=0555 openshell-sandbox /openshell-sandbox +ENTRYPOINT ["/openshell-sandbox"] +EOF + +echo "==> Building checkout supervisor image: ${supervisor_image}" +"${podman_remote[@]}" build \ + --pull=missing \ + --tag "${supervisor_image}" \ + --file "${build_context}/Containerfile" \ + "${build_context}" +"${podman_remote[@]}" image exists "${supervisor_image}" diff --git a/e2e/run.sh b/e2e/run.sh index 10155e1a77..e321084082 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -19,12 +19,14 @@ e2e_preserve_mise_dirs usage() { cat <<'EOF' Usage: - e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ + e2e/run.sh [--vm DISTRO] [--with CONFIG ...] [--guest-setup PATH] \ --gateway-config PATH --suite NAME Options: --vm DISTRO Run the gateway in a Nix test VM --with CONFIG Apply a Nix test-VM configuration; repeatable + --guest-setup PATH Run an executable setup script in the guest before the + gateway starts; requires VM mode --gateway-config PATH Fully resolved gateway TOML --suite NAME Host-side suite at e2e/suites/NAME.sh @@ -95,6 +97,8 @@ gateway_config= gateway_config_set=0 suite_name= suite_set=0 +guest_setup= +guest_setup_set=0 with_configurations=() while [ "$#" -gt 0 ]; do @@ -113,6 +117,15 @@ while [ "$#" -gt 0 ]; do with_configurations+=("$2") shift 2 ;; + --guest-setup) + require_value "$1" "$#" "${2:-}" + if [ "${guest_setup_set}" -eq 1 ]; then + die "--guest-setup may be supplied only once" + fi + guest_setup=$2 + guest_setup_set=1 + shift 2 + ;; --gateway-config) require_value "$1" "$#" "${2:-}" if [ "${gateway_config_set}" -eq 1 ]; then @@ -165,6 +178,12 @@ if [ ! -x "${suite_path}" ]; then die "suite is not executable: ${suite_path}" fi suite_path="$(resolve_file "${suite_path}")" +if [ "${guest_setup_set}" -eq 1 ]; then + guest_setup_source=${guest_setup} + if ! guest_setup="$(resolve_file "${guest_setup_source}")"; then + die "guest setup script does not exist: ${guest_setup_source}" + fi +fi mode=host if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then @@ -173,6 +192,9 @@ if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then vm=ubuntu fi fi +if [ "${guest_setup_set}" -eq 1 ] && [ "${mode}" != vm ]; then + die "--guest-setup requires VM mode" +fi if [ "${mode}" = vm ]; then if [[ ! ${vm} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then @@ -441,6 +463,10 @@ else runtime_log="${run_dir}/vm.log" guest_launcher="${run_dir}/launch-gateway.sh" guest_launcher_path="/home/openshell/.cache/openshell-e2e/bin/launch-gateway" + guest_setup_path= + if [ "${guest_setup_set}" -eq 1 ]; then + guest_setup_path="/home/openshell/.cache/openshell-e2e/bin/guest-setup" + fi config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" @@ -465,6 +491,10 @@ chmod 0600 "\${jwt_root}/signing.pem" "\${jwt_root}/public.pem" "\${jwt_root}/ki export XDG_CONFIG_HOME=\${state_root}/xdg/config export XDG_DATA_HOME=\${state_root}/xdg/data export XDG_STATE_HOME=\${state_root}/xdg/state +if [ -n '${guest_setup_path}' ]; then + echo "==> Running guest setup" + '${guest_setup_path}' +fi cd /home/openshell exec /usr/local/bin/openshell-gateway \ --config "\${config_path}" \ @@ -488,6 +518,9 @@ EOF --copy "${guest_launcher}:${guest_launcher_path}" --forward-port "${host_port}:${guest_port}" ) + if [ "${guest_setup_set}" -eq 1 ]; then + vm_args+=(--copy "${guest_setup}:${guest_setup_path}") + fi if [ "${keep}" -eq 1 ]; then vm_args+=(--keep) fi @@ -503,6 +536,7 @@ EOF child_pid=$! child_process_group=1 fi +export OPENSHELL_E2E_RUNTIME_LOG="${runtime_log}" wait_for_gateway() { local elapsed=0 diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b36a32203f..b1e53579c0 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -67,6 +67,11 @@ name = "podman_corporate_proxy" path = "tests/podman_corporate_proxy.rs" required-features = ["e2e-podman"] +[[test]] +name = "podman_shutdown" +path = "tests/podman_shutdown.rs" +required-features = ["e2e-podman"] + [[test]] name = "vm_gateway_resume" path = "tests/vm_gateway_resume.rs" diff --git a/e2e/rust/tests/podman_shutdown.rs b/e2e/rust/tests/podman_shutdown.rs new file mode 100644 index 0000000000..1da8bdaebf --- /dev/null +++ b/e2e/rust/tests/podman_shutdown.rs @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-podman")] + +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +const DELETE_ATTEMPTS: usize = 1; +const MAX_DELETE_DURATION: Duration = Duration::from_secs(55); + +#[tokio::test] +async fn podman_shutdown_transport_closes_are_expected() { + assert_eq!( + std::env::var("OPENSHELL_E2E_MODE").as_deref(), + Ok("vm"), + "Podman shutdown regression must run through the VM-backed runner" + ); + assert!( + std::env::var("OPENSHELL_E2E_WITH") + .unwrap_or_default() + .split_ascii_whitespace() + .any(|configuration| configuration == "podman"), + "Podman shutdown regression requires the Podman VM configuration" + ); + + for attempt in 1..=DELETE_ATTEMPTS { + let mut sandbox = SandboxGuard::create_keep( + &["sh", "-lc", "echo shutdown-ready; exec sleep infinity"], + "shutdown-ready", + ) + .await + .unwrap_or_else(|error| panic!("attempt {attempt}: sandbox create failed: {error}")); + let sandbox_name = sandbox.name.clone(); + + let started = Instant::now(); + let mut delete = openshell_cmd(); + delete + .args(["sandbox", "delete", &sandbox_name]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = delete + .output() + .await + .unwrap_or_else(|error| panic!("attempt {attempt}: failed to spawn delete: {error}")); + let elapsed = started.elapsed(); + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + + assert!( + output.status.success(), + "attempt {attempt}: sandbox delete failed after {elapsed:?} (exit {:?}):\n{combined}", + output.status.code() + ); + assert!( + elapsed < MAX_DELETE_DURATION, + "attempt {attempt}: sandbox delete took {elapsed:?}; expected Podman shutdown within the 45-second grace period plus overhead, in less than {MAX_DELETE_DURATION:?}" + ); + + // Reap the attached CLI process. Its best-effort second delete is a no-op. + sandbox.cleanup().await; + } +} diff --git a/e2e/suites/podman-shutdown.sh b/e2e/suites/podman-shutdown.sh new file mode 100755 index 0000000000..ed2ec48eef --- /dev/null +++ b/e2e/suites/podman-shutdown.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${OPENSHELL_E2E_RUNTIME_LOG:?runner must export OPENSHELL_E2E_RUNTIME_LOG}" + +cargo test \ + --manifest-path e2e/rust/Cargo.toml \ + --features e2e-podman \ + --test podman_shutdown \ + -- --nocapture + +# The base infrastructure suite validates lifecycle behavior. Dependent changes +# can opt into exact shutdown-transport classification assertions. +if [ "${OPENSHELL_E2E_EXPECT_SHUTDOWN_CLOSES:-0}" != 1 ]; then + exit 0 +fi + +# Give the VM gateway enough time to flush shutdown-session logs. +sleep 2 + +if grep -E \ + 'supervisor session: stream error|relay stream: inbound errored|StopSignal SIGTERM failed to stop container' \ + "${OPENSHELL_E2E_RUNTIME_LOG}"; then + echo "ERROR: shutdown emitted a forced-stop or transport-close warning" >&2 + exit 1 +fi + +for expected in \ + "supervisor session: expected transport close during teardown" \ + "relay stream: expected transport close during sandbox teardown"; do + if ! grep -Fq "${expected}" "${OPENSHELL_E2E_RUNTIME_LOG}"; then + echo "ERROR: shutdown log did not contain expected classification: ${expected}" >&2 + exit 1 + fi +done From dc30efab6afefac3c0665294d95877c0a1d1f125 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 13:49:26 -0700 Subject: [PATCH 17/29] fix(e2e): use renamed test guest app Signed-off-by: Drew Newberry --- e2e/run.sh | 20 ++++++++++---------- tasks/test.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index e321084082..5278854ad9 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Build the current checkout, run its gateway on the host or in a disposable -# Nix test VM, and execute one named host-side E2E suite against that gateway. +# Nix test guest, and execute one named host-side E2E suite against that gateway. set -Eeuo pipefail @@ -23,8 +23,8 @@ Usage: --gateway-config PATH --suite NAME Options: - --vm DISTRO Run the gateway in a Nix test VM - --with CONFIG Apply a Nix test-VM configuration; repeatable + --vm DISTRO Run the gateway in a Nix test guest + --with CONFIG Apply a Nix test-guest configuration; repeatable --guest-setup PATH Run an executable setup script in the guest before the gateway starts; requires VM mode --gateway-config PATH @@ -33,7 +33,7 @@ Options: -h, --help Show this help Omit --vm and --with to run the gateway on the host. Supplying --with without ---vm selects the Ubuntu test VM. Set OPENSHELL_E2E_KEEP=1 to retain state. +--vm selects the Ubuntu test guest. Set OPENSHELL_E2E_KEEP=1 to retain state. EOF } @@ -208,15 +208,15 @@ if [ "${mode}" = vm ]; then if ! command -v nix >/dev/null 2>&1; then die "Nix is required for VM mode" fi - if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-vm -- --list)"; then - die "failed to read the Nix test-VM catalog" + if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-guest -- --list)"; then + die "failed to read the Nix test-guest catalog" fi if ! catalog_has_entry "${vm_catalog}" Distros "${vm}"; then - die "unknown VM distro in the Nix test-VM catalog: ${vm}" + die "unknown VM distro in the Nix test-guest catalog: ${vm}" fi for configuration in "${with_configurations[@]}"; do if ! catalog_has_entry "${vm_catalog}" Configurations "${configuration}"; then - die "unknown VM configuration in the Nix test-VM catalog: ${configuration}" + die "unknown VM configuration in the Nix test-guest catalog: ${configuration}" fi done fi @@ -505,7 +505,7 @@ EOF chmod 0700 "${guest_launcher}" vm_args=( - nix run .#test-vm -- + nix run .#test-guest -- --distro "${vm}" ) for configuration in "${with_configurations[@]}"; do @@ -526,7 +526,7 @@ EOF fi vm_args+=(-- "${guest_launcher_path}") - echo "==> Starting ${vm} test VM gateway at ${gateway_endpoint}" + echo "==> Starting ${vm} test guest gateway at ${gateway_endpoint}" ( cd "${ROOT}" exec python3 -c \ diff --git a/tasks/test.toml b/tasks/test.toml index bb13690934..4f7657cfa4 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -38,7 +38,7 @@ description = "Run all end-to-end tests (Rust + Python + MCP)" depends = ["e2e:rust", "e2e:python", "e2e:mcp"] ["e2e:run"] -description = "Build the current checkout and run a named host or Nix test-VM E2E suite" +description = "Build the current checkout and run a named host or Nix test-guest E2E suite" run = "e2e/run.sh" ["e2e:gpu"] From 22cea85269b6fc1e1978c97cd5b7f78be8c55d05 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 13:52:40 -0700 Subject: [PATCH 18/29] chore(e2e): rename runner task Signed-off-by: Drew Newberry --- tasks/test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/test.toml b/tasks/test.toml index 4f7657cfa4..90e8c6f5e0 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -37,7 +37,7 @@ hide = true description = "Run all end-to-end tests (Rust + Python + MCP)" depends = ["e2e:rust", "e2e:python", "e2e:mcp"] -["e2e:run"] +["e2e:test"] description = "Build the current checkout and run a named host or Nix test-guest E2E suite" run = "e2e/run.sh" From 8e130f134a4938b19cc2a2872818d002cebbc49e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 14:44:23 -0700 Subject: [PATCH 19/29] fix(e2e): make guest smoke examples portable Signed-off-by: Drew Newberry --- e2e/guest-setup/podman-supervisor.sh | 7 +++--- e2e/run.sh | 37 ++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/e2e/guest-setup/podman-supervisor.sh b/e2e/guest-setup/podman-supervisor.sh index 303c1f029e..bacd0e8a5a 100755 --- a/e2e/guest-setup/podman-supervisor.sh +++ b/e2e/guest-setup/podman-supervisor.sh @@ -28,6 +28,10 @@ if [ -z "${XDG_RUNTIME_DIR:-}" ]; then echo "ERROR: XDG_RUNTIME_DIR is required to locate the rootless Podman socket" >&2 exit 1 fi +if ! command -v socat >/dev/null 2>&1; then + echo "ERROR: socat is required by the Podman guest configuration" >&2 + exit 1 +fi # Build through the user API service so the gateway driver sees the image in # the same Podman store even though the E2E launcher overrides XDG_DATA_HOME. @@ -36,9 +40,6 @@ podman_remote=(podman --url "unix://${XDG_RUNTIME_DIR}/podman/podman.sock") # Rootless netavark isolates its bridge from the VM host network namespace. # Relay the bridge gateway through a Unix socket so supervisors can call the # host gateway while the host-side E2E process still reaches its forwarded port. -if ! command -v socat >/dev/null 2>&1; then - sudo env DEBIAN_FRONTEND=noninteractive apt-get install -qq -y socat -fi relay_socket=/home/openshell/.cache/openshell-e2e/podman-gateway.sock rm -f "${relay_socket}" socat "UNIX-LISTEN:${relay_socket},fork" TCP:127.0.0.1:8080 & diff --git a/e2e/run.sh b/e2e/run.sh index 5278854ad9..208a979c51 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -479,9 +479,9 @@ umask 077 state_root=/home/openshell/.cache/openshell-e2e config_path=\${state_root}/gateway.toml jwt_root=\${state_root}/gateway-jwt -sudo chown "\$(id -u):\$(id -g)" "\${state_root}" +sudo chown -R "\$(id -u):\$(id -g)" /home/openshell/.cache chmod 0700 "\${state_root}" -mkdir -p "\${state_root}/xdg/config" "\${state_root}/xdg/data" "\${state_root}/xdg/state" "\${jwt_root}" +mkdir -p "\${state_root}/xdg/cache" "\${state_root}/xdg/config" "\${state_root}/xdg/data" "\${state_root}/xdg/state" "\${jwt_root}" printf '%s' '${config_payload}' | base64 --decode >"\${config_path}" printf '%s' '${jwt_signing_payload}' | base64 --decode >"\${jwt_root}/signing.pem" printf '%s' '${jwt_public_payload}' | base64 --decode >"\${jwt_root}/public.pem" @@ -489,6 +489,7 @@ printf '%s' '${jwt_kid_payload}' | base64 --decode >"\${jwt_root}/kid" chmod 0600 "\${config_path}" chmod 0600 "\${jwt_root}/signing.pem" "\${jwt_root}/public.pem" "\${jwt_root}/kid" export XDG_CONFIG_HOME=\${state_root}/xdg/config +export XDG_CACHE_HOME=\${state_root}/xdg/cache export XDG_DATA_HOME=\${state_root}/xdg/data export XDG_STATE_HOME=\${state_root}/xdg/state if [ -n '${guest_setup_path}' ]; then @@ -538,13 +539,40 @@ EOF fi export OPENSHELL_E2E_RUNTIME_LOG="${runtime_log}" +probe_gateway() { + python3 - "${OPENSHELL_BIN}" "${1}" <<'PY' +import os +import subprocess +import sys + +with open(sys.argv[2], "wb") as output: + try: + result = subprocess.run( + [sys.argv[1], "status"], + env={**os.environ, "NO_COLOR": "1"}, + stdout=output, + stderr=subprocess.STDOUT, + timeout=5, + check=False, + ) + except subprocess.TimeoutExpired: + raise SystemExit(124) +raise SystemExit(result.returncode) +PY +} + wait_for_gateway() { + local started_at=${SECONDS} local elapsed=0 local process_status local probe_log="${run_dir}/gateway-probe.log" echo "==> Waiting up to ${gateway_ready_timeout}s for gateway readiness" - while [ "${elapsed}" -lt "${gateway_ready_timeout}" ]; do + while :; do + elapsed=$((SECONDS - started_at)) + if [ "${elapsed}" -ge "${gateway_ready_timeout}" ]; then + break + fi if ! kill -0 "${child_pid}" 2>/dev/null; then wait "${child_pid}" process_status=$? @@ -555,13 +583,12 @@ wait_for_gateway() { fi return "${process_status}" fi - if NO_COLOR=1 "${OPENSHELL_BIN}" status >"${probe_log}" 2>&1 && + if probe_gateway "${probe_log}" && grep -q "Connected" "${probe_log}"; then echo "==> Gateway ready after ${elapsed}s" return 0 fi sleep 1 - elapsed=$((elapsed + 1)) done echo "ERROR: gateway did not become ready within ${gateway_ready_timeout}s" >&2 From 2d75951692ea43ac370ab551c2024792cf33cf14 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 16:45:42 -0700 Subject: [PATCH 20/29] test(e2e): remove Podman shutdown suite Signed-off-by: Drew Newberry --- e2e/rust/Cargo.toml | 5 --- e2e/rust/tests/podman_shutdown.rs | 70 ------------------------------- e2e/suites/podman-shutdown.sh | 38 ----------------- 3 files changed, 113 deletions(-) delete mode 100644 e2e/rust/tests/podman_shutdown.rs delete mode 100755 e2e/suites/podman-shutdown.sh diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b1e53579c0..b36a32203f 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -67,11 +67,6 @@ name = "podman_corporate_proxy" path = "tests/podman_corporate_proxy.rs" required-features = ["e2e-podman"] -[[test]] -name = "podman_shutdown" -path = "tests/podman_shutdown.rs" -required-features = ["e2e-podman"] - [[test]] name = "vm_gateway_resume" path = "tests/vm_gateway_resume.rs" diff --git a/e2e/rust/tests/podman_shutdown.rs b/e2e/rust/tests/podman_shutdown.rs deleted file mode 100644 index 1da8bdaebf..0000000000 --- a/e2e/rust/tests/podman_shutdown.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-podman")] - -use std::process::Stdio; -use std::time::{Duration, Instant}; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -const DELETE_ATTEMPTS: usize = 1; -const MAX_DELETE_DURATION: Duration = Duration::from_secs(55); - -#[tokio::test] -async fn podman_shutdown_transport_closes_are_expected() { - assert_eq!( - std::env::var("OPENSHELL_E2E_MODE").as_deref(), - Ok("vm"), - "Podman shutdown regression must run through the VM-backed runner" - ); - assert!( - std::env::var("OPENSHELL_E2E_WITH") - .unwrap_or_default() - .split_ascii_whitespace() - .any(|configuration| configuration == "podman"), - "Podman shutdown regression requires the Podman VM configuration" - ); - - for attempt in 1..=DELETE_ATTEMPTS { - let mut sandbox = SandboxGuard::create_keep( - &["sh", "-lc", "echo shutdown-ready; exec sleep infinity"], - "shutdown-ready", - ) - .await - .unwrap_or_else(|error| panic!("attempt {attempt}: sandbox create failed: {error}")); - let sandbox_name = sandbox.name.clone(); - - let started = Instant::now(); - let mut delete = openshell_cmd(); - delete - .args(["sandbox", "delete", &sandbox_name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = delete - .output() - .await - .unwrap_or_else(|error| panic!("attempt {attempt}: failed to spawn delete: {error}")); - let elapsed = started.elapsed(); - let combined = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )); - - assert!( - output.status.success(), - "attempt {attempt}: sandbox delete failed after {elapsed:?} (exit {:?}):\n{combined}", - output.status.code() - ); - assert!( - elapsed < MAX_DELETE_DURATION, - "attempt {attempt}: sandbox delete took {elapsed:?}; expected Podman shutdown within the 45-second grace period plus overhead, in less than {MAX_DELETE_DURATION:?}" - ); - - // Reap the attached CLI process. Its best-effort second delete is a no-op. - sandbox.cleanup().await; - } -} diff --git a/e2e/suites/podman-shutdown.sh b/e2e/suites/podman-shutdown.sh deleted file mode 100755 index ed2ec48eef..0000000000 --- a/e2e/suites/podman-shutdown.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -: "${OPENSHELL_E2E_RUNTIME_LOG:?runner must export OPENSHELL_E2E_RUNTIME_LOG}" - -cargo test \ - --manifest-path e2e/rust/Cargo.toml \ - --features e2e-podman \ - --test podman_shutdown \ - -- --nocapture - -# The base infrastructure suite validates lifecycle behavior. Dependent changes -# can opt into exact shutdown-transport classification assertions. -if [ "${OPENSHELL_E2E_EXPECT_SHUTDOWN_CLOSES:-0}" != 1 ]; then - exit 0 -fi - -# Give the VM gateway enough time to flush shutdown-session logs. -sleep 2 - -if grep -E \ - 'supervisor session: stream error|relay stream: inbound errored|StopSignal SIGTERM failed to stop container' \ - "${OPENSHELL_E2E_RUNTIME_LOG}"; then - echo "ERROR: shutdown emitted a forced-stop or transport-close warning" >&2 - exit 1 -fi - -for expected in \ - "supervisor session: expected transport close during teardown" \ - "relay stream: expected transport close during sandbox teardown"; do - if ! grep -Fq "${expected}" "${OPENSHELL_E2E_RUNTIME_LOG}"; then - echo "ERROR: shutdown log did not contain expected classification: ${expected}" >&2 - exit 1 - fi -done From f096dbf25f81756670eabb694f628007cffa8bb8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 17:14:36 -0700 Subject: [PATCH 21/29] refactor(e2e): use test guest Podman setup Signed-off-by: Drew Newberry --- e2e/guest-setup/podman-supervisor.sh | 79 ---------------------------- e2e/run.sh | 37 +------------ 2 files changed, 1 insertion(+), 115 deletions(-) delete mode 100755 e2e/guest-setup/podman-supervisor.sh diff --git a/e2e/guest-setup/podman-supervisor.sh b/e2e/guest-setup/podman-supervisor.sh deleted file mode 100755 index bacd0e8a5a..0000000000 --- a/e2e/guest-setup/podman-supervisor.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Build the supervisor image inside the disposable VM so the Podman driver uses -# the openshell-sandbox binary built from the checkout under test. - -set -euo pipefail - -supervisor_bin=/home/openshell/.cache/openshell-e2e/bin/openshell-sandbox -supervisor_image=localhost/openshell/supervisor:e2e-vm -build_context="$(mktemp -d)" - -cleanup() { - rm -rf "${build_context}" -} -trap cleanup EXIT - -if [ ! -x "${supervisor_bin}" ]; then - echo "ERROR: staged supervisor binary is missing: ${supervisor_bin}" >&2 - exit 1 -fi -if ! command -v podman >/dev/null 2>&1; then - echo "ERROR: Podman is required by the supervisor guest setup" >&2 - exit 1 -fi -if [ -z "${XDG_RUNTIME_DIR:-}" ]; then - echo "ERROR: XDG_RUNTIME_DIR is required to locate the rootless Podman socket" >&2 - exit 1 -fi -if ! command -v socat >/dev/null 2>&1; then - echo "ERROR: socat is required by the Podman guest configuration" >&2 - exit 1 -fi - -# Build through the user API service so the gateway driver sees the image in -# the same Podman store even though the E2E launcher overrides XDG_DATA_HOME. -podman_remote=(podman --url "unix://${XDG_RUNTIME_DIR}/podman/podman.sock") - -# Rootless netavark isolates its bridge from the VM host network namespace. -# Relay the bridge gateway through a Unix socket so supervisors can call the -# host gateway while the host-side E2E process still reaches its forwarded port. -relay_socket=/home/openshell/.cache/openshell-e2e/podman-gateway.sock -rm -f "${relay_socket}" -socat "UNIX-LISTEN:${relay_socket},fork" TCP:127.0.0.1:8080 & -host_relay_pid=$! -for _ in $(seq 1 50); do - [ -S "${relay_socket}" ] && break - sleep 0.1 -done -if [ ! -S "${relay_socket}" ] || ! kill -0 "${host_relay_pid}" 2>/dev/null; then - echo "ERROR: failed to start the host-side Podman gateway relay" >&2 - exit 1 -fi -env -u XDG_CONFIG_HOME -u XDG_DATA_HOME -u XDG_STATE_HOME \ - podman unshare --rootless-netns \ - socat TCP-LISTEN:8080,bind=0.0.0.0,reuseaddr,fork "UNIX-CONNECT:${relay_socket}" & -network_relay_pid=$! -sleep 1 -if ! kill -0 "${network_relay_pid}" 2>/dev/null; then - echo "ERROR: failed to start the rootless-network Podman gateway relay" >&2 - exit 1 -fi - -install -m 0555 "${supervisor_bin}" "${build_context}/openshell-sandbox" -cat >"${build_context}/Containerfile" <<'EOF' -FROM alpine:3.22 -RUN apk add --no-cache nftables iptables iptables-legacy -COPY --chmod=0555 openshell-sandbox /openshell-sandbox -ENTRYPOINT ["/openshell-sandbox"] -EOF - -echo "==> Building checkout supervisor image: ${supervisor_image}" -"${podman_remote[@]}" build \ - --pull=missing \ - --tag "${supervisor_image}" \ - --file "${build_context}/Containerfile" \ - "${build_context}" -"${podman_remote[@]}" image exists "${supervisor_image}" diff --git a/e2e/run.sh b/e2e/run.sh index 208a979c51..30d779c1a4 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -19,14 +19,12 @@ e2e_preserve_mise_dirs usage() { cat <<'EOF' Usage: - e2e/run.sh [--vm DISTRO] [--with CONFIG ...] [--guest-setup PATH] \ + e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ --gateway-config PATH --suite NAME Options: --vm DISTRO Run the gateway in a Nix test guest --with CONFIG Apply a Nix test-guest configuration; repeatable - --guest-setup PATH Run an executable setup script in the guest before the - gateway starts; requires VM mode --gateway-config PATH Fully resolved gateway TOML --suite NAME Host-side suite at e2e/suites/NAME.sh @@ -97,8 +95,6 @@ gateway_config= gateway_config_set=0 suite_name= suite_set=0 -guest_setup= -guest_setup_set=0 with_configurations=() while [ "$#" -gt 0 ]; do @@ -117,15 +113,6 @@ while [ "$#" -gt 0 ]; do with_configurations+=("$2") shift 2 ;; - --guest-setup) - require_value "$1" "$#" "${2:-}" - if [ "${guest_setup_set}" -eq 1 ]; then - die "--guest-setup may be supplied only once" - fi - guest_setup=$2 - guest_setup_set=1 - shift 2 - ;; --gateway-config) require_value "$1" "$#" "${2:-}" if [ "${gateway_config_set}" -eq 1 ]; then @@ -178,13 +165,6 @@ if [ ! -x "${suite_path}" ]; then die "suite is not executable: ${suite_path}" fi suite_path="$(resolve_file "${suite_path}")" -if [ "${guest_setup_set}" -eq 1 ]; then - guest_setup_source=${guest_setup} - if ! guest_setup="$(resolve_file "${guest_setup_source}")"; then - die "guest setup script does not exist: ${guest_setup_source}" - fi -fi - mode=host if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then mode=vm @@ -192,10 +172,6 @@ if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then vm=ubuntu fi fi -if [ "${guest_setup_set}" -eq 1 ] && [ "${mode}" != vm ]; then - die "--guest-setup requires VM mode" -fi - if [ "${mode}" = vm ]; then if [[ ! ${vm} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "invalid VM distro name: ${vm}" @@ -463,10 +439,6 @@ else runtime_log="${run_dir}/vm.log" guest_launcher="${run_dir}/launch-gateway.sh" guest_launcher_path="/home/openshell/.cache/openshell-e2e/bin/launch-gateway" - guest_setup_path= - if [ "${guest_setup_set}" -eq 1 ]; then - guest_setup_path="/home/openshell/.cache/openshell-e2e/bin/guest-setup" - fi config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" @@ -492,10 +464,6 @@ export XDG_CONFIG_HOME=\${state_root}/xdg/config export XDG_CACHE_HOME=\${state_root}/xdg/cache export XDG_DATA_HOME=\${state_root}/xdg/data export XDG_STATE_HOME=\${state_root}/xdg/state -if [ -n '${guest_setup_path}' ]; then - echo "==> Running guest setup" - '${guest_setup_path}' -fi cd /home/openshell exec /usr/local/bin/openshell-gateway \ --config "\${config_path}" \ @@ -519,9 +487,6 @@ EOF --copy "${guest_launcher}:${guest_launcher_path}" --forward-port "${host_port}:${guest_port}" ) - if [ "${guest_setup_set}" -eq 1 ]; then - vm_args+=(--copy "${guest_setup}:${guest_setup_path}") - fi if [ "${keep}" -eq 1 ]; then vm_args+=(--keep) fi From 5621f276738c48d6b5323bd1ff6e437d1a1b6a8d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 17:23:24 -0700 Subject: [PATCH 22/29] refactor(e2e): run Rust suites directly Signed-off-by: Drew Newberry --- e2e/run.sh | 15 ++++++++------- e2e/suites/smoke.sh | 11 ----------- 2 files changed, 8 insertions(+), 18 deletions(-) delete mode 100755 e2e/suites/smoke.sh diff --git a/e2e/run.sh b/e2e/run.sh index 30d779c1a4..5ff531f48e 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -27,7 +27,7 @@ Options: --with CONFIG Apply a Nix test-guest configuration; repeatable --gateway-config PATH Fully resolved gateway TOML - --suite NAME Host-side suite at e2e/suites/NAME.sh + --suite NAME Rust suite at e2e/rust/tests/NAME.rs -h, --help Show this help Omit --vm and --with to run the gateway on the host. Supplying --with without @@ -157,14 +157,10 @@ fi if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" fi -suite_path="${ROOT}/e2e/suites/${suite_name}.sh" +suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" if [ ! -f "${suite_path}" ]; then die "unknown suite: ${suite_name}" fi -if [ ! -x "${suite_path}" ]; then - die "suite is not executable: ${suite_path}" -fi -suite_path="$(resolve_file "${suite_path}")" mode=host if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then mode=vm @@ -574,8 +570,13 @@ if [ "${readiness_status}" -ne 0 ]; then fi echo "==> Running E2E suite: ${suite_name}" +cd "${ROOT}" set +e -"${suite_path}" +cargo test \ + --manifest-path e2e/rust/Cargo.toml \ + --features e2e \ + --test "${suite_name}" \ + -- --nocapture suite_status=$? set -e cleanup "${suite_status}" diff --git a/e2e/suites/smoke.sh b/e2e/suites/smoke.sh deleted file mode 100755 index 84757e70d5..0000000000 --- a/e2e/suites/smoke.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -exec cargo test \ - --manifest-path e2e/rust/Cargo.toml \ - --features e2e \ - --test smoke \ - -- --nocapture From bafd43ed5fc184d8403584cdcbbe642065e771c8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 17:46:17 -0700 Subject: [PATCH 23/29] refactor(e2e): simplify suite runner Signed-off-by: Drew Newberry --- e2e/run.sh | 155 +++++++++++++++++------------------------------------ 1 file changed, 50 insertions(+), 105 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index 5ff531f48e..6be85e50e3 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -90,22 +90,15 @@ catalog_has_entry() { } vm= -vm_set=0 gateway_config= -gateway_config_set=0 suite_name= -suite_set=0 with_configurations=() while [ "$#" -gt 0 ]; do case "$1" in --vm) require_value "$1" "$#" "${2:-}" - if [ "${vm_set}" -eq 1 ]; then - die "--vm may be supplied only once" - fi vm=$2 - vm_set=1 shift 2 ;; --with) @@ -115,20 +108,12 @@ while [ "$#" -gt 0 ]; do ;; --gateway-config) require_value "$1" "$#" "${2:-}" - if [ "${gateway_config_set}" -eq 1 ]; then - die "--gateway-config may be supplied only once" - fi gateway_config=$2 - gateway_config_set=1 shift 2 ;; --suite) require_value "$1" "$#" "${2:-}" - if [ "${suite_set}" -eq 1 ]; then - die "--suite may be supplied only once" - fi suite_name=$2 - suite_set=1 shift 2 ;; -h | --help) @@ -141,10 +126,10 @@ while [ "$#" -gt 0 ]; do esac done -if [ "${gateway_config_set}" -ne 1 ]; then +if [ -z "${gateway_config}" ]; then die "--gateway-config is required" fi -if [ "${suite_set}" -ne 1 ]; then +if [ -z "${suite_name}" ]; then die "--suite is required" fi if ! command -v python3 >/dev/null 2>&1; then @@ -162,9 +147,9 @@ if [ ! -f "${suite_path}" ]; then die "unknown suite: ${suite_name}" fi mode=host -if [ "${vm_set}" -eq 1 ] || [ "${#with_configurations[@]}" -gt 0 ]; then +if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then mode=vm - if [ "${vm_set}" -eq 0 ]; then + if [ -z "${vm}" ]; then vm=ubuntu fi fi @@ -180,6 +165,9 @@ if [ "${mode}" = vm ]; then if ! command -v nix >/dev/null 2>&1; then die "Nix is required for VM mode" fi + if ! command -v base64 >/dev/null 2>&1; then + die "base64 is required for VM mode" + fi if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-guest -- --list)"; then die "failed to read the Nix test-guest catalog" fi @@ -226,10 +214,9 @@ if [ -n "${CARGO_BUILD_JOBS:-}" ]; then fi cd "${ROOT}" -target_dir="$( - mise x -- cargo metadata --format-version=1 --no-deps | - python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])' -)" +target_dir="$(e2e_cargo_target_dir "${ROOT}")" + +ensure_build_nofile_limit echo "==> Building native host openshell CLI" mise x -- cargo build "${cargo_jobs[@]}" -p openshell-cli --bin openshell @@ -237,7 +224,6 @@ host_cli_bin="${target_dir}/debug/openshell" echo "==> Preparing ${linux_musl_target} build target" mise x -- rustup target add "${linux_musl_target}" >/dev/null -ensure_build_nofile_limit echo "==> Building Linux openshell-sandbox (${linux_musl_target})" mise x -- cargo zigbuild "${cargo_jobs[@]}" \ @@ -248,7 +234,6 @@ mise x -- cargo zigbuild "${cargo_jobs[@]}" \ linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" host_gateway_bin= -guest_cli_bin= guest_gateway_bin= if [ "${mode}" = host ]; then echo "==> Building native host openshell-gateway" @@ -258,17 +243,6 @@ if [ "${mode}" = host ]; then --features bundled-z3 host_gateway_bin="${target_dir}/debug/openshell-gateway" else - if ! command -v base64 >/dev/null 2>&1; then - die "base64 is required for VM mode" - fi - echo "==> Building Linux openshell CLI (${linux_musl_target})" - CXXSTDLIB=c++ mise x -- cargo zigbuild "${cargo_jobs[@]}" \ - --release \ - --target "${linux_musl_target}" \ - -p openshell-cli \ - --bin openshell - guest_cli_bin="${target_dir}/${linux_musl_target}/release/openshell" - echo "==> Preparing ${linux_gateway_rust_target} build target" mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" @@ -289,32 +263,25 @@ else guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" fi -for binary in "${host_cli_bin}" "${linux_sandbox_bin}"; do +expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}") +if [ "${mode}" = host ]; then + expected_binaries+=("${host_gateway_bin}") +else + expected_binaries+=("${guest_gateway_bin}") +fi +for binary in "${expected_binaries[@]}"; do if [ ! -x "${binary}" ]; then echo "ERROR: expected built binary at ${binary}" >&2 exit 1 fi done -if [ "${mode}" = host ] && [ ! -x "${host_gateway_bin}" ]; then - echo "ERROR: expected built gateway at ${host_gateway_bin}" >&2 - exit 1 -fi -if [ "${mode}" = vm ]; then - for binary in "${guest_cli_bin}" "${guest_gateway_bin}"; do - if [ ! -x "${binary}" ]; then - echo "ERROR: expected built guest binary at ${binary}" >&2 - exit 1 - fi - done -fi run_parent="${ROOT}/.cache/openshell-e2e/runs" mkdir -p "${run_parent}" run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")" child_pid= -child_process_group=0 runtime_log= -portable_jwt_dir="${ROOT}/.cache/openshell-e2e/gateway-jwt" +portable_jwt_dir= keep=0 if [ "${OPENSHELL_E2E_KEEP:-0}" = 1 ]; then keep=1 @@ -324,15 +291,11 @@ fi # shellcheck disable=SC2329 stop_child() { local pid=$1 - local process_group=$2 - local signal_target=${pid} + local signal_target="-${pid}" if [ -z "${pid}" ] || ! kill -0 "${pid}" 2>/dev/null; then return fi - if [ "${process_group}" -eq 1 ]; then - signal_target="-${pid}" - fi kill -TERM -- "${signal_target}" 2>/dev/null || true for _ in $(seq 1 30); do if ! kill -0 "${pid}" 2>/dev/null; then @@ -349,24 +312,25 @@ stop_child() { # Invoked by EXIT, INT, and TERM traps. # shellcheck disable=SC2329 cleanup() { - local incoming_status=$? - local status=${1:-${incoming_status}} + local status=$? trap - EXIT INT TERM - stop_child "${child_pid}" "${child_process_group}" + stop_child "${child_pid}" if [ "${status}" -ne 0 ] && [ -n "${runtime_log}" ] && [ -f "${runtime_log}" ]; then echo "=== ${mode} gateway log ===" >&2 cat "${runtime_log}" >&2 echo "=== end ${mode} gateway log ===" >&2 fi if [ "${keep}" -eq 1 ]; then - echo "Kept E2E wrapper state at ${run_dir}" >&2 + echo "Kept E2E runner state at ${run_dir}" >&2 else - rm -f \ - "${portable_jwt_dir}/signing.pem" \ - "${portable_jwt_dir}/public.pem" \ - "${portable_jwt_dir}/kid" - rmdir "${portable_jwt_dir}" 2>/dev/null || true + if [ -n "${portable_jwt_dir}" ]; then + rm -f \ + "${portable_jwt_dir}/signing.pem" \ + "${portable_jwt_dir}/public.pem" \ + "${portable_jwt_dir}/kid" + rmdir "${portable_jwt_dir}" 2>/dev/null || true + fi rm -rf "${run_dir}" fi exit "${status}" @@ -378,12 +342,15 @@ trap 'exit 143' TERM jwt_source_dir="${run_dir}/gateway-jwt" e2e_generate_gateway_jwt "${jwt_source_dir}" -mkdir -p "${portable_jwt_dir}" -install -m 0600 "${jwt_source_dir}/signing.pem" "${portable_jwt_dir}/signing.pem" -install -m 0600 "${jwt_source_dir}/public.pem" "${portable_jwt_dir}/public.pem" -install -m 0600 "${jwt_source_dir}/kid" "${portable_jwt_dir}/kid" +if [ "${mode}" = host ]; then + portable_jwt_dir="${ROOT}/.cache/openshell-e2e/gateway-jwt" + mkdir -p "${portable_jwt_dir}" + install -m 0600 "${jwt_source_dir}/signing.pem" "${portable_jwt_dir}/signing.pem" + install -m 0600 "${jwt_source_dir}/public.pem" "${portable_jwt_dir}/public.pem" + install -m 0600 "${jwt_source_dir}/kid" "${portable_jwt_dir}/kid" +fi -host_port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')" +host_port="$(e2e_pick_port)" guest_port= if [ "${mode}" = vm ]; then guest_port=8080 @@ -396,45 +363,33 @@ mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_STATE_HOME}" gateway_name="openshell-e2e-${mode}-${host_port}" gateway_endpoint="http://127.0.0.1:${host_port}" -e2e_register_plaintext_gateway \ - "${XDG_CONFIG_HOME}" \ - "${gateway_name}" \ - "${gateway_endpoint}" \ - "${host_port}" - -with_contract="${with_configurations[*]}" export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" export OPENSHELL_GATEWAY="${gateway_name}" export OPENSHELL_BIN="${host_cli_bin}" -export OPENSHELL_E2E_MODE="${mode}" -export OPENSHELL_E2E_VM="${vm}" -export OPENSHELL_E2E_WITH="${with_contract}" -export OPENSHELL_E2E_GATEWAY_CONFIG="${gateway_config}" -export OPENSHELL_E2E_HOST_PORT="${host_port}" -export OPENSHELL_E2E_GUEST_PORT="${guest_port}" - -staged_supervisor="${ROOT}/.cache/openshell-e2e/bin/openshell-sandbox" -mkdir -p "$(dirname "${staged_supervisor}")" -ln -sfn "${linux_sandbox_bin}" "${staged_supervisor}" if [ "${mode}" = host ]; then e2e_align_docker_host_with_cli_context + staged_supervisor="${ROOT}/.cache/openshell-e2e/bin/openshell-sandbox" + mkdir -p "$(dirname "${staged_supervisor}")" + ln -sfn "${linux_sandbox_bin}" "${staged_supervisor}" + runtime_log="${run_dir}/gateway.log" echo "==> Starting host gateway at ${gateway_endpoint}" ( cd "${ROOT}" - exec "${host_gateway_bin}" \ + exec python3 -c \ + 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ + "${host_gateway_bin}" \ --config "${gateway_config}" \ --bind-address 127.0.0.1 \ --port "${host_port}" \ --disable-tls ) >"${runtime_log}" 2>&1 & child_pid=$! - child_process_group=0 else runtime_log="${run_dir}/vm.log" guest_launcher="${run_dir}/launch-gateway.sh" - guest_launcher_path="/home/openshell/.cache/openshell-e2e/bin/launch-gateway" + guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" @@ -477,7 +432,6 @@ EOF vm_args+=(--with "${configuration}") done vm_args+=( - --copy "${guest_cli_bin}:/usr/local/bin/openshell" --copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway" --copy "${linux_sandbox_bin}:/home/openshell/.cache/openshell-e2e/bin/openshell-sandbox" --copy "${guest_launcher}:${guest_launcher_path}" @@ -496,9 +450,7 @@ EOF "${vm_args[@]}" ) >"${runtime_log}" 2>&1 & child_pid=$! - child_process_group=1 fi -export OPENSHELL_E2E_RUNTIME_LOG="${runtime_log}" probe_gateway() { python3 - "${OPENSHELL_BIN}" "${1}" <<'PY' @@ -535,8 +487,11 @@ wait_for_gateway() { break fi if ! kill -0 "${child_pid}" 2>/dev/null; then - wait "${child_pid}" - process_status=$? + if wait "${child_pid}"; then + process_status=0 + else + process_status=$? + fi child_pid= echo "ERROR: ${mode} gateway process exited before becoming ready" >&2 if [ "${process_status}" -eq 0 ]; then @@ -561,22 +516,12 @@ wait_for_gateway() { return 1 } -set +e wait_for_gateway -readiness_status=$? -set -e -if [ "${readiness_status}" -ne 0 ]; then - cleanup "${readiness_status}" -fi echo "==> Running E2E suite: ${suite_name}" cd "${ROOT}" -set +e cargo test \ --manifest-path e2e/rust/Cargo.toml \ --features e2e \ --test "${suite_name}" \ -- --nocapture -suite_status=$? -set -e -cleanup "${suite_status}" From b9931da84d7d32a7ea465f57171d94f0ecb64e5d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 21:14:42 -0700 Subject: [PATCH 24/29] refactor(e2e): isolate runner runtime state Signed-off-by: Drew Newberry --- e2e/run.sh | 63 ++++++++++++++++------------------- e2e/support/gateway-common.sh | 8 ++++- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/e2e/run.sh b/e2e/run.sh index 6be85e50e3..eebe38cc54 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -214,7 +214,7 @@ if [ -n "${CARGO_BUILD_JOBS:-}" ]; then fi cd "${ROOT}" -target_dir="$(e2e_cargo_target_dir "${ROOT}")" +target_dir="$(e2e_cargo_target_dir "${ROOT}" mise x -- cargo)" ensure_build_nofile_limit @@ -281,12 +281,25 @@ mkdir -p "${run_parent}" run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")" child_pid= runtime_log= -portable_jwt_dir= keep=0 if [ "${OPENSHELL_E2E_KEEP:-0}" = 1 ]; then keep=1 fi +start_child() { + local working_dir=$1 + local log_path=$2 + shift 2 + + ( + cd "${working_dir}" + exec python3 -c \ + 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ + "$@" + ) >"${log_path}" 2>&1 & + child_pid=$! +} + # Invoked by the EXIT trap through cleanup. # shellcheck disable=SC2329 stop_child() { @@ -324,13 +337,6 @@ cleanup() { if [ "${keep}" -eq 1 ]; then echo "Kept E2E runner state at ${run_dir}" >&2 else - if [ -n "${portable_jwt_dir}" ]; then - rm -f \ - "${portable_jwt_dir}/signing.pem" \ - "${portable_jwt_dir}/public.pem" \ - "${portable_jwt_dir}/kid" - rmdir "${portable_jwt_dir}" 2>/dev/null || true - fi rm -rf "${run_dir}" fi exit "${status}" @@ -341,14 +347,12 @@ trap 'exit 130' INT trap 'exit 143' TERM jwt_source_dir="${run_dir}/gateway-jwt" -e2e_generate_gateway_jwt "${jwt_source_dir}" +host_runtime_dir= if [ "${mode}" = host ]; then - portable_jwt_dir="${ROOT}/.cache/openshell-e2e/gateway-jwt" - mkdir -p "${portable_jwt_dir}" - install -m 0600 "${jwt_source_dir}/signing.pem" "${portable_jwt_dir}/signing.pem" - install -m 0600 "${jwt_source_dir}/public.pem" "${portable_jwt_dir}/public.pem" - install -m 0600 "${jwt_source_dir}/kid" "${portable_jwt_dir}/kid" + host_runtime_dir="${run_dir}/host-runtime" + jwt_source_dir="${host_runtime_dir}/.cache/openshell-e2e/gateway-jwt" fi +e2e_generate_gateway_jwt "${jwt_source_dir}" host_port="$(e2e_pick_port)" guest_port= @@ -369,23 +373,20 @@ export OPENSHELL_BIN="${host_cli_bin}" if [ "${mode}" = host ]; then e2e_align_docker_host_with_cli_context - staged_supervisor="${ROOT}/.cache/openshell-e2e/bin/openshell-sandbox" + staged_supervisor="${host_runtime_dir}/.cache/openshell-e2e/bin/openshell-sandbox" mkdir -p "$(dirname "${staged_supervisor}")" ln -sfn "${linux_sandbox_bin}" "${staged_supervisor}" runtime_log="${run_dir}/gateway.log" echo "==> Starting host gateway at ${gateway_endpoint}" - ( - cd "${ROOT}" - exec python3 -c \ - 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ - "${host_gateway_bin}" \ - --config "${gateway_config}" \ - --bind-address 127.0.0.1 \ - --port "${host_port}" \ - --disable-tls - ) >"${runtime_log}" 2>&1 & - child_pid=$! + start_child \ + "${host_runtime_dir}" \ + "${runtime_log}" \ + "${host_gateway_bin}" \ + --config "${gateway_config}" \ + --bind-address 127.0.0.1 \ + --port "${host_port}" \ + --disable-tls else runtime_log="${run_dir}/vm.log" guest_launcher="${run_dir}/launch-gateway.sh" @@ -443,13 +444,7 @@ EOF vm_args+=(-- "${guest_launcher_path}") echo "==> Starting ${vm} test guest gateway at ${gateway_endpoint}" - ( - cd "${ROOT}" - exec python3 -c \ - 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ - "${vm_args[@]}" - ) >"${runtime_log}" 2>&1 & - child_pid=$! + start_child "${ROOT}" "${runtime_log}" "${vm_args[@]}" fi probe_gateway() { diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index 6b9e6a0956..252cb9a6de 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -7,6 +7,12 @@ e2e_cargo_target_dir() { local root=$1 + shift + local cargo_command=(cargo) + + if [ "$#" -gt 0 ]; then + cargo_command=("$@") + fi if [ -n "${CARGO_TARGET_DIR:-}" ]; then case "${CARGO_TARGET_DIR}" in @@ -16,7 +22,7 @@ e2e_cargo_target_dir() { return 0 fi - cargo metadata --format-version=1 --no-deps \ + "${cargo_command[@]}" metadata --format-version=1 --no-deps \ | python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])' } From 294539a491bff42f64c6707acf70965320edfe2f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 12:51:00 -0700 Subject: [PATCH 25/29] ci(release): test Linux installs in VM guests Signed-off-by: Drew Newberry --- .agents/skills/test-release-canary/SKILL.md | 47 +++++- .github/workflows/release-canary.yml | 169 ++++++++------------ architecture/build.md | 6 + 3 files changed, 109 insertions(+), 113 deletions(-) diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 4bf7d38ae3..025818c5d2 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -12,11 +12,14 @@ The Release Canary (`.github/workflows/release-canary.yml`) smoke-tests the arti | Job | Runner | Verifies | |---|---|---| | `macos` | `macos-latest-xlarge` | `install.sh` resolves the Homebrew formula, brew installs the cask, and `openshell status` reaches the brew-services–backed local gateway with the VM driver. | -| `ubuntu` | `ubuntu-latest` | `install.sh` installs the Debian package, the post-install systemd user service starts, and `openshell status` reaches the local gateway with the Docker driver. | -| `fedora` | `fedora:latest` container | `install.sh` installs the RPM packages, the local gateway starts under Podman, and `openshell status` succeeds. | +| `linux-vm` (Ubuntu) | `ubuntu-24.04` / `ubuntu-24.04-arm` | The Nix VM harness boots Ubuntu 24.04 on AMD64 and ARM64, configures Docker, installs the rolling `dev` Debian packages, and verifies the local gateway. | +| `linux-vm` (Fedora) | `ubuntu-24.04` / `ubuntu-24.04-arm` | The Nix VM harness boots Fedora 44 on AMD64 and ARM64, configures rootless Podman with enforcing SELinux, installs the rolling `dev` RPM packages, and verifies the local gateway. | +| `ubuntu-snap` | `ubuntu-latest` | The Snap produced by the triggering Release Dev run installs with its required interfaces and reaches the Docker-backed gateway. | | `kubernetes` | `ubuntu-latest` + kind | `helm install oci://ghcr.io/nvidia/openshell/helm-chart --version 0.0.0-dev` succeeds in a kind cluster, the gateway pod becomes Ready, port-forward exposes 8080, and the released CLI registers the in-cluster gateway and runs `openshell status` against it. | -`install.sh` defaults to the *latest tagged* release — the canary is therefore checking that the most recent public release still installs, not the just-published `dev` build. The `kubernetes` job is the exception: it pins to `0.0.0-dev` chart + `:dev` images. +The Linux VM matrix sets `OPENSHELL_VERSION=dev`, so it tests the packages published by Release Dev. The Snap job downloads artifacts from the triggering run, and Kubernetes pins the `0.0.0-dev` chart and `:dev` images. The macOS job retains the installer's default latest-tagged release behavior. + +On Linux, the VM harness uses KVM when `/dev/kvm` is readable and writable. It falls back automatically to native-architecture QEMU TCG when KVM is unavailable, which is expected on public ARM64 runners. The workflow does not cross-emulate architectures. ## Trigger paths @@ -31,7 +34,7 @@ on: ``` - **Automatic.** Every successful `Release Dev` run (on `main` or a manual dispatch of Release Dev) fires the canary. Each job gates on `github.event.workflow_run.conclusion == 'success'` so a failed Release Dev does not run the canary. -- **Manual.** `workflow_dispatch` lets you run the canary on demand against any branch's workflow definition. +- **Manual.** `workflow_dispatch` lets you run the canary on demand against any branch's workflow definition. The Snap job is skipped because a manual run has no triggering Release Dev artifact. When dispatched manually, `github.event.workflow_run.head_sha` is empty and the workflow falls back to `github.sha` (the branch tip) for the `install.sh` URL. @@ -59,9 +62,36 @@ gh run view --log-failed ## Iterating on the canary itself -When you change `release-canary.yml` on a branch, a manual dispatch on that branch tests *your branch's workflow logic* against *main's published artifacts* (`0.0.0-dev` chart, `:dev` images, latest tagged install.sh assets). This is what you want for iterating on the canary — you're validating that the canary still works against known-good artifacts. +When you change `release-canary.yml` on a branch, a manual dispatch on that branch tests *your branch's workflow logic* against the floating published artifacts (`dev` Linux packages, `0.0.0-dev` chart, and `:dev` images). The macOS job continues to exercise the latest tagged Homebrew release. + +Note `install.sh` is pulled from `raw.githubusercontent.com/NVIDIA/OpenShell/${head_sha}/install.sh`, so changes to `install.sh` on your branch are exercised. The selected binary release follows the per-job behavior described above. + +## Local Linux VM reproduction + +Reproduce each native-architecture Linux canary with the same VM app: + +```shell +nix run .#test-vm -- \ + --distro ubuntu \ + --with docker \ + -- env \ + OPENSHELL_CANARY_DRIVER=docker \ + OPENSHELL_CANARY_INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/$(git rev-parse HEAD)/install.sh" \ + bash -lc ' + set -euo pipefail + mkdir -p "${HOME}/.config/openshell" + printf "OPENSHELL_DRIVERS=%s\n" "${OPENSHELL_CANARY_DRIVER}" \ + > "${HOME}/.config/openshell/gateway.env" + curl -LsSf "${OPENSHELL_CANARY_INSTALL_SH_URL}" | + OPENSHELL_VERSION=dev sh + openshell --version + openshell status + ' +``` -Note `install.sh` is pulled from `raw.githubusercontent.com/NVIDIA/OpenShell/${head_sha}/install.sh`, so changes to `install.sh` on your branch *are* exercised even though the binaries it downloads are from the latest public tag. +For the RPM path, replace `--distro ubuntu --with docker` with +`--distro fedora --with podman --with selinux` and set +`OPENSHELL_CANARY_DRIVER=podman`. ## Testing artifacts from a specific SHA @@ -107,8 +137,9 @@ Loopback registration auto-derives the gateway name to `openshell` if `--name` i | Symptom | Likely cause | Where to look | |---|---|---| -| `macos`/`ubuntu`/`fedora` job fails on `install.sh` | Latest tagged release missing an asset, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. | -| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_DRIVERS` env in the "Ensure …" step. | +| `macos`/`linux-vm` job fails on `install.sh` | Published release asset missing, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. | +| `linux-vm` fails before SSH | Cloud-image download, QEMU startup, or guest boot failure. | The runner prints its QEMU and serial logs automatically on failure; check whether it selected KVM or TCG. | +| `macos`/`linux-vm` job fails on `openshell status` | Local gateway service did not start (systemd/brew/Docker/Podman). Often a driver issue. | Installer diagnostics and the configured `OPENSHELL_DRIVERS` value in the job log. | | `kubernetes` job fails on `helm install --wait` | Chart did not deploy in 5 min — usually image pull failure or readiness probe failing. | "Diagnostics on failure" step dumps `helm status`, manifest, pod describe, pod logs. | | `kubernetes` job fails on `kubectl wait` | Gateway pod stuck `CrashLoopBackOff` or `ImagePullBackOff`. | Diagnostics dump; check `:dev` image existence at `ghcr.io/nvidia/openshell/gateway`. | | `kubernetes` job fails on `openshell gateway add` or `status` | Port-forward not reachable, or CLI/gateway proto mismatch. | `port-forward.log` and `openshell gateway list` in the diagnostics dump. | diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 896d12d190..b009e2a0d8 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -30,120 +30,79 @@ jobs: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh openshell status - ubuntu: - name: Ubuntu Docker + linux-vm: + name: Linux VM (${{ matrix.distro }}, ${{ matrix.architecture }}) if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - timeout-minutes: 20 + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + architecture: amd64 + distro: ubuntu + driver: docker + configurations: docker + - runner: ubuntu-24.04-arm + architecture: arm64 + distro: ubuntu + driver: docker + configurations: docker + - runner: ubuntu-24.04 + architecture: amd64 + distro: fedora + driver: podman + configurations: podman selinux + - runner: ubuntu-24.04-arm + architecture: arm64 + distro: fedora + driver: podman + configurations: podman selinux steps: - - name: Ensure Docker - run: | - if ! command -v docker >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y docker.io - fi - sudo systemctl start docker || sudo service docker start - mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\n' > "${HOME}/.config/openshell/gateway.env" - docker info + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + persist-credentials: false - - name: Install and check status - run: | - curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh - openshell status + - name: Install Nix + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 + with: + enable_kvm: true + github_access_token: ${{ github.token }} - fedora: - name: Fedora RPM - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - runs-on: linux-amd64-cpu8 - timeout-minutes: 20 - env: - FEDORA_CANARY_CONTAINER: openshell-fedora-canary-${{ github.run_id }}-${{ github.run_attempt }} - steps: - - name: Start Fedora systemd container and root user manager + - name: Install dev release and check status + env: + TEST_VM_CONFIGURATIONS: ${{ matrix.configurations }} + TEST_VM_DISTRO: ${{ matrix.distro }} + TEST_VM_DRIVER: ${{ matrix.driver }} + TEST_VM_INSTALL_SH_URL: https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh run: | set -euo pipefail - docker run --detach \ - --name "${FEDORA_CANARY_CONTAINER}" \ - --privileged \ - --cgroupns=host \ - --tmpfs /run \ - --tmpfs /tmp \ - --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \ - fedora:latest \ - bash -lc 'dnf install -y curl dbus-daemon podman systemd && exec /usr/sbin/init' - - for _ in $(seq 1 120); do - if docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then - break - fi - if [ "$(docker inspect -f '{{.State.Running}}' "${FEDORA_CANARY_CONTAINER}")" != "true" ]; then - echo "::error::Fedora systemd container exited before systemd became reachable" - docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 - done - - if ! docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then - echo "::error::Fedora systemd container did not become reachable within 120s" - docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true - exit 1 - fi - - docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \ - HOME=/root \ - XDG_RUNTIME_DIR=/run/user/0 \ - DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ - bash -s <<'EOF' - set -euo pipefail - # install.sh manages the RPM gateway as a systemd user unit. This - # container is booted with systemd as PID 1, but it still has no - # login session. Start root's user manager explicitly so the - # installer can test service restart and gateway registration - # instead of its "restart later" fallback. - mkdir -p "${XDG_RUNTIME_DIR}" - chmod 700 "${XDG_RUNTIME_DIR}" - systemctl start user-runtime-dir@0.service || true - systemctl start user@0.service - - for _ in $(seq 1 30); do - if systemctl --user daemon-reload; then - break - fi - sleep 1 + read -r -a configurations <<< "${TEST_VM_CONFIGURATIONS}" + configuration_args=() + for configuration in "${configurations[@]}"; do + configuration_args+=(--with "${configuration}") done - if ! systemctl --user daemon-reload; then - systemctl status user@0.service --no-pager >&2 || true - journalctl -u user@0.service --no-pager -n 80 >&2 || true - systemctl --user status --no-pager >&2 || true - exit 1 - fi - EOF - - name: Install and check status - run: | - set -euo pipefail - - docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \ - HOME=/root \ - XDG_RUNTIME_DIR=/run/user/0 \ - DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ - INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh" \ - bash -s <<'EOF' - set -euo pipefail - mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=podman\n' > "${HOME}/.config/openshell/gateway.env" - podman info - curl -LsSf "${INSTALL_SH_URL}" | sh - openshell status - EOF - - - name: Stop Fedora systemd container - if: always() - run: | - docker rm -f "${FEDORA_CANARY_CONTAINER}" >/dev/null 2>&1 || true + nix run .#test-vm -- \ + --distro "${TEST_VM_DISTRO}" \ + "${configuration_args[@]}" \ + -- \ + env \ + "OPENSHELL_CANARY_DRIVER=${TEST_VM_DRIVER}" \ + "OPENSHELL_CANARY_INSTALL_SH_URL=${TEST_VM_INSTALL_SH_URL}" \ + bash -lc ' + set -euo pipefail + mkdir -p "${HOME}/.config/openshell" + printf "OPENSHELL_DRIVERS=%s\n" "${OPENSHELL_CANARY_DRIVER}" \ + > "${HOME}/.config/openshell/gateway.env" + curl -LsSf "${OPENSHELL_CANARY_INSTALL_SH_URL}" | + OPENSHELL_VERSION=dev sh + openshell --version + openshell status + ' ubuntu-snap: name: Ubuntu Snap diff --git a/architecture/build.md b/architecture/build.md index c64ecf7c39..fb563a4c11 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -177,6 +177,12 @@ The high-level CI model: 5. Gate jobs verify that the mirror branch matches the PR head, or that the merge-group workflow ran for the queued SHA, and that the expected non-gate workflow actually ran. 6. Release workflows rebuild and publish binaries, wheels, images, and docs. +After each successful Release Dev run, the release canary smoke-tests published +artifacts. Linux package installs run in native-architecture Ubuntu and Fedora +QEMU guests on public AMD64 and ARM64 GitHub runners. The VM harness uses KVM +when available and falls back to TCG otherwise. Separate jobs retain macOS +Homebrew, Ubuntu Snap, and kind-based Helm coverage. + See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. ## Docs Site From 09390c13f84e7ebd7bf7ed2c2480cf6c22071a5f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 14:38:23 -0700 Subject: [PATCH 26/29] ci(release): benchmark arm64 TCG across Linux hosts Signed-off-by: Drew Newberry --- .agents/skills/test-release-canary/SKILL.md | 15 +++++- .github/workflows/release-canary.yml | 52 +++++++++++++++++++-- architecture/build.md | 5 ++ flake.nix | 13 +++++- nix/test-guest/README.md | 12 ++--- nix/test-guest/default.nix | 30 ++++++++++-- nix/test-guest/run.sh | 2 + 7 files changed, 113 insertions(+), 16 deletions(-) diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 025818c5d2..fcd568288b 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -19,7 +19,7 @@ The Release Canary (`.github/workflows/release-canary.yml`) smoke-tests the arti The Linux VM matrix sets `OPENSHELL_VERSION=dev`, so it tests the packages published by Release Dev. The Snap job downloads artifacts from the triggering run, and Kubernetes pins the `0.0.0-dev` chart and `:dev` images. The macOS job retains the installer's default latest-tagged release behavior. -On Linux, the VM harness uses KVM when `/dev/kvm` is readable and writable. It falls back automatically to native-architecture QEMU TCG when KVM is unavailable, which is expected on public ARM64 runners. The workflow does not cross-emulate architectures. +On Linux, the VM harness uses KVM when `/dev/kvm` is readable and writable. It falls back automatically to native-architecture QEMU TCG when KVM is unavailable, which is expected on public ARM64 runners. Normal canaries do not cross-emulate architectures. The manual `arm64_tcg_benchmark` mode is the exception: it boots the same pinned ARM64 Ubuntu image with TCG on x86_64 and ARM64 Linux hosts. ## Trigger paths @@ -28,6 +28,10 @@ The workflow has two triggers: ```yaml on: workflow_dispatch: + inputs: + arm64_tcg_benchmark: + type: boolean + default: false workflow_run: workflows: ["Release Dev"] types: [completed] @@ -35,6 +39,7 @@ on: - **Automatic.** Every successful `Release Dev` run (on `main` or a manual dispatch of Release Dev) fires the canary. Each job gates on `github.event.workflow_run.conclusion == 'success'` so a failed Release Dev does not run the canary. - **Manual.** `workflow_dispatch` lets you run the canary on demand against any branch's workflow definition. The Snap job is skipped because a manual run has no triggering Release Dev artifact. +- **ARM64 TCG benchmark.** A manual dispatch with `arm64_tcg_benchmark=true` skips the artifact canaries and boots the pinned ARM64 Ubuntu guest on both public Linux host architectures. Use it to compare QEMU/TCG startup without involving macOS. When dispatched manually, `github.event.workflow_run.head_sha` is empty and the workflow falls back to `github.sha` (the branch tip) for the `install.sh` URL. @@ -46,6 +51,14 @@ Run the canary as-is on the current branch: gh workflow run release-canary.yml --ref "$(git branch --show-current)" ``` +Run only the ARM64 TCG boot benchmark: + +```shell +gh workflow run release-canary.yml \ + --ref "$(git branch --show-current)" \ + -f arm64_tcg_benchmark=true +``` + Watch the run that starts: ```shell diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index b009e2a0d8..ad8dc6795e 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -2,6 +2,12 @@ name: Release Canary on: workflow_dispatch: + inputs: + arm64_tcg_benchmark: + description: Boot the same ARM64 Ubuntu guest with TCG on x86_64 and ARM64 Linux hosts + required: false + default: false + type: boolean workflow_run: workflows: ["Release Dev"] types: [completed] @@ -17,7 +23,7 @@ defaults: jobs: macos: name: macOS Homebrew - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.arm64_tcg_benchmark) || github.event.workflow_run.conclusion == 'success' }} runs-on: macos-latest-xlarge timeout-minutes: 20 steps: @@ -32,7 +38,7 @@ jobs: linux-vm: name: Linux VM (${{ matrix.distro }}, ${{ matrix.architecture }}) - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.arm64_tcg_benchmark) || github.event.workflow_run.conclusion == 'success' }} runs-on: ${{ matrix.runner }} timeout-minutes: 90 strategy: @@ -86,6 +92,8 @@ jobs: configuration_args+=(--with "${configuration}") done + # The variables in the single-quoted script expand inside the guest. + # shellcheck disable=SC2016 nix run .#test-vm -- \ --distro "${TEST_VM_DISTRO}" \ "${configuration_args[@]}" \ @@ -104,6 +112,44 @@ jobs: openshell status ' + arm64-tcg-benchmark: + name: ARM64 TCG boot (${{ matrix.host_architecture }} host) + if: ${{ github.event_name == 'workflow_dispatch' && inputs.arm64_tcg_benchmark }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + host_architecture: x86_64 + - runner: ubuntu-24.04-arm + host_architecture: aarch64 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Install Nix + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 + with: + enable_kvm: false + github_access_token: ${{ github.token }} + + - name: Report host + run: | + set -euo pipefail + uname -a + lscpu + + - name: Boot pinned ARM64 Ubuntu guest with TCG + run: | + set -euo pipefail + nix run .#test-vm-arm64-tcg -- \ + --distro ubuntu \ + -- true + ubuntu-snap: name: Ubuntu Snap if: ${{ github.event.workflow_run.conclusion == 'success' }} @@ -156,7 +202,7 @@ jobs: kubernetes: name: Kubernetes Helm (kind) - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.arm64_tcg_benchmark) || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest timeout-minutes: 20 env: diff --git a/architecture/build.md b/architecture/build.md index fb563a4c11..311b121d64 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -183,6 +183,11 @@ QEMU guests on public AMD64 and ARM64 GitHub runners. The VM harness uses KVM when available and falls back to TCG otherwise. Separate jobs retain macOS Homebrew, Ubuntu Snap, and kind-based Helm coverage. +A manual `arm64_tcg_benchmark` dispatch mode skips the artifact canaries and +boots the same pinned ARM64 Ubuntu cloud image under QEMU TCG on public x86_64 +and ARM64 Linux runners. This isolates host-dependent TCG boot behavior before +the production ARM64 canary strategy changes. + See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. ## Docs Site diff --git a/flake.nix b/flake.nix index 08943cbb60..c99da271b2 100644 --- a/flake.nix +++ b/flake.nix @@ -38,10 +38,19 @@ programs.nixfmt.enable = true; }; testGuest = import ./nix/test-guest { inherit pkgs; }; + testGuestArm64Tcg = import ./nix/test-guest { + inherit pkgs; + accelerator = "tcg"; + architecture = "aarch64"; + useQemuFirmware = true; + }; in { - apps.test-guest = testGuest.app; - apps.test-guest-cache = testGuest.cacheApp; + apps = { + test-guest = testGuest.app; + test-guest-cache = testGuest.cacheApp; + } + // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { devShells.default = pkgs.mkShell { packages = with pkgs; [ diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index c52cc0b28e..ced9a3e61c 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -5,14 +5,14 @@ SPDX-License-Identifier: Apache-2.0 # Test Guests -This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. +This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. A separate Linux-only app boots the pinned ARM64 guest with TCG on either an x86_64 or ARM64 host for performance comparisons. ## Requirements - Nix with flakes enabled. -- Apple Silicon macOS with HVF, or a native-architecture Linux host. Linux uses KVM when `/dev/kvm` is available and falls back to QEMU TCG otherwise. +- Apple Silicon macOS with HVF, or a Linux host. The default app uses KVM when `/dev/kvm` is available and falls back to QEMU TCG otherwise. - Enough local capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay. -- Native-architecture artifacts. TCG emulates the guest CPU on Linux but does not enable cross-architecture guests. +- Artifacts matching the guest architecture. The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay. @@ -46,7 +46,7 @@ nix/test-guest/ - `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments. - `README.md` documents the supported combinations and developer interface. -The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-guest` and `test-guest-cache` apps. Debian artifact creation remains outside the guest harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist. +The root [`flake.nix`](../../flake.nix) exposes this directory as the native-architecture `test-guest` app, the `test-guest-cache` app, and the Linux-only `test-guest-arm64-tcg` app. Debian artifact creation remains outside the guest harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist. ## Supported configurations @@ -197,7 +197,7 @@ nix run .#test-guest -- \ -- openshell --version ``` -For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the host and guest architecture. +For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the guest architecture. `--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS, Fedora, and Rocky Linux. This prototype can install an existing RPM but does not build one. @@ -250,7 +250,7 @@ Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log f ## Current limitations -- Host and guest architectures must match. +- The default `test-guest` app uses a guest matching the host architecture. `test-guest-arm64-tcg` is the supported cross-architecture exception on Linux. - TCG is slower than hardware virtualization and uses a longer SSH readiness timeout. - Prepared cache entries are architecture-specific and match the exact ordered configuration list. - OCI pulls transfer a complete compressed standalone disk; incremental disk layers are not implemented. diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index f34fb062aa..444b4441e6 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -3,15 +3,37 @@ # PROTOTYPE: Composable distro VMs for installing and exercising artifacts. -{ pkgs }: +{ + pkgs, + architecture ? if pkgs.stdenv.hostPlatform.isAarch64 then "aarch64" else "x86_64", + accelerator ? null, + useQemuFirmware ? false, +}: let - isAarch64 = pkgs.stdenv.hostPlatform.isAarch64; + hostArchitecture = if pkgs.stdenv.hostPlatform.isAarch64 then "aarch64" else "x86_64"; + isAarch64 = architecture == "aarch64"; isDarwin = pkgs.stdenv.hostPlatform.isDarwin; - architecture = if isAarch64 then "aarch64" else "x86_64"; - qemu = pkgs.qemu.override { hostCpuOnly = true; }; + guestMatchesHost = architecture == hostArchitecture; + qemu = pkgs.qemu.override { hostCpuOnly = guestMatchesHost; }; qemuBinary = if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; + selectedAccelerator = + if accelerator != null then + accelerator + else if isDarwin then + if guestMatchesHost then "hvf" else "tcg" + else if guestMatchesHost then + "kvm" + else + "tcg"; + firmwareCode = + if useQemuFirmware && isAarch64 then + "${qemu}/share/qemu/edk2-aarch64-code.fd" + else + pkgs.OVMF.firmware; + firmwareVars = + if useQemuFirmware && isAarch64 then "${qemu}/share/qemu/edk2-arm-vars.fd" else pkgs.OVMF.variables; distros = { ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index cee6afbe4f..387ee9464a 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -376,6 +376,7 @@ fi cp "${TEST_GUEST_FIRMWARE_VARS}" "${vars}" chmod 0600 "${vars}" +boot_started_seconds=${SECONDS} for attempt in $(seq 1 5); do if [ -n "${requested_ssh_port}" ]; then ssh_port=${requested_ssh_port} @@ -485,6 +486,7 @@ if [ "${ssh_ready}" -ne 1 ]; then echo "SSH did not become ready within ${ssh_wait_seconds} seconds" >&2 exit 1 fi +echo "==> SSH ready after $((SECONDS - boot_started_seconds)) seconds" echo "==> Validating ${distro}" # Profile values come from the trusted Nix-generated catalog. From a102a49c016382fee42a6ceea4ed1ec72fa82e6a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Jul 2026 15:22:10 -0700 Subject: [PATCH 27/29] fix(vm): use QEMU firmware for ARM TCG fallback Signed-off-by: Drew Newberry --- .agents/skills/test-release-canary/SKILL.md | 2 +- architecture/build.md | 10 ++++++---- nix/test-guest/README.md | 4 ++-- nix/test-guest/run.sh | 7 +++++++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index fcd568288b..03fff93892 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -19,7 +19,7 @@ The Release Canary (`.github/workflows/release-canary.yml`) smoke-tests the arti The Linux VM matrix sets `OPENSHELL_VERSION=dev`, so it tests the packages published by Release Dev. The Snap job downloads artifacts from the triggering run, and Kubernetes pins the `0.0.0-dev` chart and `:dev` images. The macOS job retains the installer's default latest-tagged release behavior. -On Linux, the VM harness uses KVM when `/dev/kvm` is readable and writable. It falls back automatically to native-architecture QEMU TCG when KVM is unavailable, which is expected on public ARM64 runners. Normal canaries do not cross-emulate architectures. The manual `arm64_tcg_benchmark` mode is the exception: it boots the same pinned ARM64 Ubuntu image with TCG on x86_64 and ARM64 Linux hosts. +On Linux, the VM harness uses KVM when `/dev/kvm` is readable and writable. It falls back automatically to native-architecture QEMU TCG when KVM is unavailable, which is expected on public ARM64 runners. The ARM64 fallback switches from the default firmware to QEMU's bundled EDK2 firmware, which is validated under TCG. Normal canaries do not cross-emulate architectures. The manual `arm64_tcg_benchmark` mode is the exception: it boots the same pinned ARM64 Ubuntu image with TCG on x86_64 and ARM64 Linux hosts. ## Trigger paths diff --git a/architecture/build.md b/architecture/build.md index 311b121d64..c5d3918486 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -180,13 +180,15 @@ The high-level CI model: After each successful Release Dev run, the release canary smoke-tests published artifacts. Linux package installs run in native-architecture Ubuntu and Fedora QEMU guests on public AMD64 and ARM64 GitHub runners. The VM harness uses KVM -when available and falls back to TCG otherwise. Separate jobs retain macOS -Homebrew, Ubuntu Snap, and kind-based Helm coverage. +when available and falls back to TCG otherwise. The ARM64 fallback uses QEMU's +bundled EDK2 firmware, which is validated under TCG on both public Linux host +architectures. Separate jobs retain macOS Homebrew, Ubuntu Snap, and kind-based +Helm coverage. A manual `arm64_tcg_benchmark` dispatch mode skips the artifact canaries and boots the same pinned ARM64 Ubuntu cloud image under QEMU TCG on public x86_64 -and ARM64 Linux runners. This isolates host-dependent TCG boot behavior before -the production ARM64 canary strategy changes. +and ARM64 Linux runners. The comparison keeps the native ARM64 runner as the +production host while verifying that x86_64 TCG remains a viable fallback. See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index ced9a3e61c..75562c1687 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # Test Guests -This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. A separate Linux-only app boots the pinned ARM64 guest with TCG on either an x86_64 or ARM64 host for performance comparisons. +This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable. The ARM64 fallback switches to QEMU's bundled EDK2 firmware, which is validated with TCG. A separate Linux-only app boots the pinned ARM64 guest with TCG on either an x86_64 or ARM64 host for performance comparisons. ## Requirements @@ -237,7 +237,7 @@ Arguments after `--` are executed inside the guest. Without a command, the runne Each invocation checks for an exact prepared local cache entry. On a miss, Nix realizes the hash-pinned cloud image and the runner applies the selected configurations. It then: 1. Creates a temporary QCOW2 overlay backed by the prepared cache disk or pinned cloud image. -2. Boots QEMU with HVF, KVM, or the Linux TCG fallback. +2. Boots QEMU with HVF, KVM, or the Linux TCG fallback. Native ARM64 Linux guests use QEMU's bundled EDK2 firmware when KVM is unavailable. 3. Creates a fresh cloud-init instance and ephemeral SSH key. 4. Applies the selected Ansible configurations only when the base is not prepared. 5. Installs or copies the supplied artifacts. diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 387ee9464a..874dcd0773 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -245,6 +245,13 @@ if [ "${TEST_GUEST_ACCELERATOR}" = kvm ] && TEST_GUEST_ACCELERATOR=tcg test_vm_cpu=max ssh_wait_seconds=600 + if [ "${TEST_VM_ARCHITECTURE}" = aarch64 ]; then + : "${TEST_VM_TCG_FIRMWARE_CODE:?missing QEMU ARM TCG firmware code}" + : "${TEST_VM_TCG_FIRMWARE_VARS:?missing QEMU ARM TCG firmware variables}" + TEST_VM_FIRMWARE_CODE=${TEST_VM_TCG_FIRMWARE_CODE} + TEST_VM_FIRMWARE_VARS=${TEST_VM_TCG_FIRMWARE_VARS} + echo "==> Using QEMU-bundled ARM firmware for TCG fallback" + fi fi # shellcheck disable=SC1090 From 6d7798720ee3463fa8a47e7363b7e4b8d6f39340 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 12:03:28 -0700 Subject: [PATCH 28/29] fix(ci): use renamed test guest apps Signed-off-by: Drew Newberry --- .agents/skills/test-release-canary/SKILL.md | 2 +- .github/workflows/release-canary.yml | 4 ++-- nix/test-guest/README.md | 12 ++++++++++ nix/test-guest/default.nix | 10 ++++++--- nix/test-guest/run.sh | 25 ++++++++++++++------- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 03fff93892..493578af7e 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -84,7 +84,7 @@ Note `install.sh` is pulled from `raw.githubusercontent.com/NVIDIA/OpenShell/${h Reproduce each native-architecture Linux canary with the same VM app: ```shell -nix run .#test-vm -- \ +nix run .#test-guest -- \ --distro ubuntu \ --with docker \ -- env \ diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index ad8dc6795e..e78f2b959e 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -94,7 +94,7 @@ jobs: # The variables in the single-quoted script expand inside the guest. # shellcheck disable=SC2016 - nix run .#test-vm -- \ + nix run .#test-guest -- \ --distro "${TEST_VM_DISTRO}" \ "${configuration_args[@]}" \ -- \ @@ -146,7 +146,7 @@ jobs: - name: Boot pinned ARM64 Ubuntu guest with TCG run: | set -euo pipefail - nix run .#test-vm-arm64-tcg -- \ + nix run .#test-guest-arm64-tcg -- \ --distro ubuntu \ -- true diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 75562c1687..748385f1c0 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -63,6 +63,18 @@ List the available distros and configurations: nix run .#test-guest -- --list ``` +## Benchmark ARM64 TCG across Linux hosts + +Boot the pinned ARM64 Ubuntu image with QEMU TCG, validate the guest over SSH, and shut it down: + +```shell +nix run .#test-guest-arm64-tcg -- \ + --distro ubuntu \ + -- true +``` + +Run the same command on x86_64 and ARM64 Linux hosts to compare TCG boot behavior. The app holds the ARM64 cloud image, QEMU version, firmware, vCPU count, and memory constant. The runner prints the elapsed time from QEMU startup until SSH becomes ready. + ## Open an interactive VM Boot a base Ubuntu VM: diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index 444b4441e6..cb1a9571f6 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -95,10 +95,14 @@ let export OPENSHELL_TEST_GUEST_RUNNER=${./run.sh} export TEST_GUEST_BASH=${pkgs.bash}/bin/bash export TEST_GUEST_QEMU=${qemuBinary} - export TEST_GUEST_FIRMWARE_CODE=${pkgs.OVMF.firmware} - export TEST_GUEST_FIRMWARE_VARS=${pkgs.OVMF.variables} + export TEST_GUEST_FIRMWARE_CODE=${firmwareCode} + export TEST_GUEST_FIRMWARE_VARS=${firmwareVars} + ${pkgs.lib.optionalString isAarch64 '' + export TEST_GUEST_TCG_FIRMWARE_CODE=${qemu}/share/qemu/edk2-aarch64-code.fd + export TEST_GUEST_TCG_FIRMWARE_VARS=${qemu}/share/qemu/edk2-arm-vars.fd + ''} export TEST_GUEST_MACHINE=${if isAarch64 then "virt" else "q35"} - export TEST_GUEST_ACCELERATOR=${if isDarwin then "hvf" else "kvm"} + export TEST_GUEST_ACCELERATOR=${selectedAccelerator} export TEST_GUEST_ARCHITECTURE=${architecture} export TEST_GUEST_ANSIBLE_VERSION=${pkgs.python3Packages.ansible-core.version} export TEST_GUEST_CACHE_GENERATION=1 diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 874dcd0773..f6a734b2cd 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -237,19 +237,27 @@ for copy_spec in "${copies[@]}"; do esac done -test_vm_cpu=host -ssh_wait_seconds=180 +case "${TEST_GUEST_ACCELERATOR}" in +tcg) + test_vm_cpu=max + ssh_wait_seconds=600 + ;; +*) + test_vm_cpu=host + ssh_wait_seconds=180 + ;; +esac if [ "${TEST_GUEST_ACCELERATOR}" = kvm ] && { [ ! -c /dev/kvm ] || [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; }; then echo "==> /dev/kvm is unavailable; falling back to QEMU/TCG" TEST_GUEST_ACCELERATOR=tcg test_vm_cpu=max ssh_wait_seconds=600 - if [ "${TEST_VM_ARCHITECTURE}" = aarch64 ]; then - : "${TEST_VM_TCG_FIRMWARE_CODE:?missing QEMU ARM TCG firmware code}" - : "${TEST_VM_TCG_FIRMWARE_VARS:?missing QEMU ARM TCG firmware variables}" - TEST_VM_FIRMWARE_CODE=${TEST_VM_TCG_FIRMWARE_CODE} - TEST_VM_FIRMWARE_VARS=${TEST_VM_TCG_FIRMWARE_VARS} + if [ "${TEST_GUEST_ARCHITECTURE}" = aarch64 ]; then + : "${TEST_GUEST_TCG_FIRMWARE_CODE:?missing QEMU ARM TCG firmware code}" + : "${TEST_GUEST_TCG_FIRMWARE_VARS:?missing QEMU ARM TCG firmware variables}" + TEST_GUEST_FIRMWARE_CODE=${TEST_GUEST_TCG_FIRMWARE_CODE} + TEST_GUEST_FIRMWARE_VARS=${TEST_GUEST_TCG_FIRMWARE_VARS} echo "==> Using QEMU-bundled ARM firmware for TCG fallback" fi fi @@ -476,7 +484,8 @@ scp_args=( echo "==> Waiting up to ${ssh_wait_seconds} seconds for SSH on 127.0.0.1:${ssh_port}" ssh_ready=0 -for _ in $(seq 1 "$((ssh_wait_seconds * 4))"); do +ssh_deadline_seconds=$((boot_started_seconds + ssh_wait_seconds)) +while [ "${SECONDS}" -lt "${ssh_deadline_seconds}" ]; do if ! kill -0 "${qemu_pid}" 2>/dev/null; then wait "${qemu_pid}" || true qemu_pid= From 4d4bc1457070232fb6e7eedf254fee925902fdf6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 28 Jul 2026 21:30:13 -0700 Subject: [PATCH 29/29] fix(nix): restore ARM64 TCG guest app Signed-off-by: Drew Newberry --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index c99da271b2..36ee2e7b5e 100644 --- a/flake.nix +++ b/flake.nix @@ -51,6 +51,8 @@ test-guest-cache = testGuest.cacheApp; } // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { + test-guest-arm64-tcg = testGuestArm64Tcg.app; + }; devShells.default = pkgs.mkShell { packages = with pkgs; [