diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml new file mode 100644 index 0000000..3d75908 --- /dev/null +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -0,0 +1,303 @@ +name: dory backend smoke + +# End-to-end smoke for `hops local --backend dory` on public GHA macOS. +# Mirrors on-pr-kind-smoke.yaml: start → doctor → dual registry → stop/start → destroy. +# +# Install contract (mandatory — not brew cask / official release): +# 1. Clone patrickleet/dory @ feat/hops-local-integration +# 2. Build in-pipeline (xcodebuild with deterministic -derivedDataPath) +# 3. Put scripts/dory on PATH +# 4. Provide ~/.dory/engine.sock before hops start (see engine note below) +# +# Engine socket on public GHA (important): +# - Dory's native dory-hv engine needs Hypervisor.framework nested virt. +# Dory's own CI documents that GitHub-hosted macOS runners are VMs and +# CANNOT run that engine (see dory benchmark.yml). Observed: Debug Dory.app +# builds, but never creates ~/.dory on macos-15 arm (engine.sock absent). +# - hops requires engine.sock (not only dory.sock). To still exercise the +# hops dory backend (dory k8s enable, context `dory`, registry plumbing) +# on public GHA, we provision a nested-virt Docker daemon with Colima +# (proven on macos-15-intel) and expose it as ~/.dory/engine.sock when +# the native app does not produce one. +# - Prefer trying the built app first (xattr clear + ad-hoc sign + open); +# fall back to Colima-backed engine.sock if sock never appears. +# +# Runner: macos-15-intel — nested virt for Colima (same pin as colima smoke). +# Prefer localhost:30500 for registry traffic (macOS 15 Local Network Privacy). +# After cold start, wait for CoreDNS/node Ready before registry pods; long +# Ready timeouts + stop/start rollout restart (colima nested-virt lessons). + +on: + # PR trigger so the workflow runs from the PR branch (workflow_dispatch only + # works once the file exists on the default branch). + pull_request: + workflow_dispatch: + +jobs: + dory-smoke: + # Nested-virt-capable Intel pin (not bare macos-latest / not arm-only). + # Native dory-hv wants Apple silicon, but public GHA cannot run it; this + # job uses Colima as the docker daemon behind engine.sock when needed. + runs-on: macos-15-intel + timeout-minutes: 90 + steps: + - name: Checkout hops-cli + uses: actions/checkout@v4 + + - name: Checkout patrickleet/dory (hops integration branch) + uses: actions/checkout@v4 + with: + repository: patrickleet/dory + ref: feat/hops-local-integration + path: dory-src + + - name: Select Xcode + run: | + set -euxo pipefail + newest="$(ls -d /Applications/Xcode*.app | sort -V | tail -1)" + echo "Using Xcode at $newest" + echo "DEVELOPER_DIR=$newest/Contents/Developer" >> "$GITHUB_ENV" + sudo xcode-select -s "$newest/Contents/Developer" + + - name: Build Dory from source + working-directory: dory-src + run: | + set -euxo pipefail + # Match dory CI: Metal toolchain download can be flaky; best-effort. + xcodebuild -downloadComponent MetalToolchain || true + # Prefer deterministic product path via -derivedDataPath (not scripts/build.sh + # alone, which may use a user-local DerivedData path). + xcodebuild -project Dory.xcodeproj -scheme Dory \ + -destination 'platform=macOS' -configuration Debug \ + -derivedDataPath build \ + build CODE_SIGNING_ALLOWED=NO + test -d build/Build/Products/Debug/Dory.app + # Clear provenance/quarantine that can leave Debug bundles unlaunchable + # (dory scripts/build.sh does the same for DerivedData products). + xattr -cr build/Build/Products/Debug/Dory.app 2>/dev/null || true + xattr -dr com.apple.provenance build/Build/Products/Debug/Dory.app 2>/dev/null || true + xattr -dr com.apple.quarantine build/Build/Products/Debug/Dory.app 2>/dev/null || true + codesign --force --deep --sign - build/Build/Products/Debug/Dory.app 2>/dev/null || true + + - name: Install dory CLI on PATH + run: | + set -euxo pipefail + chmod +x dory-src/scripts/dory + echo "$GITHUB_WORKSPACE/dory-src/scripts" >> "$GITHUB_PATH" + test -x dory-src/scripts/dory + dory-src/scripts/dory version || dory-src/scripts/dory k8s status || true + + - name: Install colima, docker, kubectl, helm + run: | + set -euxo pipefail + # colima provides nested-virt docker when native dory-hv cannot run. + brew install colima docker kubectl helm + colima version + docker --version + kubectl version --client + helm version --short + + - name: Ensure engine.sock (app native or Colima surrogate) + run: | + set -euxo pipefail + DORY_APP="$GITHUB_WORKSPACE/dory-src/build/Build/Products/Debug/Dory.app" + test -d "$DORY_APP" + mkdir -p "$HOME/.dory" + + ARCH="$(uname -m)" + # dory-hv requires Apple silicon; Intel hosts never produce a native sock. + # Public GHA arm also cannot nested-virt the engine (dory CI docs). Try + # native only briefly on arm; always fall through to Colima if missing. + if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "arm64e" ]; then + open "$DORY_APP" || true + for i in $(seq 1 12); do + if [ -S "$HOME/.dory/engine.sock" ]; then + echo "native engine.sock ready after ${i} attempts" + ls -la "$HOME/.dory" + echo "DORY_ENGINE_SOURCE=native" >> "$GITHUB_ENV" + exit 0 + fi + sleep 5 + done + pkill -f 'Dory.app/Contents/MacOS/Dory' || true + echo "native engine.sock absent; bootstrapping Colima docker as engine.sock" + else + echo "host arch $ARCH: native dory-hv unsupported; Colima surrogate" + fi + + # --- Colima-backed engine.sock (public GHA nested virt for docker) --- + # Sizing fits macos-15-intel (~4 CPU / ~14 GiB). Disk 40 for registry PVC + # when hops later creates the 20Gi registry volume inside k3s-on-docker. + colima start --runtime docker --cpu 3 --memory 8 --disk 40 + DOCKER_SOCK="" + for candidate in \ + "$HOME/.colima/default/docker.sock" \ + "$HOME/.colima/docker.sock" \ + /var/run/docker.sock + do + if [ -S "$candidate" ]; then + DOCKER_SOCK="$candidate" + break + fi + done + if [ -z "$DOCKER_SOCK" ]; then + echo "colima started but no docker.sock found" >&2 + colima status || true + ls -la "$HOME/.colima" || true + ls -la "$HOME/.colima/default" || true + exit 1 + fi + # Prefer real path via env (dory's dk() uses DORY_ENGINE_SOCK) and + # keep symlink so hops preflight (`~/.dory/engine.sock`) passes. + ln -sfn "$DOCKER_SOCK" "$HOME/.dory/engine.sock" + ln -sfn "$DOCKER_SOCK" "$HOME/.dory/dory.sock" + test -S "$HOME/.dory/engine.sock" || test -e "$HOME/.dory/engine.sock" + echo "DORY_ENGINE_SOCK=$DOCKER_SOCK" >> "$GITHUB_ENV" + echo "DOCKER_HOST=unix://$DOCKER_SOCK" >> "$GITHUB_ENV" + export DORY_ENGINE_SOCK="$DOCKER_SOCK" + export DOCKER_HOST="unix://$DOCKER_SOCK" + docker info >/dev/null + docker ps -a || true + echo "engine.sock → $DOCKER_SOCK (colima surrogate)" + ls -la "$HOME/.dory" + echo "DORY_ENGINE_SOURCE=colima-surrogate" >> "$GITHUB_ENV" + + - uses: Swatinem/rust-cache@v2 + + - name: Build hops + run: cargo build + + - name: Host resources (for diagnosis) + run: | + set -euxo pipefail + sysctl -n hw.ncpu || true + sysctl hw.memsize || true + uname -m + df -h / || true + echo "DORY_ENGINE_SOURCE=${DORY_ENGINE_SOURCE:-unknown}" + + - name: hops local start --backend dory + run: | + set -euxo pipefail + export DOCKER_HOST="${DOCKER_HOST:-unix://$HOME/.dory/engine.sock}" + export DORY_ENGINE_SOCK="${DORY_ENGINE_SOCK:-$HOME/.dory/engine.sock}" + # Clean slate: partial dory-k8s can report create-time drift (exit 3). + dory k8s disable || true + docker ps -a || true + ./target/debug/hops-cli local start --backend dory + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS (nested virt settles) + run: | + set -euxo pipefail + kubectl --context dory wait --for=condition=Ready nodes --all --timeout=120s + kubectl --context dory -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl --context dory -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl --context dory -n kube-system get pods -o wide || true + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + for attempt in 1 2 3 4 5; do + if docker pull public.ecr.aws/docker/library/busybox:stable; then + break + fi + echo "docker pull failed (attempt $attempt); sleeping..." + sleep $((attempt * 15)) + if [ "$attempt" -eq 5 ]; then + exit 1 + fi + done + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + kubectl --context dory run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + kubectl --context dory run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + if ! kubectl --context dory wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then + echo "==== smoke pods not Ready ====" + kubectl --context dory get pods smoke-svc-name smoke-localhost -o wide || true + kubectl --context dory describe pod smoke-svc-name smoke-localhost || true + kubectl --context dory get events -A --field-selector involvedObject.name=smoke-svc-name || true + kubectl --context dory get events -A --field-selector involvedObject.name=smoke-localhost || true + exit 1 + fi + kubectl --context dory delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + export DOCKER_HOST="${DOCKER_HOST:-unix://$HOME/.dory/engine.sock}" + export DORY_ENGINE_SOCK="${DORY_ENGINE_SOCK:-$HOME/.dory/engine.sock}" + ./target/debug/hops-cli local stop + # After docker stop/start of dory-k8s, give the apiserver a beat before + # hops re-runs helm (openapi can time out immediately after node Ready). + sleep 20 + ./target/debug/hops-cli local start + if ! kubectl --context dory -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s; then + echo "deployments not Available after resume; restarting..." + kubectl --context dory -n crossplane-system get pods -o wide || true + kubectl --context dory -n crossplane-system rollout restart \ + deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true + kubectl --context dory -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s + fi + ./target/debug/hops-cli local doctor + + - name: Debug dump on failure + if: failure() + run: | + set +e + echo "==== host ====" + sysctl -n hw.ncpu + sysctl hw.memsize + uname -m + df -h / + echo "DORY_ENGINE_SOURCE=${DORY_ENGINE_SOURCE:-unknown}" + echo "==== dory / colima ====" + ls -la "$HOME/.dory" || true + dory version || true + dory k8s status || true + colima status || true + colima list || true + export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + docker info 2>&1 | head -40 || true + echo "==== cluster ====" + kubectl --context dory get nodes,pods -A -o wide || true + echo "==== smoke pods (default) ====" + kubectl --context dory describe pod smoke-svc-name smoke-localhost 2>/dev/null || true + echo "==== crossplane-system ====" + kubectl --context dory describe pods -n crossplane-system || true + kubectl --context dory get events -A --sort-by=.lastTimestamp | tail -100 || true + ./target/debug/hops-cli local doctor || true + + - name: hops local destroy + if: always() + run: | + set +e + export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + ./target/debug/hops-cli local destroy || true + + - name: Teardown Dory + Colima + if: always() + run: | + set +e + dory k8s disable || true + pkill -f 'Dory.app/Contents/MacOS/Dory' || true + colima stop || true + colima delete --force || true + ls -la "$HOME/.dory" || true diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs index 1c0dfed..00ad26b 100644 --- a/src/commands/local/backend/dory.rs +++ b/src/commands/local/backend/dory.rs @@ -329,15 +329,31 @@ fn run_dory_enable(recreate: bool) -> Result<(), Box> { let args = dory_enable_args(recreate); match run_cmd("dory", &args) { Ok(()) => hold_through_engine_launch_window(), - Err(err) if !recreate && err.to_string().contains("exit status: 3") => Err(format!( - "{}\nhint: run `hops local reset --backend dory` to recreate the dory cluster and apply create-time config drift", - err - ) - .into()), + // Exit 3 = create-time config drift (ports / registries bind). hops just + // wrote the desired ports+registries files; applying them requires + // recreate. Auto-retry once so `local start` is not a dead end after + // hops updates publish config (reset remains available explicitly). + // + // Exit 1 after "did not become Ready" is common on stop→start: docker + // start of the existing k3s container can leave the node stuck past + // dory's wait window. Recreate once (same as `dory k8s disable && enable`). + Err(err) if !recreate && should_recreate_on_enable_error(&err.to_string()) => { + log::warn!( + "dory k8s enable failed ({err}); recreating cluster once..." + ); + let _ = run_cmd("dory", &["k8s", "disable"]); + run_dory_enable(true) + } Err(err) => Err(err), } } +fn should_recreate_on_enable_error(msg: &str) -> bool { + // run_cmd only surfaces exit status (stderr is inherited), so match codes. + // 3 = create-time config drift; 1 = dory k8s_wait_ready / generic enable fail. + msg.contains("exit status: 3") || msg.contains("exit status: 1") +} + fn dory_enable_args(recreate: bool) -> Vec<&'static str> { let mut args = vec!["k8s", "enable"]; if recreate { @@ -725,6 +741,20 @@ mod tests { ); } + #[test] + fn recreate_on_enable_matches_dory_exit_codes() { + assert!(should_recreate_on_enable_error( + "dory exited with exit status: 3" + )); + assert!(should_recreate_on_enable_error( + "dory exited with exit status: 1" + )); + assert!(!should_recreate_on_enable_error( + "dory exited with exit status: 2" + )); + assert!(!should_recreate_on_enable_error("connection refused")); + } + #[test] fn start_rejects_size_flags() { let size = SizeArgs { diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index b151675..19d4e42 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -76,10 +76,12 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box> = None; + for attempt in 1..=6 { + wait_for_kubernetes()?; + match run_cmd("helm", &helm_args) { + Ok(()) => { + last_err = None; + break; + } + Err(e) => { + log::warn!("helm install attempt {attempt}/6 failed: {e}"); + last_err = Some(e); + std::thread::sleep(std::time::Duration::from_secs(20)); + } + } + } + if let Some(e) = last_err { + return Err(e); + } + } // 6. Wait for Crossplane core deployment. // rbac-manager can flap under nested-virt resource pressure; the core diff --git a/tests/dory_smoke_workflow.rs b/tests/dory_smoke_workflow.rs new file mode 100644 index 0000000..98cca83 --- /dev/null +++ b/tests/dory_smoke_workflow.rs @@ -0,0 +1,139 @@ +//! Structural checks for the shipped dory macOS smoke workflow. +//! +//! Drives the real `.github/workflows/on-pr-dory-smoke.yaml` so the clone+build +//! install contract and kind-parity smoke steps cannot silently regress. + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-dory-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected dory smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn dory_smoke_workflow_clone_build_install_contract() { + let text = workflow_text(); + assert!( + text.contains("patrickleet/dory"), + "must clone patrickleet/dory" + ); + assert!( + text.contains("feat/hops-local-integration"), + "must pin hops integration branch" + ); + assert!( + text.contains("xcodebuild") && text.contains("derivedDataPath"), + "must build in-pipeline with deterministic derivedDataPath" + ); + assert!( + text.contains("GITHUB_PATH") && text.contains("scripts"), + "must put scripts/dory on PATH" + ); + assert!( + text.contains("engine.sock"), + "must ensure ~/.dory/engine.sock before hops start" + ); + assert!( + !text.contains("brew install --cask") && !text.contains("homebrew/cask"), + "must not use brew cask as primary install" + ); + assert!( + text.contains("workflow_dispatch") && text.contains("pull_request:"), + "must support pull_request (PR-branch runs) and workflow_dispatch" + ); + // Nested-virt pin for Colima-backed engine.sock on public GHA. + assert!( + text.lines() + .any(|l| l.trim() == "runs-on: macos-15-intel"), + "must pin macos-15-intel for nested virt (not bare macos-latest alone)" + ); +} + +#[test] +fn dory_smoke_workflow_kind_parity_when_engine_boots() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend dory", + "local doctor", + "localhost:30500", + "registry.crossplane-system.svc.cluster.local:5000", + "--context dory", + "local stop", + "local destroy", + ] { + assert!( + text.contains(needle), + "dory smoke missing kind-parity fragment: {needle}" + ); + } + // stop/start resume: start without --backend after stop + assert!( + text.contains("local start\n") + || text + .lines() + .any(|l| l.trim() == "./target/debug/hops-cli local start"), + "must start again without --backend after stop" + ); +} + +#[test] +fn dory_smoke_workflow_carries_colima_lessons() { + let text = workflow_text(); + assert!( + text.contains("kube-dns") || text.contains("coredns"), + "must wait for CoreDNS/kube-dns before registry round-trip" + ); + assert!( + text.contains("--timeout=420s"), + "must use a long Ready/Available timeout for nested-virt lag" + ); + assert!( + text.contains("rollout restart"), + "stop/start resume must rollout-restart stalled deployments" + ); + assert!( + text.contains("Debug dump on failure") && text.contains("engine.sock"), + "failure dump must capture dory/engine diagnostics" + ); + assert!( + text.contains("describe pod smoke-svc-name") + || text.contains("describe pod smoke-svc-name smoke-localhost"), + "failure dump must describe smoke pods in default ns" + ); + assert!( + text.to_lowercase().contains("localhost:30500"), + "must exercise localhost:30500 registry path" + ); +} + +#[test] +fn dory_smoke_workflow_public_gha_engine_fallback() { + let text = workflow_text(); + // Public GHA cannot run dory-hv; workflow must document and implement a + // Colima-backed engine.sock so hops --backend dory remains testable. + assert!( + text.contains("colima start") && text.contains("engine.sock"), + "must bootstrap Colima docker as engine.sock when native sock is absent" + ); + assert!( + text.to_lowercase().contains("surrogate") + || text.contains("colima-surrogate") + || text.contains("Colima-backed"), + "must document Colima engine.sock surrogate for public GHA" + ); + assert!( + text.contains("xattr") || text.contains("codesign"), + "must clear quarantine/sign Debug Dory.app before open" + ); +}