From cdd267e9ac46f31d688c13f28858996c5c0b466c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Mon, 6 Jul 2026 21:22:16 +0200 Subject: [PATCH 1/4] Stop idle App Engine versions after deploy App Engine Flexible keeps an always-on VM for every SERVING version regardless of traffic, so the staging version and superseded prod versions left SERVING after each deploy keep costing ~$212/mo each. The pipeline promoted traffic but never deactivated old versions. - Stop the staging version after its integration tests pass. - After a production promote, keep the live version plus the newest 2 prod versions warm for rollback and stop every other SERVING version; delete stopped versions beyond the newest 20 to respect App Engine's 210-versions-per-service limit. Stopping (not deleting) preserves rollback via `gcloud app versions start` + `set-traffic`. Adds stop_app_engine_version.sh and cleanup_app_engine_versions.sh, plus two jobs in push.yml. Co-Authored-By: Claude Opus 4.8 --- .../scripts/cleanup_app_engine_versions.sh | 121 ++++++++++++++++++ .github/scripts/stop_app_engine_version.sh | 27 ++++ .github/workflows/push.yml | 62 +++++++++ ...app-engine-idle-version-cleanup.changed.md | 1 + 4 files changed, 211 insertions(+) create mode 100755 .github/scripts/cleanup_app_engine_versions.sh create mode 100755 .github/scripts/stop_app_engine_version.sh create mode 100644 changelog.d/app-engine-idle-version-cleanup.changed.md diff --git a/.github/scripts/cleanup_app_engine_versions.sh b/.github/scripts/cleanup_app_engine_versions.sh new file mode 100755 index 000000000..83d821c18 --- /dev/null +++ b/.github/scripts/cleanup_app_engine_versions.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Deactivate idle App Engine versions after a production promote. +# +# App Engine Flexible keeps at least one always-on VM for every SERVING version, +# regardless of traffic, so old versions left SERVING after a promote keep +# costing money even at 0 traffic (~$212/mo per 4 vCPU / 24 GB version). The +# deploy pipeline promotes traffic with set-traffic but never deactivates the +# versions it supersedes, so they pile up. +# +# This script, run after promote-production: +# 1. Keeps the version currently receiving traffic PLUS the newest +# KEEP_WARM_PROD prod-* versions SERVING (a warm rollback window), and STOPs +# every other SERVING version (older prod + any leftover staging). +# 2. DELETEs STOPPED versions beyond the newest KEEP_STOPPED, to stay under +# App Engine's 210-versions-per-service limit. +# +# Stopping (not deleting) preserves rollback: a stopped version can be brought +# back with `gcloud app versions start ` then `gcloud app services set-traffic +# ${SERVICE} --splits==1`. The version currently serving traffic is never +# stopped or deleted. +# +# Set DRY_RUN=1 to print the plan without changing anything. +# +# Written to run on bash 3.2 (no mapfile / associative arrays) so it can be +# dry-run locally on macOS as well as in CI. + +set -euo pipefail + +APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}" +KEEP_WARM_PROD="${KEEP_WARM_PROD:-2}" +KEEP_STOPPED="${KEEP_STOPPED:-20}" +DRY_RUN="${DRY_RUN:-0}" + +common_args=("--service=${APP_ENGINE_SERVICE}") +if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then + common_args+=("--project=${APP_ENGINE_PROJECT}") +fi + +contains() { + local needle="$1" + shift + local item + for item in "$@"; do + [[ "${item}" == "${needle}" ]] && return 0 + done + return 1 +} + +read_versions() { + # Args: extra gcloud filter. Emits version ids newest-first, one per line. + local filter="$1" + gcloud app versions list "${common_args[@]}" \ + --filter="${filter}" \ + --sort-by="~version.createTime" \ + --format="value(id)" +} + +# Versions currently receiving any traffic — never touched. +serving_traffic=() +while IFS= read -r v; do + [[ -n "${v}" ]] && serving_traffic+=("${v}") +done < <(read_versions "version.servingStatus=SERVING AND traffic_split>0") + +# All SERVING versions, newest first. +serving_all=() +while IFS= read -r v; do + [[ -n "${v}" ]] && serving_all+=("${v}") +done < <(read_versions "version.servingStatus=SERVING") + +# Newest KEEP_WARM_PROD prod-* versions to keep warm for instant rollback. +warm_prod=() +while IFS= read -r v; do + [[ -n "${v}" ]] && warm_prod+=("${v}") +done < <(printf '%s\n' "${serving_all[@]:-}" | grep '^prod-' | head -n "${KEEP_WARM_PROD}") + +# Keep set = traffic-serving versions + warm prod versions. +keep=("${serving_traffic[@]:-}" "${warm_prod[@]:-}") + +# Stop every SERVING version not in the keep set. +to_stop=() +for v in "${serving_all[@]:-}"; do + [[ -z "${v}" ]] && continue + contains "${v}" "${keep[@]:-}" && continue + to_stop+=("${v}") +done + +echo "Service: ${APP_ENGINE_SERVICE}" +echo "Serving traffic (never touched): ${serving_traffic[*]:-}" +echo "Keeping warm (rollback window): ${warm_prod[*]:-}" +echo "Stopping (${#to_stop[@]}): ${to_stop[*]:-}" + +if [[ "${#to_stop[@]}" -gt 0 ]]; then + if [[ "${DRY_RUN}" == "1" ]]; then + echo "[dry-run] would stop ${#to_stop[@]} version(s)" + else + gcloud app versions stop "${to_stop[@]}" "${common_args[@]}" --quiet + fi +fi + +# Delete STOPPED versions beyond the newest KEEP_STOPPED (version-quota hygiene). +stopped_all=() +while IFS= read -r v; do + [[ -n "${v}" ]] && stopped_all+=("${v}") +done < <(read_versions "version.servingStatus=STOPPED") + +to_delete=() +if [[ "${#stopped_all[@]}" -gt "${KEEP_STOPPED}" ]]; then + to_delete=("${stopped_all[@]:${KEEP_STOPPED}}") +fi + +echo "Stopped versions: ${#stopped_all[@]} (keeping newest ${KEEP_STOPPED})" +echo "Deleting (${#to_delete[@]}): ${to_delete[*]:-}" + +if [[ "${#to_delete[@]}" -gt 0 ]]; then + if [[ "${DRY_RUN}" == "1" ]]; then + echo "[dry-run] would delete ${#to_delete[@]} version(s)" + else + gcloud app versions delete "${to_delete[@]}" "${common_args[@]}" --quiet + fi +fi diff --git a/.github/scripts/stop_app_engine_version.sh b/.github/scripts/stop_app_engine_version.sh new file mode 100755 index 000000000..de36545dd --- /dev/null +++ b/.github/scripts/stop_app_engine_version.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Stop (deactivate) a single App Engine version. +# +# App Engine Flexible keeps at least one VM running for every SERVING version +# regardless of traffic, so a version left SERVING after its tests keeps costing +# money. Stopping it drops its instances to zero (no billing) while keeping the +# version deployed and restartable (`gcloud app versions start`) for rollback. + +set -euo pipefail + +: "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}" + +APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}" + +stop_args=( + "${APP_ENGINE_VERSION}" + "--service=${APP_ENGINE_SERVICE}" + "--quiet" +) + +if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then + stop_args+=("--project=${APP_ENGINE_PROJECT}") +fi + +echo "Stopping App Engine version ${APP_ENGINE_VERSION} (service ${APP_ENGINE_SERVICE})" +gcloud app versions stop "${stop_args[@]}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fd14daccd..d86b10a94 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -286,6 +286,37 @@ jobs: API_BASE_URL: ${{ needs.deploy-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: ${{ needs.deploy-staging.outputs.version }} + stop-staging-app-engine-version: + name: Stop staging App Engine version + runs-on: ubuntu-latest + needs: + - deploy-staging + - integration-tests-staging + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: staging + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: GCP authentication + uses: "google-github-actions/auth@v2" + with: + workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}" + service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" + - name: Set up GCloud + uses: "google-github-actions/setup-gcloud@v2" + # Flex keeps a VM running for every SERVING version regardless of traffic, + # so the just-tested staging version would keep costing money. Stop it now + # that its integration tests have passed; the next deploy makes a fresh one. + - name: Stop staging App Engine version + run: bash .github/scripts/stop_app_engine_version.sh + env: + APP_ENGINE_VERSION: ${{ needs.deploy-staging.outputs.version }} + integration-tests-staging-cloud-run: name: Run Cloud Run staging integration tests runs-on: ubuntu-latest @@ -468,6 +499,37 @@ jobs: env: APP_ENGINE_VERSION: ${{ needs.deploy-production-candidate.outputs.version }} + cleanup-prod-app-engine-versions: + name: Clean up idle production App Engine versions + runs-on: ubuntu-latest + needs: promote-production + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: production + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: GCP authentication + uses: "google-github-actions/auth@v2" + with: + workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}" + service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" + - name: Set up GCloud + uses: "google-github-actions/setup-gcloud@v2" + # Keep the live version plus the newest KEEP_WARM_PROD prod versions warm + # for instant rollback; stop every other SERVING version (older prod + + # leftover staging) so idle Flex VMs stop billing; prune stopped versions + # beyond KEEP_STOPPED to stay under the 210-versions-per-service limit. + - name: Stop superseded versions and prune old stopped versions + run: bash .github/scripts/cleanup_app_engine_versions.sh + env: + KEEP_WARM_PROD: "2" + KEEP_STOPPED: "20" + deploy-cloud-run-candidate: name: Deploy production Cloud Run candidate runs-on: ubuntu-latest diff --git a/changelog.d/app-engine-idle-version-cleanup.changed.md b/changelog.d/app-engine-idle-version-cleanup.changed.md new file mode 100644 index 000000000..102a9ab05 --- /dev/null +++ b/changelog.d/app-engine-idle-version-cleanup.changed.md @@ -0,0 +1 @@ +Deploys now stop idle App Engine versions automatically. The staging version is stopped after its integration tests pass, and after a production promote only the two newest production versions stay warm while every other superseded version is stopped (with stopped versions pruned beyond the newest 20). This removes always-on Flexible-environment VM cost from versions that serve no traffic, while keeping recent versions available for rollback via `gcloud app versions start`. From 0b064e76b747180105768a3b6af0932a8e9f8656 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Mon, 6 Jul 2026 22:00:22 +0200 Subject: [PATCH 2/4] Keep 10 prod + 3 staging stopped versions instead of 20 pooled The stopped-version retention was a single newest-20 pool shared across prod and staging. Since only prod versions are meaningful rollback targets, keep the newest 10 stopped prod versions and only the newest 3 stopped staging versions; delete everything else stopped (older prod, older staging, and legacy timestamp-named versions). Co-Authored-By: Claude Opus 4.8 --- .../scripts/cleanup_app_engine_versions.sh | 43 +++++++++++++++---- .github/workflows/push.yml | 8 ++-- ...app-engine-idle-version-cleanup.changed.md | 2 +- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/scripts/cleanup_app_engine_versions.sh b/.github/scripts/cleanup_app_engine_versions.sh index 83d821c18..bc0f011a2 100755 --- a/.github/scripts/cleanup_app_engine_versions.sh +++ b/.github/scripts/cleanup_app_engine_versions.sh @@ -12,8 +12,12 @@ # 1. Keeps the version currently receiving traffic PLUS the newest # KEEP_WARM_PROD prod-* versions SERVING (a warm rollback window), and STOPs # every other SERVING version (older prod + any leftover staging). -# 2. DELETEs STOPPED versions beyond the newest KEEP_STOPPED, to stay under -# App Engine's 210-versions-per-service limit. +# 2. DELETEs stopped versions beyond the retention window: keeps the newest +# KEEP_STOPPED_PROD prod-* and KEEP_STOPPED_STAGING staging-* stopped +# versions, and deletes every other stopped version (older prod, older +# staging, and legacy timestamp-named versions), to stay under App Engine's +# 210-versions-per-service limit. Prod is kept deeper because only prod +# versions are meaningful rollback targets. # # Stopping (not deleting) preserves rollback: a stopped version can be brought # back with `gcloud app versions start ` then `gcloud app services set-traffic @@ -29,7 +33,8 @@ set -euo pipefail APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}" KEEP_WARM_PROD="${KEEP_WARM_PROD:-2}" -KEEP_STOPPED="${KEEP_STOPPED:-20}" +KEEP_STOPPED_PROD="${KEEP_STOPPED_PROD:-10}" +KEEP_STOPPED_STAGING="${KEEP_STOPPED_STAGING:-3}" DRY_RUN="${DRY_RUN:-0}" common_args=("--service=${APP_ENGINE_SERVICE}") @@ -98,18 +103,40 @@ if [[ "${#to_stop[@]}" -gt 0 ]]; then fi fi -# Delete STOPPED versions beyond the newest KEEP_STOPPED (version-quota hygiene). +# Delete stopped versions beyond the retention window (version-quota hygiene). +# Keep the newest KEEP_STOPPED_PROD stopped prod-* versions (cold rollback +# targets) and the newest KEEP_STOPPED_STAGING stopped staging-* versions; delete +# everything else stopped — older prod, older staging, and legacy timestamp-named +# versions. Serving/warm versions are SERVING, so never appear in this set. stopped_all=() while IFS= read -r v; do [[ -n "${v}" ]] && stopped_all+=("${v}") done < <(read_versions "version.servingStatus=STOPPED") +# Newest KEEP_STOPPED_PROD stopped prod-* versions to keep. +keep_stopped_prod=() +while IFS= read -r v; do + [[ -n "${v}" ]] && keep_stopped_prod+=("${v}") +done < <(printf '%s\n' "${stopped_all[@]:-}" | grep '^prod-' | head -n "${KEEP_STOPPED_PROD}") + +# Newest KEEP_STOPPED_STAGING stopped staging-* versions to keep. +keep_stopped_staging=() +while IFS= read -r v; do + [[ -n "${v}" ]] && keep_stopped_staging+=("${v}") +done < <(printf '%s\n' "${stopped_all[@]:-}" | grep '^staging-' | head -n "${KEEP_STOPPED_STAGING}") + +keep_stopped=("${keep_stopped_prod[@]:-}" "${keep_stopped_staging[@]:-}") + to_delete=() -if [[ "${#stopped_all[@]}" -gt "${KEEP_STOPPED}" ]]; then - to_delete=("${stopped_all[@]:${KEEP_STOPPED}}") -fi +for v in "${stopped_all[@]:-}"; do + [[ -z "${v}" ]] && continue + contains "${v}" "${keep_stopped[@]:-}" && continue + to_delete+=("${v}") +done -echo "Stopped versions: ${#stopped_all[@]} (keeping newest ${KEEP_STOPPED})" +echo "Stopped versions: ${#stopped_all[@]}" +echo "Keeping stopped prod (${#keep_stopped_prod[@]}): ${keep_stopped_prod[*]:-}" +echo "Keeping stopped staging (${#keep_stopped_staging[@]}): ${keep_stopped_staging[*]:-}" echo "Deleting (${#to_delete[@]}): ${to_delete[*]:-}" if [[ "${#to_delete[@]}" -gt 0 ]]; then diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d86b10a94..a05984545 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -522,13 +522,15 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" # Keep the live version plus the newest KEEP_WARM_PROD prod versions warm # for instant rollback; stop every other SERVING version (older prod + - # leftover staging) so idle Flex VMs stop billing; prune stopped versions - # beyond KEEP_STOPPED to stay under the 210-versions-per-service limit. + # leftover staging) so idle Flex VMs stop billing; delete stopped versions + # beyond the newest KEEP_STOPPED_PROD prod / KEEP_STOPPED_STAGING staging to + # stay under the 210-versions-per-service limit. - name: Stop superseded versions and prune old stopped versions run: bash .github/scripts/cleanup_app_engine_versions.sh env: KEEP_WARM_PROD: "2" - KEEP_STOPPED: "20" + KEEP_STOPPED_PROD: "10" + KEEP_STOPPED_STAGING: "3" deploy-cloud-run-candidate: name: Deploy production Cloud Run candidate diff --git a/changelog.d/app-engine-idle-version-cleanup.changed.md b/changelog.d/app-engine-idle-version-cleanup.changed.md index 102a9ab05..ae415f535 100644 --- a/changelog.d/app-engine-idle-version-cleanup.changed.md +++ b/changelog.d/app-engine-idle-version-cleanup.changed.md @@ -1 +1 @@ -Deploys now stop idle App Engine versions automatically. The staging version is stopped after its integration tests pass, and after a production promote only the two newest production versions stay warm while every other superseded version is stopped (with stopped versions pruned beyond the newest 20). This removes always-on Flexible-environment VM cost from versions that serve no traffic, while keeping recent versions available for rollback via `gcloud app versions start`. +Deploys now stop idle App Engine versions automatically. The staging version is stopped after its integration tests pass, and after a production promote only the two newest production versions stay warm while every other superseded version is stopped. Stopped versions are then pruned to the newest 10 production and newest 3 staging (older prod, older staging, and legacy timestamp-named versions are deleted). This removes always-on Flexible-environment VM cost from versions that serve no traffic, while keeping recent production versions available for rollback via `gcloud app versions start`. From 19e8c58994c393fc0efa12f66d88b601ed5bf465 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 7 Jul 2026 15:44:37 +0200 Subject: [PATCH 3/4] Keep cleanup retention counts in the script, not workflow env Remove the KEEP_WARM_PROD / KEEP_STOPPED_PROD / KEEP_STOPPED_STAGING env block from the cleanup job so the retention counts have a single source of truth (the script defaults) instead of being duplicated in push.yml. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/push.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a05984545..facdd54b2 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -520,17 +520,13 @@ jobs: service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" - name: Set up GCloud uses: "google-github-actions/setup-gcloud@v2" - # Keep the live version plus the newest KEEP_WARM_PROD prod versions warm - # for instant rollback; stop every other SERVING version (older prod + - # leftover staging) so idle Flex VMs stop billing; delete stopped versions - # beyond the newest KEEP_STOPPED_PROD prod / KEEP_STOPPED_STAGING staging to - # stay under the 210-versions-per-service limit. + # Keep the live version plus the 2 newest prod versions warm for instant + # rollback; stop every other SERVING version (older prod + leftover staging) + # so idle Flex VMs stop billing; keep the newest 10 stopped prod and 3 + # stopped staging versions and delete the rest to stay under the + # 210-versions-per-service limit. Retention counts are defaults in the script. - name: Stop superseded versions and prune old stopped versions run: bash .github/scripts/cleanup_app_engine_versions.sh - env: - KEEP_WARM_PROD: "2" - KEEP_STOPPED_PROD: "10" - KEEP_STOPPED_STAGING: "3" deploy-cloud-run-candidate: name: Deploy production Cloud Run candidate From 440707515e3553d79079d9d822e8c5f42ece80bb Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 7 Jul 2026 16:36:51 +0200 Subject: [PATCH 4/4] Add fail-safe guard and unit tests for App Engine cleanup - cleanup_app_engine_versions.sh: abort before stopping if versions are SERVING but none report traffic, so a failed/empty traffic query cannot deactivate the live service. Delete already only targets STOPPED, so the live version can never be deleted. - tests/unit/test_app_engine_cleanup_scripts.py: exercise both scripts with a stubbed gcloud (mirroring test_cloud_run_deploy_scripts.py) covering live/warm retention, protection of an older traffic-serving version, delete-only-stopped, the fail-safe guard, missing-staging robustness, and the stop script's required-version check. Co-Authored-By: Claude Opus 4.8 --- .../scripts/cleanup_app_engine_versions.sh | 10 + tests/unit/test_app_engine_cleanup_scripts.py | 257 ++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 tests/unit/test_app_engine_cleanup_scripts.py diff --git a/.github/scripts/cleanup_app_engine_versions.sh b/.github/scripts/cleanup_app_engine_versions.sh index bc0f011a2..af77a7978 100755 --- a/.github/scripts/cleanup_app_engine_versions.sh +++ b/.github/scripts/cleanup_app_engine_versions.sh @@ -73,6 +73,16 @@ while IFS= read -r v; do [[ -n "${v}" ]] && serving_all+=("${v}") done < <(read_versions "version.servingStatus=SERVING") +# Fail safe: if there are SERVING versions but none report traffic, we cannot +# identify the live service. Refuse to stop anything rather than risk +# deactivating production (e.g. if the traffic query returns nothing). The +# delete step never runs either, since we exit here. +if [[ "${#serving_all[@]}" -gt 0 && "${#serving_traffic[@]}" -eq 0 ]]; then + echo "ERROR: ${#serving_all[@]} SERVING version(s) but none are receiving traffic;" >&2 + echo "refusing to stop versions (cannot identify the live service). Aborting." >&2 + exit 1 +fi + # Newest KEEP_WARM_PROD prod-* versions to keep warm for instant rollback. warm_prod=() while IFS= read -r v; do diff --git a/tests/unit/test_app_engine_cleanup_scripts.py b/tests/unit/test_app_engine_cleanup_scripts.py new file mode 100644 index 000000000..cdf101c16 --- /dev/null +++ b/tests/unit/test_app_engine_cleanup_scripts.py @@ -0,0 +1,257 @@ +"""Unit tests for the App Engine idle-version cleanup scripts. + +`cleanup_app_engine_versions.sh` and `stop_app_engine_version.sh` are exercised +by running them under bash with a stubbed ``gcloud`` on ``PATH`` (mirroring the +approach in ``test_cloud_run_deploy_scripts.py``). The stub returns canned +``gcloud app versions list`` output based on the ``--filter`` and records any +``stop``/``delete`` invocations, so the tests assert the selection logic without +touching real infrastructure. All cleanup runs use ``DRY_RUN=1``. +""" + +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +CLEANUP_SCRIPT = ".github/scripts/cleanup_app_engine_versions.sh" +STOP_SCRIPT = ".github/scripts/stop_app_engine_version.sh" + +# A stub `gcloud` that answers `app versions list` from MOCK_* env vars (each a +# newline-joined, newest-first id list) and appends any stop/delete calls to +# $GCLOUD_CALLS so tests can assert nothing mutating ran under DRY_RUN. +_FAKE_GCLOUD = """#!/usr/bin/env bash +filter="" +for a in "$@"; do + case "$a" in --filter=*) filter="${a#--filter=}";; esac +done +case "$*" in + *"versions list"*) + case "$filter" in + *"traffic_split>0"*) [ -n "${MOCK_TRAFFIC:-}" ] && printf '%s\\n' "$MOCK_TRAFFIC";; + *"servingStatus=SERVING"*) [ -n "${MOCK_SERVING:-}" ] && printf '%s\\n' "$MOCK_SERVING";; + *"servingStatus=STOPPED"*) [ -n "${MOCK_STOPPED:-}" ] && printf '%s\\n' "$MOCK_STOPPED";; + esac;; + *"versions stop"*) echo "STOP $*" >> "$GCLOUD_CALLS";; + *"versions delete"*) echo "DELETE $*" >> "$GCLOUD_CALLS";; +esac +exit 0 +""" + + +def _run( + script: str, + tmp_path: Path, + *, + serving: list[str] | None = None, + traffic: list[str] | None = None, + stopped: list[str] | None = None, + extra_env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], Path]: + stub_dir = tmp_path / "bin" + stub_dir.mkdir(exist_ok=True) + gcloud = stub_dir / "gcloud" + gcloud.write_text(_FAKE_GCLOUD, encoding="utf-8") + gcloud.chmod(0o755) + + calls = tmp_path / "gcloud_calls.log" + calls.write_text("", encoding="utf-8") + + env = { + "HOME": os.environ.get("HOME", ""), + "PATH": f"{stub_dir}:{os.environ['PATH']}", + "DRY_RUN": "1", + "APP_ENGINE_PROJECT": "test-project", + "GCLOUD_CALLS": str(calls), + "MOCK_SERVING": "\n".join(serving or []), + "MOCK_TRAFFIC": "\n".join(traffic or []), + "MOCK_STOPPED": "\n".join(stopped or []), + } + env.update(extra_env or {}) + + result = subprocess.run( + ["bash", script], + cwd=REPO, + env=env, + text=True, + capture_output=True, + check=False, + ) + return result, calls + + +def _list_after(label: str, stdout: str) -> list[str]: + """Return the space-separated ids printed after a `Label[ (n)]: ...` line.""" + match = re.search(rf"{re.escape(label)}(?: \(\d+\))?:\s*(.*)", stdout) + assert match is not None, f"missing '{label}' line in:\n{stdout}" + body = match.group(1).strip() + return [] if body == "" else body.split() + + +# --- syntax -------------------------------------------------------------- + + +def test_cleanup_scripts_are_shell_syntax_valid(): + for script in (CLEANUP_SCRIPT, STOP_SCRIPT): + result = subprocess.run( + ["bash", "-n", script], cwd=REPO, capture_output=True, text=True + ) + assert result.returncode == 0, f"{script}: {result.stderr}" + + +# --- cleanup: stop selection --------------------------------------------- + + +def test_keeps_live_and_warm_prod_and_stops_the_rest(tmp_path): + result, calls = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-110", "prod-108", "staging-110", "staging-108", "prod-106"], + traffic=["prod-110"], + stopped=["prod-104", "prod-102"], + ) + assert result.returncode == 0, result.stderr + assert _list_after("Serving traffic (never touched)", result.stdout) == ["prod-110"] + assert _list_after("Keeping warm (rollback window)", result.stdout) == [ + "prod-110", + "prod-108", + ] + # Every SERVING version except the live one and the 2 newest prod is stopped. + assert _list_after("Stopping", result.stdout) == [ + "staging-110", + "staging-108", + "prod-106", + ] + # The live version is never in the stop or delete sets. + assert "prod-110" not in _list_after("Stopping", result.stdout) + assert "prod-110" not in _list_after("Deleting", result.stdout) + # DRY_RUN performs no mutations. + assert calls.read_text() == "" + + +def test_traffic_holder_is_protected_even_when_it_is_an_older_version(tmp_path): + # Simulates a manual rollback: traffic sits on prod-106 while prod-110/108 + # are newer. The older, traffic-serving version must NOT be stopped. + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-110", "prod-108", "prod-106", "staging-110"], + traffic=["prod-106"], + stopped=[], + ) + assert result.returncode == 0, result.stderr + assert "prod-106" not in _list_after("Stopping", result.stdout) + assert _list_after("Serving traffic (never touched)", result.stdout) == ["prod-106"] + + +# --- cleanup: delete selection / retention ------------------------------- + + +def test_deletes_only_stopped_beyond_retention_window(tmp_path): + stopped_prod = [f"prod-{n}" for n in range(120, 98, -2)] # 11 prod, newest first + stopped_staging = [f"staging-{n}" for n in range(120, 108, -2)] # 6 staging + legacy = ["20260101t120000", "20251201t120000"] + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-130"], + traffic=["prod-130"], + stopped=stopped_prod + stopped_staging + legacy, + ) + assert result.returncode == 0, result.stderr + + kept_prod = _list_after("Keeping stopped prod", result.stdout) + kept_staging = _list_after("Keeping stopped staging", result.stdout) + deleted = _list_after("Deleting", result.stdout) + + assert kept_prod == stopped_prod[:10] # newest 10 prod + assert kept_staging == stopped_staging[:3] # newest 3 staging + # Everything else stopped is deleted: 1 oldest prod + 3 older staging + 2 legacy. + assert set(deleted) == {stopped_prod[10]} | set(stopped_staging[3:]) | set(legacy) + # Never deletes a SERVING version. + assert "prod-130" not in deleted + + +def test_delete_only_targets_stopped_versions(tmp_path): + # A SERVING (but idle) version must be eligible for STOP, never DELETE. + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-110", "prod-108", "prod-106"], + traffic=["prod-110"], + stopped=[], + ) + assert result.returncode == 0, result.stderr + assert _list_after("Stopping", result.stdout) == ["prod-106"] + assert _list_after("Deleting", result.stdout) == [] + + +# --- cleanup: fail-safe guard -------------------------------------------- + + +def test_aborts_when_serving_versions_exist_but_none_serve_traffic(tmp_path): + # If the traffic query returns nothing while versions are SERVING, we must + # NOT stop anything (we cannot identify the live service). + result, calls = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-104", "prod-102", "prod-100"], + traffic=[], + stopped=["prod-98"], + ) + assert result.returncode != 0 + assert "refusing to stop" in result.stderr.lower() + assert "Stopping" not in result.stdout + assert calls.read_text() == "" + + +def test_no_serving_versions_is_not_an_error(tmp_path): + # Nothing serving (all already stopped) is a valid state, not the guard case. + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=[], + traffic=[], + stopped=["prod-104", "prod-102"], + ) + assert result.returncode == 0, result.stderr + assert _list_after("Stopping", result.stdout) == [] + + +# --- cleanup: robustness ------------------------------------------------- + + +def test_handles_absence_of_staging_versions(tmp_path): + # grep '^staging-' matches nothing; the script must not crash under set -e. + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-110"], + traffic=["prod-110"], + stopped=["prod-108", "prod-106", "20250101t000000"], + ) + assert result.returncode == 0, result.stderr + assert _list_after("Keeping stopped staging", result.stdout) == [] + assert _list_after("Deleting", result.stdout) == ["20250101t000000"] + + +# --- stop_app_engine_version.sh ------------------------------------------ + + +def test_stop_script_requires_a_version(tmp_path): + result, _ = _run(STOP_SCRIPT, tmp_path, extra_env={"APP_ENGINE_VERSION": ""}) + assert result.returncode != 0 + assert "APP_ENGINE_VERSION is required" in result.stderr + + +def test_stop_script_stops_the_named_version(tmp_path): + result, calls = _run( + STOP_SCRIPT, + tmp_path, + extra_env={"APP_ENGINE_VERSION": "staging-2405-abc1234", "DRY_RUN": "0"}, + ) + assert result.returncode == 0, result.stderr + logged = calls.read_text() + assert "STOP" in logged and "staging-2405-abc1234" in logged