Stop idle App Engine versions after deploy

.github/scripts/cleanup_app_engine_versions.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+
+# Deactivate idle App Engine versions after a production promote.
+#
+# App Engine Flexible keeps at least one always-on VM for every SERVING version,
+# regardless of traffic, so old versions left SERVING after a promote keep
+# costing money even at 0 traffic (~$212/mo per 4 vCPU / 24 GB version). The
+# deploy pipeline promotes traffic with set-traffic but never deactivates the
+# versions it supersedes, so they pile up.
+#
+# This script, run after promote-production:
+#   1. Keeps the version currently receiving traffic PLUS the newest
+#      KEEP_WARM_PROD prod-* versions SERVING (a warm rollback window), and STOPs
+#      every other SERVING version (older prod + any leftover staging).
+#   2. DELETEs stopped versions beyond the retention window: keeps the newest
+#      KEEP_STOPPED_PROD prod-* and KEEP_STOPPED_STAGING staging-* stopped
+#      versions, and deletes every other stopped version (older prod, older
+#      staging, and legacy timestamp-named versions), to stay under App Engine's
+#      210-versions-per-service limit. Prod is kept deeper because only prod
+#      versions are meaningful rollback targets.
+#
+# Stopping (not deleting) preserves rollback: a stopped version can be brought
+# back with `gcloud app versions start <v>` then `gcloud app services set-traffic
+# ${SERVICE} --splits=<v>=1`. The version currently serving traffic is never
+# stopped or deleted.
+#
+# Set DRY_RUN=1 to print the plan without changing anything.
+#
+# Written to run on bash 3.2 (no mapfile / associative arrays) so it can be
+# dry-run locally on macOS as well as in CI.
+
+set -euo pipefail
+
+APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}"
+KEEP_WARM_PROD="${KEEP_WARM_PROD:-2}"
+KEEP_STOPPED_PROD="${KEEP_STOPPED_PROD:-10}"
+KEEP_STOPPED_STAGING="${KEEP_STOPPED_STAGING:-3}"
+DRY_RUN="${DRY_RUN:-0}"
+
+common_args=("--service=${APP_ENGINE_SERVICE}")
+if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
+  common_args+=("--project=${APP_ENGINE_PROJECT}")
+fi
+
+contains() {
+  local needle="$1"
+  shift
+  local item
+  for item in "$@"; do
+    [[ "${item}" == "${needle}" ]] && return 0
+  done
+  return 1
+}
+
+read_versions() {
+  # Args: extra gcloud filter. Emits version ids newest-first, one per line.
+  local filter="$1"
+  gcloud app versions list "${common_args[@]}" \
+    --filter="${filter}" \
+    --sort-by="~version.createTime" \
+    --format="value(id)"
+}
+
+# Versions currently receiving any traffic — never touched.
+serving_traffic=()
+while IFS= read -r v; do
+  [[ -n "${v}" ]] && serving_traffic+=("${v}")
+done < <(read_versions "version.servingStatus=SERVING AND traffic_split>0")
+
+# All SERVING versions, newest first.
+serving_all=()
+while IFS= read -r v; do
+  [[ -n "${v}" ]] && serving_all+=("${v}")
+done < <(read_versions "version.servingStatus=SERVING")
+
+# Fail safe: if there are SERVING versions but none report traffic, we cannot
+# identify the live service. Refuse to stop anything rather than risk
+# deactivating production (e.g. if the traffic query returns nothing). The
+# delete step never runs either, since we exit here.
+if [[ "${#serving_all[@]}" -gt 0 && "${#serving_traffic[@]}" -eq 0 ]]; then
+  echo "ERROR: ${#serving_all[@]} SERVING version(s) but none are receiving traffic;" >&2
+  echo "refusing to stop versions (cannot identify the live service). Aborting." >&2
+  exit 1
+fi
+
+# Newest KEEP_WARM_PROD prod-* versions to keep warm for instant rollback.
+warm_prod=()
+while IFS= read -r v; do
+  [[ -n "${v}" ]] && warm_prod+=("${v}")
+done < <(printf '%s\n' "${serving_all[@]:-}" | grep '^prod-' | head -n "${KEEP_WARM_PROD}")
+
+# Keep set = traffic-serving versions + warm prod versions.
+keep=("${serving_traffic[@]:-}" "${warm_prod[@]:-}")
+
+# Stop every SERVING version not in the keep set.
+to_stop=()
+for v in "${serving_all[@]:-}"; do
+  [[ -z "${v}" ]] && continue
+  contains "${v}" "${keep[@]:-}" && continue
+  to_stop+=("${v}")
+done
+
+echo "Service:                         ${APP_ENGINE_SERVICE}"
+echo "Serving traffic (never touched): ${serving_traffic[*]:-<none>}"
+echo "Keeping warm (rollback window):  ${warm_prod[*]:-<none>}"
+echo "Stopping (${#to_stop[@]}):                     ${to_stop[*]:-<none>}"
+
+if [[ "${#to_stop[@]}" -gt 0 ]]; then
+  if [[ "${DRY_RUN}" == "1" ]]; then
+    echo "[dry-run] would stop ${#to_stop[@]} version(s)"
+  else
+    gcloud app versions stop "${to_stop[@]}" "${common_args[@]}" --quiet
+  fi
+fi
+
+# Delete stopped versions beyond the retention window (version-quota hygiene).
+# Keep the newest KEEP_STOPPED_PROD stopped prod-* versions (cold rollback
+# targets) and the newest KEEP_STOPPED_STAGING stopped staging-* versions; delete
+# everything else stopped — older prod, older staging, and legacy timestamp-named
+# versions. Serving/warm versions are SERVING, so never appear in this set.
+stopped_all=()
+while IFS= read -r v; do
+  [[ -n "${v}" ]] && stopped_all+=("${v}")
+done < <(read_versions "version.servingStatus=STOPPED")
+
+# Newest KEEP_STOPPED_PROD stopped prod-* versions to keep.
+keep_stopped_prod=()
+while IFS= read -r v; do
+  [[ -n "${v}" ]] && keep_stopped_prod+=("${v}")
+done < <(printf '%s\n' "${stopped_all[@]:-}" | grep '^prod-' | head -n "${KEEP_STOPPED_PROD}")
+
+# Newest KEEP_STOPPED_STAGING stopped staging-* versions to keep.
+keep_stopped_staging=()
+while IFS= read -r v; do
+  [[ -n "${v}" ]] && keep_stopped_staging+=("${v}")
+done < <(printf '%s\n' "${stopped_all[@]:-}" | grep '^staging-' | head -n "${KEEP_STOPPED_STAGING}")
+
+keep_stopped=("${keep_stopped_prod[@]:-}" "${keep_stopped_staging[@]:-}")
+
+to_delete=()
+for v in "${stopped_all[@]:-}"; do
+  [[ -z "${v}" ]] && continue
+  contains "${v}" "${keep_stopped[@]:-}" && continue
+  to_delete+=("${v}")
+done
+
+echo "Stopped versions:                ${#stopped_all[@]}"
+echo "Keeping stopped prod (${#keep_stopped_prod[@]}):          ${keep_stopped_prod[*]:-<none>}"
+echo "Keeping stopped staging (${#keep_stopped_staging[@]}):       ${keep_stopped_staging[*]:-<none>}"
+echo "Deleting (${#to_delete[@]}):                     ${to_delete[*]:-<none>}"
+
+if [[ "${#to_delete[@]}" -gt 0 ]]; then
+  if [[ "${DRY_RUN}" == "1" ]]; then
+    echo "[dry-run] would delete ${#to_delete[@]} version(s)"
+  else
+    gcloud app versions delete "${to_delete[@]}" "${common_args[@]}" --quiet
+  fi
+fi

.github/scripts/stop_app_engine_version.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Stop (deactivate) a single App Engine version.
+#
+# App Engine Flexible keeps at least one VM running for every SERVING version
+# regardless of traffic, so a version left SERVING after its tests keeps costing
+# money. Stopping it drops its instances to zero (no billing) while keeping the
+# version deployed and restartable (`gcloud app versions start`) for rollback.
+
+set -euo pipefail
+
+: "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}"
+
+APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}"
+
+stop_args=(
+  "${APP_ENGINE_VERSION}"
+  "--service=${APP_ENGINE_SERVICE}"
+  "--quiet"
+)
+
+if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
+  stop_args+=("--project=${APP_ENGINE_PROJECT}")
+fi
+
+echo "Stopping App Engine version ${APP_ENGINE_VERSION} (service ${APP_ENGINE_SERVICE})"
+gcloud app versions stop "${stop_args[@]}"

.github/workflows/push.yml

@@ -286,6 +286,37 @@ jobs:
           API_BASE_URL: ${{ needs.deploy-staging.outputs.url }}
           STAGING_API_TEST_PROBE_ID: ${{ needs.deploy-staging.outputs.version }}
 
+  stop-staging-app-engine-version:
+    name: Stop staging App Engine version
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-staging
+      - integration-tests-staging
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    environment: staging
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: GCP authentication
+        uses: "google-github-actions/auth@v2"
+        with:
+          workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
+      - name: Set up GCloud
+        uses: "google-github-actions/setup-gcloud@v2"
+      # Flex keeps a VM running for every SERVING version regardless of traffic,
+      # so the just-tested staging version would keep costing money. Stop it now
+      # that its integration tests have passed; the next deploy makes a fresh one.
+      - name: Stop staging App Engine version
+        run: bash .github/scripts/stop_app_engine_version.sh
+        env:
+          APP_ENGINE_VERSION: ${{ needs.deploy-staging.outputs.version }}
+
   integration-tests-staging-cloud-run:
     name: Run Cloud Run staging integration tests
     runs-on: ubuntu-latest
@@ -468,6 +499,35 @@ jobs:
         env:
           APP_ENGINE_VERSION: ${{ needs.deploy-production-candidate.outputs.version }}
 
+  cleanup-prod-app-engine-versions:
+    name: Clean up idle production App Engine versions
+    runs-on: ubuntu-latest
+    needs: promote-production
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    environment: production
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: GCP authentication
+        uses: "google-github-actions/auth@v2"
+        with:
+          workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
+      - name: Set up GCloud
+        uses: "google-github-actions/setup-gcloud@v2"
+      # Keep the live version plus the 2 newest prod versions warm for instant
+      # rollback; stop every other SERVING version (older prod + leftover staging)
+      # so idle Flex VMs stop billing; keep the newest 10 stopped prod and 3
+      # stopped staging versions and delete the rest to stay under the
+      # 210-versions-per-service limit. Retention counts are defaults in the script.
+      - name: Stop superseded versions and prune old stopped versions
+        run: bash .github/scripts/cleanup_app_engine_versions.sh
+
   deploy-cloud-run-candidate:
     name: Deploy production Cloud Run candidate
     runs-on: ubuntu-latest

changelog.d/app-engine-idle-version-cleanup.changed.md

@@ -0,0 +1 @@
+Deploys now stop idle App Engine versions automatically. The staging version is stopped after its integration tests pass, and after a production promote only the two newest production versions stay warm while every other superseded version is stopped. Stopped versions are then pruned to the newest 10 production and newest 3 staging (older prod, older staging, and legacy timestamp-named versions are deleted). This removes always-on Flexible-environment VM cost from versions that serve no traffic, while keeping recent production versions available for rollback via `gcloud app versions start`.