ci: improve runner disk space usage situation

.github/workflows/matrix.yml

@@ -74,12 +74,71 @@ jobs:
         with:
           egress-policy: audit
 
+      - name: report disk space (before)
+        run: df -hT -x tmpfs -x devtmpfs -x squashfs -x efivarfs -x overlay
+      - name: reclaim runner disk space
+        # The runner image preinstalls tooling irrelevant to kernel builds
+        # (language runtimes, SDKs, browsers, cloud CLIs - tens of GB) on
+        # the root disk, which still holds the docker/containerd image
+        # store even with the build trees routed to a scratch disk. Absent
+        # paths are no-ops, so this tolerates image changes across runner
+        # classes. This step must run before any tool-installing action:
+        # it removes /opt/hostedtoolcache wholesale, and e.g.
+        # cosign-installer places cosign there (installers re-create what
+        # they need when they run afterwards).
+        run: |
+          sudo rm -rf \
+            /usr/local/lib/android \
+            /usr/share/dotnet \
+            /usr/lib/jvm \
+            /usr/share/swift \
+            /usr/local/.ghcup \
+            /usr/local/julia* \
+            /usr/local/share/chromium \
+            /usr/local/share/powershell \
+            /opt/microsoft \
+            /opt/google \
+            /opt/az \
+            /opt/hostedtoolcache \
+            /usr/lib/google-cloud-sdk \
+            /home/runner/.rustup \
+            /home/runner/.cargo
+          docker system prune -af >/dev/null 2>&1 || true
+          docker builder prune -af >/dev/null 2>&1 || true
+      - name: report disk space (after cleanup)
+        run: df -hT -x tmpfs -x devtmpfs -x squashfs -x efivarfs -x overlay
       - name: checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4
         with:
           submodules: recursive
       - name: install cosign
         uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+      - name: prepare scratch dir
+        # Some runner generations carry a secondary temp disk at /mnt; its
+        # size varies by VM SKU and is not reliably documented, so route the
+        # build trees there only when it is a distinct filesystem with more
+        # free space than the root disk. The device check comes first:
+        # comparing two df readings of the same filesystem can differ by a
+        # few KiB of measurement jitter and must not trigger the redirect.
+        # sudo because /mnt is root-owned while docker.sh and the
+        # in-container build user both run unprivileged.
+        run: |
+          ROOT_DEV="$(stat -c %d /)"
+          MNT_DEV="$(stat -c %d /mnt 2>/dev/null || echo "${ROOT_DEV}")"
+          if [ "${MNT_DEV}" = "${ROOT_DEV}" ]; then
+            echo "/mnt is not a distinct filesystem; keeping build trees on /"
+            exit 0
+          fi
+          ROOT_AVAIL="$(df --output=avail -k / | tail -1 | tr -d ' ')"
+          MNT_AVAIL="$(df --output=avail -k /mnt | tail -1 | tr -d ' ')"
+          if [ "${MNT_AVAIL}" -gt "${ROOT_AVAIL}" ]; then
+            sudo mkdir -p /mnt/kernel-scratch
+            sudo chmod a+rwX /mnt/kernel-scratch
+            echo "KERNEL_SCRATCH_DIR=/mnt/kernel-scratch" >> "${GITHUB_ENV}"
+            echo "using /mnt scratch (${MNT_AVAIL}K avail vs ${ROOT_AVAIL}K on /)"
+          else
+            echo "keeping build trees on / (${ROOT_AVAIL}K avail vs ${MNT_AVAIL}K on /mnt)"
+          fi
       - name: docker setup linux-kernel-oci
         run: sudo python3 ./hack/build/docker-setup.py
       - name: docker setup buildx
@@ -135,6 +194,10 @@ jobs:
           compression-level: 0
       - name: run docker script
         run: sh -x docker.sh
+      - name: report disk space (after)
+        # always() so ENOSPC failures still show where the disk went.
+        if: always()
+        run: df -hT -x tmpfs -x devtmpfs -x squashfs -x efivarfs -x overlay
       - name: upload digests
         # Only produced when publishing — push-by-digest path writes digests.json.
         if: ${{ inputs.publish }}

hack/build/firmware.sh

@@ -37,16 +37,21 @@ fi
 # Note that this assumes the archive is a .tar file, and has already been validated elsewhere.
 if [ -n "${FIRMWARE_URL}" ]; then
 	echo "untarring firmware at $FIRMWARE_URL"
-	tar -xf "${FIRMWARE_URL}" -C "${FIRMWARE_OUTPUT_PATH}" --strip-components=1
-	# For amdgpu zone kernel, we only want the amdgpu firmwares, so remove the rest to keep the addons small
 	if [ "${KERNEL_FLAVOR}" = "zone-amdgpu" ]; then
+		# Only the amdgpu/ subtree ends up in the addons; extracting just
+		# those members instead of the whole tree (and deleting the rest
+		# afterwards) keeps peak disk usage down - the full extraction has
+		# taken out builders with ENOSPC.
+		tar -xf "${FIRMWARE_URL}" -C "${FIRMWARE_OUTPUT_PATH}" --strip-components=1 \
+			--wildcards 'linux-firmware-*/amdgpu/*'
 		OLDDIR=$PWD
 		cd "${FIRMWARE_OUTPUT_PATH}"
-		find . -maxdepth 1 ! -name 'amdgpu' ! -name "." -exec rm -rf {} +
 		# Compress firmwares on-disk
 		# As of 6.x kernels the kconfig explicitly says you must use crc32 or none, not the default crc64.
 		xz -C crc32 amdgpu/*
 		cd "${OLDDIR}"
+	else
+		tar -xf "${FIRMWARE_URL}" -C "${FIRMWARE_OUTPUT_PATH}" --strip-components=1
 	fi
 fi
 

hack/build/generate-docker-script.py

@@ -127,6 +127,22 @@ def docker_compile(
     lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"]
     lines += ['mkdir -p "${HOME}/.cache/kernel-sccache" && chmod -R a+rwX "${HOME}/.cache/kernel-sccache"']
 
+    # The extracted kernel source and object trees are the dominant disk
+    # consumers of a build. By default they live in the container's writable
+    # layer (host root disk); setting KERNEL_SCRATCH_DIR redirects them onto a
+    # host directory via bind mounts. CI sets it only when the runner has a
+    # secondary disk with measurably more free space than / (see the "prepare
+    # scratch dir" step in matrix.yml); local builds leave it unset.
+    scratch_dir = os.getenv("KERNEL_SCRATCH_DIR")
+    if scratch_dir:
+        # chmod only the subdirectories this script creates and owns: in CI
+        # the scratch dir itself is root-owned (created by a sudo'd workflow
+        # step), so a chmod on it would fail for the unprivileged build user.
+        lines += [
+            'mkdir -p "%s/src" "%s/obj" && chmod a+rwX "%s/src" "%s/obj"'
+            % (scratch_dir, scratch_dir, scratch_dir, scratch_dir)
+        ]
+
     for arch in archs:
         platform = arch_to_platform(arch)
         compile_command = [
@@ -152,6 +168,11 @@ def docker_compile(
             "-v", quoted("${HOME}/.cache/kernel-sccache:/home/build/.cache/sccache"),
             "-v", quoted("${PWD}/target:/build/target"),
         ]
+        if scratch_dir:
+            compile_command += [
+                "-v", quoted("%s/src:/build/src" % scratch_dir),
+                "-v", quoted("%s/obj:/build/obj" % scratch_dir),
+            ]
         if has_firmware:
             compile_command += [
                 "-e", quoted("FIRMWARE_URL=/build/override-firmware.tar.xz"),