diff --git a/.github/workflows/phpunit-tests.yml b/.github/workflows/phpunit-tests.yml index 81afd70449141..727a428e27859 100644 --- a/.github/workflows/phpunit-tests.yml +++ b/.github/workflows/phpunit-tests.yml @@ -37,6 +37,7 @@ on: # Confirm any changes to relevant workflow files. - '.github/workflows/phpunit-tests.yml' - '.github/workflows/reusable-phpunit-tests-*.yml' + - '.github/workflows/reusable-prepare-gutenberg.yml' workflow_dispatch: # Once weekly On Sundays at 00:00 UTC. schedule: @@ -54,6 +55,20 @@ concurrency: permissions: {} jobs: + # Downloads and verifies the Gutenberg build once for all PHPUnit jobs. + # + # This condition is the union of the conditions on the jobs that need it, reduced. + # The org matrices require `WordPress/wordpress-develop` or a pull request, and the + # fork matrix requires a pull request, so `wordpress-develop` or a pull request + # covers every case. Keep it in step with those jobs: a narrower condition orphans + # them, because a job that needs a skipped job is skipped too, and a broader one + # downloads a build that nothing consumes. + prepare-gutenberg: + uses: ./.github/workflows/reusable-prepare-gutenberg.yml + permissions: + contents: read + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }} + # # Creates a PHPUnit test job for each PHP/MySQL combination. # @@ -64,6 +79,7 @@ jobs: test-with-mysql: name: PHP ${{ matrix.php }} uses: ./.github/workflows/reusable-phpunit-tests-v3.yml + needs: prepare-gutenberg permissions: contents: read secrets: @@ -130,6 +146,8 @@ jobs: phpunit-config: ${{ matrix.multisite && 'tests/phpunit/multisite.xml' || 'phpunit.xml.dist' }} tests-domain: ${{ matrix.tests-domain }} report: ${{ matrix.report || false }} + gutenberg-artifact: gutenberg-build + gutenberg-sha: ${{ needs.prepare-gutenberg.outputs.gutenberg-sha }} # # Creates a PHPUnit test job for each PHP/MariaDB combination. @@ -143,6 +161,7 @@ jobs: test-with-mariadb: name: PHP ${{ matrix.php }} uses: ./.github/workflows/reusable-phpunit-tests-v3.yml + needs: prepare-gutenberg permissions: contents: read secrets: @@ -182,6 +201,8 @@ jobs: memcached: ${{ matrix.memcached }} phpunit-config: ${{ matrix.multisite && 'tests/phpunit/multisite.xml' || 'phpunit.xml.dist' }} report: false + gutenberg-artifact: gutenberg-build + gutenberg-sha: ${{ needs.prepare-gutenberg.outputs.gutenberg-sha }} # # Creates PHPUnit test jobs to test MariaDB and MySQL innovation releases. @@ -197,6 +218,7 @@ jobs: test-innovation-releases: name: PHP ${{ matrix.php }} uses: ./.github/workflows/reusable-phpunit-tests-v3.yml + needs: prepare-gutenberg permissions: contents: read secrets: @@ -228,6 +250,8 @@ jobs: memcached: ${{ matrix.memcached }} phpunit-config: ${{ matrix.multisite && 'tests/phpunit/multisite.xml' || 'phpunit.xml.dist' }} report: false + gutenberg-artifact: gutenberg-build + gutenberg-sha: ${{ needs.prepare-gutenberg.outputs.gutenberg-sha }} # # Runs the HTML API test group. @@ -240,6 +264,7 @@ jobs: html-api-test-groups: name: ${{ matrix.label }} uses: ./.github/workflows/reusable-phpunit-tests-v3.yml + needs: prepare-gutenberg permissions: contents: read secrets: @@ -260,6 +285,8 @@ jobs: db-type: ${{ matrix.db-type }} db-version: ${{ matrix.db-version }} phpunit-test-groups: ${{ matrix.phpunit-test-groups }} + gutenberg-artifact: gutenberg-build + gutenberg-sha: ${{ needs.prepare-gutenberg.outputs.gutenberg-sha }} # # Runs unit tests for forks. @@ -271,6 +298,7 @@ jobs: limited-matrix-for-forks: name: PHP ${{ matrix.php }} uses: ./.github/workflows/reusable-phpunit-tests-v3.yml + needs: prepare-gutenberg permissions: contents: read secrets: @@ -320,6 +348,8 @@ jobs: db-type: ${{ matrix.db-type }} memcached: ${{ matrix.memcached || false }} phpunit-test-groups: ${{ matrix.phpunit-test-groups || '' }} + gutenberg-artifact: gutenberg-build + gutenberg-sha: ${{ needs.prepare-gutenberg.outputs.gutenberg-sha }} slack-notifications: name: Slack Notifications @@ -327,7 +357,7 @@ jobs: permissions: actions: read contents: read - needs: [ test-with-mysql, test-with-mariadb, test-innovation-releases, html-api-test-groups, limited-matrix-for-forks ] + needs: [ prepare-gutenberg, test-with-mysql, test-with-mariadb, test-innovation-releases, html-api-test-groups, limited-matrix-for-forks ] if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} with: calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} diff --git a/.github/workflows/reusable-phpunit-tests-v3.yml b/.github/workflows/reusable-phpunit-tests-v3.yml index dfe678e82a0da..df0945cd4df62 100644 --- a/.github/workflows/reusable-phpunit-tests-v3.yml +++ b/.github/workflows/reusable-phpunit-tests-v3.yml @@ -72,6 +72,16 @@ on: required: false type: boolean default: false + gutenberg-artifact: + description: 'The name of a same-workflow artifact containing the prepared Gutenberg build.' + required: false + type: string + default: '' + gutenberg-sha: + description: 'The immutable Gutenberg source SHA verified by the calling workflow.' + required: false + type: string + default: '' secrets: CODECOV_TOKEN: description: 'The Codecov token required for uploading reports.' @@ -101,6 +111,7 @@ jobs: # Performs the following steps: # - Sets environment variables. # - Checks out the repository. + # - Downloads the prepared Gutenberg build when provided by the calling workflow. # - Sets up Node.js. # - Sets up PHP. # - Installs Composer dependencies. @@ -135,6 +146,15 @@ jobs: show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} persist-credentials: false + - name: Download prepared Gutenberg build + if: ${{ inputs.gutenberg-artifact != '' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + # Without a run ID, this action reads only from the caller's current workflow run. + name: ${{ inputs.gutenberg-artifact }} + path: gutenberg + digest-mismatch: error + - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -166,6 +186,9 @@ jobs: - name: Build WordPress run: npm run build:dev + env: + # The producer resolved this once. Matrix jobs must not re-resolve mutable GHCR tags. + GUTENBERG_EXPECTED_SHA: ${{ inputs.gutenberg-sha }} - name: General debug information run: | diff --git a/.github/workflows/reusable-prepare-gutenberg.yml b/.github/workflows/reusable-prepare-gutenberg.yml new file mode 100644 index 0000000000000..7e481ddc89f2b --- /dev/null +++ b/.github/workflows/reusable-prepare-gutenberg.yml @@ -0,0 +1,87 @@ +## +# A reusable workflow that downloads and verifies the Gutenberg build once per +# calling workflow run. +## +name: Prepare Gutenberg build + +on: + workflow_call: + outputs: + gutenberg-sha: + description: 'The immutable Gutenberg source SHA verified by this workflow.' + value: ${{ jobs.prepare-gutenberg.outputs.gutenberg-sha }} + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + prepare-gutenberg: + name: Gutenberg + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + gutenberg-sha: ${{ steps.download.outputs.gutenberg-sha }} + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + # Resolve the Gutenberg ref from the exact commit that started this workflow run. + ref: ${{ github.sha }} + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: '.nvmrc' + + - name: Resolve Gutenberg metadata + id: metadata + run: node tools/gutenberg/download.js --metadata + + - name: Restore verified Gutenberg archive + id: restore + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .ci/gutenberg/archive.tar.gz + key: gutenberg-oci-archive-v1-${{ steps.metadata.outputs.source-sha }}-sha256-${{ steps.metadata.outputs.blob-sha256 }} + + - name: Download verified Gutenberg archive + if: ${{ steps.restore.outputs.cache-hit != 'true' }} + run: | + rm -f .ci/gutenberg/archive.tar.gz + node tools/gutenberg/download.js \ + --archive .ci/gutenberg/archive.tar.gz \ + --source-sha "${{ steps.metadata.outputs.source-sha }}" \ + --blob-sha256 "${{ steps.metadata.outputs.blob-sha256 }}" \ + --blob-size "${{ steps.metadata.outputs.blob-size }}" + + - name: Verify and extract Gutenberg build + id: download + run: | + node tools/gutenberg/download.js \ + --archive .ci/gutenberg/archive.tar.gz \ + --source-sha "${{ steps.metadata.outputs.source-sha }}" \ + --blob-sha256 "${{ steps.metadata.outputs.blob-sha256 }}" \ + --blob-size "${{ steps.metadata.outputs.blob-size }}" + echo "gutenberg-sha=${{ steps.metadata.outputs.source-sha }}" >> "$GITHUB_OUTPUT" + + - name: Save verified Gutenberg archive + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name == 'push' && startsWith(github.ref, 'refs/heads/') && github.ref_protected == true && steps.restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .ci/gutenberg/archive.tar.gz + key: gutenberg-oci-archive-v1-${{ steps.metadata.outputs.source-sha }}-sha256-${{ steps.metadata.outputs.blob-sha256 }} + + - name: Upload Gutenberg build + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gutenberg-build + path: gutenberg/ + if-no-files-found: error + include-hidden-files: true + retention-days: 1 diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index d6fe09904a925..ffc650ec370fb 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -8,6 +8,10 @@ on: paths: - '.github/workflows/test-coverage.yml' - '.github/workflows/reusable-phpunit-tests-v3.yml' + - '.github/workflows/reusable-prepare-gutenberg.yml' + - 'tools/gutenberg/**' + - 'package*.json' + - '.nvmrc' - 'docker-compose.yml' - 'phpunit.xml.dist' - 'tests/phpunit/multisite.xml' @@ -17,6 +21,10 @@ on: paths: - '.github/workflows/test-coverage.yml' - '.github/workflows/reusable-phpunit-tests-v3.yml' + - '.github/workflows/reusable-prepare-gutenberg.yml' + - 'tools/gutenberg/**' + - 'package*.json' + - '.nvmrc' - 'docker-compose.yml' - 'phpunit.xml.dist' - 'tests/phpunit/multisite.xml' @@ -42,12 +50,20 @@ env: PUPPETEER_SKIP_DOWNLOAD: true jobs: + # Downloads and verifies the Gutenberg build once for all coverage jobs. + prepare-gutenberg: + uses: ./.github/workflows/reusable-prepare-gutenberg.yml + permissions: + contents: read + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + # # Creates a PHPUnit test jobs for generating code coverage reports. # test-coverage-report: name: ${{ matrix.multisite && 'Multisite' || 'Single site' }} report uses: ./.github/workflows/reusable-phpunit-tests-v3.yml + needs: prepare-gutenberg permissions: contents: read if: ${{ github.repository == 'WordPress/wordpress-develop' }} @@ -60,6 +76,8 @@ jobs: php: '8.3' multisite: ${{ matrix.multisite }} coverage-report: ${{ matrix.coverage-report }} + gutenberg-artifact: gutenberg-build + gutenberg-sha: ${{ needs.prepare-gutenberg.outputs.gutenberg-sha }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -69,7 +87,7 @@ jobs: permissions: actions: read contents: read - needs: [ test-coverage-report ] + needs: [ prepare-gutenberg, test-coverage-report ] if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} with: calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} diff --git a/.gitignore b/.gitignore index 15876fa47fee8..16928245b9496 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ wp-tests-config.php /packagehash.txt /.gutenberg-hash /artifacts +/.ci/gutenberg/archive.tar.gz /setup.log /coverage diff --git a/package.json b/package.json index a4aff38cf3dd7..30b887e3ed0b6 100644 --- a/package.json +++ b/package.json @@ -137,6 +137,7 @@ "test:php": "node ./tools/local-env/scripts/docker.js run --rm php ./vendor/bin/phpunit", "test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt", "test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js", + "test:tools": "node --test tools/gutenberg/tests/", "test:visual": "wp-scripts test-playwright --config tests/visual-regression/playwright.config.js", "typecheck:php": "node ./tools/local-env/scripts/docker.js run --rm php composer phpstan", "gutenberg:copy": "node tools/gutenberg/copy.js", diff --git a/tools/gutenberg/download.js b/tools/gutenberg/download.js index fd76c6c7a7836..4cda1fa5475ac 100644 --- a/tools/gutenberg/download.js +++ b/tools/gutenberg/download.js @@ -1,214 +1,782 @@ #!/usr/bin/env node +/* eslint-disable no-console */ /** * Download Gutenberg Repository Script. * * This script downloads a pre-built Gutenberg tar.gz artifact from the GitHub - * Container Registry and extracts it into the ./gutenberg directory. Any - * existing gutenberg directory is removed before extraction. + * Container Registry and extracts it into the ./gutenberg directory. An + * existing gutenberg directory is replaced only after verification and + * extraction succeed. * - * The artifact is identified by the "gutenberg.sha" value in the root - * package.json, which is used as the OCI tag for the gutenberg-wp-develop-build - * package on GitHub Container Registry. The value is normally a Git SHA, but - * may also be a mutable tag (e.g. "trunk", "pr-12345") in a pull request that - * wants to track the latest build of a stream. When the ref is a mutable tag, - * the script resolves it to the immutable SHA tag for the actual blob fetch - * and falls back to the mutable tag's manifest when the immutable tag is - * unavailable. - * - * @package WordPress + * @package */ const { spawn } = require( 'child_process' ); +const crypto = require( 'crypto' ); const fs = require( 'fs' ); +const os = require( 'os' ); +const path = require( 'path' ); const { Readable } = require( 'stream' ); const { pipeline } = require( 'stream/promises' ); -const zlib = require( 'zlib' ); +const { Transform } = require( 'stream' ); const { + rootDir, gutenbergDir, readGutenbergConfig, fetchGhcrToken, fetchManifest, } = require( './utils' ); +const MAX_DOWNLOAD_ATTEMPTS = 3; +const RETRY_DELAY_MS = 2000; +const DOWNLOAD_TIMEOUT_MS = 120000; +const SOURCE_SHA_PATTERN = /^[a-f0-9]{40}$/i; +const BLOB_SHA256_PATTERN = /^[a-f0-9]{64}$/i; + /** - * Resolve the manifest to use for downloading. + * Convert bytes into a readable string for download diagnostics. * - * For immutable refs (SHA values), the ref is used directly. + * @param {number} bytes Number of bytes. + * @return {string} Formatted byte count. + */ +function formatBytes( bytes ) { + return `${ ( bytes / 1024 / 1024 ).toFixed( 2 ) } MiB (${ bytes } bytes)`; +} + +/** + * Wait before retrying a failed download. * - * For mutable refs, the mutable tag's manifest is fetched first and the - * `image.revision` annotation is read. The corresponding immutable SHA tag is - * then preferred. If the immutable SHA tag is unavailable, fall back to the - * manifest already fetched via the mutable tag. + * @param {number} milliseconds Time to wait in milliseconds. + * @return {Promise} Resolves after the requested delay. + */ +function delay( milliseconds ) { + return new Promise( ( resolve ) => setTimeout( resolve, milliseconds ) ); +} + +/** + * Create an error that retains the HTTP status code for retry decisions. * - * @param {{ ref: string, ghcrRepo: string, isMutable: boolean }} config - * @param {string} token - * @return {Promise<{ manifest: Record, resolvedRef: string }>} + * @param {string} message Error message. + * @param {number} status HTTP status code. + * @return {Error & { status: number }} Error with status code. */ -async function resolveDownloadManifest( config, token ) { - const { ref, ghcrRepo, isMutable } = config; +function createHttpError( message, status ) { + const error = /** @type {Error & { status: number }} */ ( + new Error( message ) + ); + error.status = status; + return error; +} + +/** + * Determine whether a failed download might succeed on a later attempt. + * + * @param {Error & { status?: number }} error Download error. + * @return {boolean} Whether the error is retryable. + */ +function isRetryableDownloadError( error ) { + return ( + ! error.status || + error.status === 408 || + error.status === 429 || + error.status >= 500 + ); +} + +/** + * Normalize an immutable Gutenberg source SHA. + * + * @param {string} sourceSha Source SHA. + * @return {string} Lowercase source SHA. + */ +function normalizeSourceSha( sourceSha ) { + if ( ! SOURCE_SHA_PATTERN.test( sourceSha ) ) { + throw new Error( + `Expected a 40-character Gutenberg SHA, received: ${ sourceSha }` + ); + } - const initialManifest = await fetchManifest( ref, ghcrRepo, token ); + return sourceSha.toLowerCase(); +} + +/** + * Normalize an OCI blob SHA-256 value. + * + * @param {string} blobSha256 OCI blob SHA-256 value, with or without prefix. + * @return {string} Lowercase SHA-256 hex value. + */ +function normalizeBlobSha256( blobSha256 ) { + const value = blobSha256.replace( /^sha256:/i, '' ); + if ( ! BLOB_SHA256_PATTERN.test( value ) ) { + throw new Error( `Unsupported OCI layer digest: ${ blobSha256 }` ); + } + + return value.toLowerCase(); +} + +/** + * Validate metadata used to identify and verify a compressed blob. + * + * @param {{ sourceSha: string, blobSha256: string, blobSize: number }} metadata Blob metadata. + * @return {{ sourceSha: string, blobSha256: string, blobSize: number }} Normalized metadata. + */ +function normalizeMetadata( metadata ) { + const blobSize = Number( metadata.blobSize ); + if ( ! Number.isSafeInteger( blobSize ) || blobSize < 0 ) { + throw new Error( `Invalid OCI layer size: ${ metadata.blobSize }` ); + } + + return { + sourceSha: normalizeSourceSha( metadata.sourceSha ), + blobSha256: normalizeBlobSha256( metadata.blobSha256 ), + blobSize, + }; +} + +/** + * Read metadata for the first layer of an OCI manifest. + * + * @param {Record} manifest OCI manifest. + * @param {string} sourceSha Immutable Gutenberg source SHA. + * @return {{ sourceSha: string, blobSha256: string, blobSize: number }} Blob metadata. + */ +function getManifestMetadata( manifest, sourceSha ) { + const layer = manifest?.layers?.[ 0 ]; + if ( ! layer?.digest ) { + throw new Error( 'No layer digest found in manifest' ); + } + + return normalizeMetadata( { + sourceSha, + blobSha256: layer.digest, + blobSize: layer.size, + } ); +} + +/** + * Resolve the immutable manifest used to identify a Gutenberg build. + * + * Mutable refs first resolve their image revision, then fetch the manifest + * again at that immutable revision. A mutable manifest is never used for a + * download. + * + * @param {{ ref: string, ghcrRepo: string, isMutable: boolean }} config Gutenberg configuration. + * @param {string} token GHCR pull token. + * @param {(ref: string, ghcrRepo: string, token: string) => Promise>} [fetchManifestFn] Manifest fetcher. + * @return {Promise<{ manifest: Record, sourceSha: string }>} Immutable manifest and source SHA. + */ +async function resolveDownloadManifest( + config, + token, + fetchManifestFn = fetchManifest +) { + const { ref, ghcrRepo, isMutable } = config; if ( ! isMutable ) { - return { manifest: initialManifest, resolvedRef: ref }; + const sourceSha = normalizeSourceSha( ref ); + return { + manifest: await fetchManifestFn( sourceSha, ghcrRepo, token ), + sourceSha, + }; } + const mutableManifest = await fetchManifestFn( ref, ghcrRepo, token ); const revision = - initialManifest?.annotations?.[ 'org.opencontainers.image.revision' ]; - if ( ! revision ) { - console.log( - `ā„¹ļø No image.revision annotation on "${ ref }"; using mutable tag for download.` + mutableManifest?.annotations?.[ 'org.opencontainers.image.revision' ]; + if ( ! revision || ! SOURCE_SHA_PATTERN.test( revision ) ) { + throw new Error( + `Manifest for mutable ref "${ ref }" has no valid org.opencontainers.image.revision SHA` ); - return { manifest: initialManifest, resolvedRef: ref }; } - try { - const immutableManifest = await fetchManifest( revision, ghcrRepo, token ); - return { manifest: immutableManifest, resolvedRef: revision }; - } catch ( error ) { - if ( /** @type {{ status?: number }} */ ( error ).status === 404 ) { - console.log( - `ā„¹ļø Immutable SHA tag ${ revision } unavailable; falling back to mutable tag "${ ref }".` - ); - return { manifest: initialManifest, resolvedRef: ref }; - } - throw error; + const sourceSha = revision.toLowerCase(); + return { + manifest: await fetchManifestFn( sourceSha, ghcrRepo, token ), + sourceSha, + }; +} + +/** + * Resolve the source SHA and blob metadata for a Gutenberg build. + * + * @param {{ ref: string, ghcrRepo: string, isMutable: boolean }} config Gutenberg configuration. + * @param {string} token GHCR pull token. + * @param {(ref: string, ghcrRepo: string, token: string) => Promise>} [fetchManifestFn] Manifest fetcher. + * @return {Promise<{ sourceSha: string, blobSha256: string, blobSize: number }>} Resolved blob metadata. + */ +async function resolveDownloadMetadata( + config, + token, + fetchManifestFn = fetchManifest +) { + const { manifest, sourceSha } = await resolveDownloadManifest( + config, + token, + fetchManifestFn + ); + return getManifestMetadata( manifest, sourceSha ); +} + +/** + * Write resolved metadata to the GitHub Actions step output file. + * + * @param {{ sourceSha: string, blobSha256: string, blobSize: number }} metadata Blob metadata. + * @param {string} outputPath GitHub Actions output file. + * @return {void} + */ +function writeMetadataOutput( metadata, outputPath ) { + const normalized = normalizeMetadata( metadata ); + if ( ! outputPath ) { + throw new Error( + 'GITHUB_OUTPUT is required when resolving Gutenberg metadata' + ); } + + fs.appendFileSync( + outputPath, + `source-sha=${ normalized.sourceSha }\n` + + `blob-sha256=${ normalized.blobSha256 }\n` + + `blob-size=${ normalized.blobSize }\n` + ); } /** - * Main execution function. + * Download a blob to disk and verify its SHA-256 digest and byte count. + * + * @param {string} url Blob URL. + * @param {string} token Bearer token for GHCR. + * @param {{ blobSha256: string, blobSize: number }} metadata Blob metadata. + * @param {string} destination Path where the compressed blob is written. + * @return {Promise} Resolves after the download is verified. */ -async function main() { - console.log( 'šŸ” Checking Gutenberg configuration...' ); +async function downloadAndVerifyBlob( url, token, metadata, destination ) { + const { blobSha256, blobSize } = normalizeMetadata( { + sourceSha: '0'.repeat( 40 ), + ...metadata, + } ); + const response = await fetch( url, { + headers: { + Authorization: `Bearer ${ token }`, + }, + signal: AbortSignal.timeout( DOWNLOAD_TIMEOUT_MS ), + } ); - /* - * Read Gutenberg configuration from package.json. - * - * Note: ghcr stands for GitHub Container Registry where wordpress-develop ready builds of the Gutenberg plugin - * are published by the Gutenberg build-plugin-zip workflow. - */ - let config; - try { - config = readGutenbergConfig(); - console.log( - ` Ref: ${ config.ref }${ - config.isMutable ? ' (mutable tag)' : '' - }` + console.log( ` Response: ${ response.status } ${ response.statusText }` ); + + if ( ! response.ok ) { + throw createHttpError( + `Failed to download blob: ${ response.status } ${ response.statusText }`, + response.status ); - console.log( ` GHCR repository: ${ config.ghcrRepo }` ); - } catch ( error ) { - console.error( 'āŒ Error reading package.json:', /** @type {Error} */ ( error ).message ); - process.exit( 1 ); } - // Step 1: Get an anonymous GHCR token for pulling. - console.log( '\nšŸ”‘ Fetching GHCR token...' ); - let token; - try { - token = await fetchGhcrToken( config.ghcrRepo ); - console.log( 'āœ… Token acquired' ); - } catch ( error ) { - console.error( 'āŒ Failed to fetch token:', /** @type {Error} */ ( error ).message ); - process.exit( 1 ); + if ( ! response.body ) { + throw new Error( 'Blob response has no body' ); } - // Step 2: Resolve the manifest to use for download. - console.log( `\nšŸ“‹ Fetching manifest for ${ config.ref }...` ); - let manifest, resolvedRef; + const contentLength = Number( response.headers.get( 'content-length' ) ); + if ( Number.isFinite( contentLength ) && contentLength >= 0 ) { + console.log( ` Content-Length: ${ formatBytes( contentLength ) }` ); + } + console.log( ` Manifest size: ${ formatBytes( blobSize ) }` ); + + let downloadedBytes = 0; + const hash = crypto.createHash( 'sha256' ); + const meter = new Transform( { + transform( chunk, _encoding, callback ) { + downloadedBytes += chunk.length; + hash.update( chunk ); + callback( null, chunk ); + }, + } ); + try { - ( { manifest, resolvedRef } = await resolveDownloadManifest( - config, - token - ) ); - if ( resolvedRef !== config.ref ) { - console.log( ` Resolved to immutable SHA tag: ${ resolvedRef }` ); - } + await pipeline( + Readable.fromWeb( + /** @type {import('stream/web').ReadableStream} */ ( + response.body + ) + ), + meter, + fs.createWriteStream( destination ) + ); } catch ( error ) { - console.error( 'āŒ Failed to fetch manifest:', /** @type {Error} */ ( error ).message ); - process.exit( 1 ); + throw new Error( + `Download interrupted after ${ formatBytes( downloadedBytes ) }: ${ + /** @type {Error} */ ( error ).message + }` + ); } - const digest = manifest?.layers?.[ 0 ]?.digest; - if ( ! digest ) { - console.error( 'āŒ No layer digest found in manifest' ); - process.exit( 1 ); + if ( + Number.isFinite( contentLength ) && + downloadedBytes !== contentLength + ) { + throw new Error( + `Downloaded ${ formatBytes( + downloadedBytes + ) }, but Content-Length was ${ formatBytes( contentLength ) }` + ); } - console.log( `āœ… Blob digest: ${ digest }` ); - // Remove existing gutenberg directory so the extraction is clean. - if ( fs.existsSync( gutenbergDir ) ) { - console.log( '\nšŸ—‘ļø Removing existing gutenberg directory...' ); - fs.rmSync( gutenbergDir, { recursive: true, force: true } ); + if ( downloadedBytes !== blobSize ) { + throw new Error( + `Downloaded ${ formatBytes( + downloadedBytes + ) }, but manifest size was ${ formatBytes( blobSize ) }` + ); } - fs.mkdirSync( gutenbergDir, { recursive: true } ); + const actualSha256 = hash.digest( 'hex' ); + if ( actualSha256 !== blobSha256 ) { + throw new Error( + `SHA-256 mismatch: expected ${ blobSha256 } but received ${ actualSha256 }` + ); + } - /* - * Step 3: Stream the blob directly through gunzip into tar, writing - * into ./gutenberg with no temporary file on disk. - */ - console.log( `\nšŸ“„ Downloading and extracting artifact...` ); - try { - const response = await fetch( `https://ghcr.io/v2/${ config.ghcrRepo }/blobs/${ digest }`, { - headers: { - Authorization: `Bearer ${ token }`, - }, - } ); - if ( ! response.ok ) { - throw new Error( `Failed to download blob: ${ response.status } ${ response.statusText }` ); + console.log( + `āœ… Downloaded ${ formatBytes( downloadedBytes ) } and verified SHA-256` + ); +} + +/** + * Remove partial archives left by a failed or interrupted download. + * + * @param {string} archivePath Stable archive path. + * @return {void} + */ +function removePartialArchives( archivePath ) { + const directory = path.dirname( archivePath ); + const prefix = `${ path.basename( archivePath ) }.partial-`; + if ( ! fs.existsSync( directory ) ) { + return; + } + + for ( const file of fs.readdirSync( directory ) ) { + if ( file.startsWith( prefix ) ) { + fs.rmSync( path.join( directory, file ), { force: true } ); } - if ( ! response.body ) { - throw new Error( 'Blob response has no body' ); + } +} + +/** + * Download a blob with bounded retries. Each attempt writes to a unique + * partial path, which is renamed to the stable path only after verification. + * + * @param {string} url Blob URL. + * @param {string} token Bearer token for GHCR. + * @param {{ sourceSha: string, blobSha256: string, blobSize: number }} metadata Blob metadata. + * @param {string} archivePath Stable archive path. + * @param {(milliseconds: number) => Promise} [delayFn] Retry delay function. + * @return {Promise} Path to the verified compressed blob. + */ +async function downloadBlobWithRetries( + url, + token, + metadata, + archivePath, + delayFn = delay +) { + const normalized = normalizeMetadata( metadata ); + fs.mkdirSync( path.dirname( archivePath ), { recursive: true } ); + fs.rmSync( archivePath, { force: true } ); + removePartialArchives( archivePath ); + + for ( let attempt = 1; attempt <= MAX_DOWNLOAD_ATTEMPTS; attempt++ ) { + const partialPath = `${ archivePath }.partial-${ process.pid }-${ attempt }`; + console.log( + `\nšŸ“„ Download attempt ${ attempt }/${ MAX_DOWNLOAD_ATTEMPTS }...` + ); + fs.rmSync( partialPath, { force: true } ); + + try { + await downloadAndVerifyBlob( url, token, normalized, partialPath ); + fs.renameSync( partialPath, archivePath ); + return archivePath; + } catch ( error ) { + const downloadError = /** @type {Error & { status?: number }} */ ( + error + ); + fs.rmSync( partialPath, { force: true } ); + fs.rmSync( archivePath, { force: true } ); + removePartialArchives( archivePath ); + console.error( + `āŒ Download attempt ${ attempt } failed: ${ downloadError.message }` + ); + + if ( + attempt === MAX_DOWNLOAD_ATTEMPTS || + ! isRetryableDownloadError( downloadError ) + ) { + throw downloadError; + } + + console.log( + ` Retrying in ${ RETRY_DELAY_MS / 1000 } seconds...` + ); + await delayFn( RETRY_DELAY_MS ); } + } + + throw new Error( 'Download failed without an error' ); +} + +/** + * Verify an archive against the independently resolved blob metadata. + * + * @param {string} archivePath Archive path. + * @param {{ sourceSha: string, blobSha256: string, blobSize: number }} metadata Blob metadata. + * @return {void} + */ +function verifyArchive( archivePath, metadata ) { + const normalized = normalizeMetadata( metadata ); + const stat = fs.statSync( archivePath ); + if ( stat.size !== normalized.blobSize ) { + throw new Error( + `Archive size mismatch: expected ${ formatBytes( + normalized.blobSize + ) } but received ${ formatBytes( stat.size ) }` + ); + } - /* - * Spawn tar to read from stdin and extract into gutenbergDir. - * `tar` is available on macOS, Linux, and Windows 10+. - */ - const tar = spawn( 'tar', [ '-x', '-C', gutenbergDir ], { - stdio: [ 'pipe', 'inherit', 'inherit' ], + const actualSha256 = crypto + .createHash( 'sha256' ) + .update( fs.readFileSync( archivePath ) ) + .digest( 'hex' ); + if ( actualSha256 !== normalized.blobSha256 ) { + throw new Error( + `Archive SHA-256 mismatch: expected ${ normalized.blobSha256 } but received ${ actualSha256 }` + ); + } +} + +/** + * Extract a verified archive into a staging directory, then replace the + * existing Gutenberg directory. + * + * @param {string} archivePath Path to the verified compressed blob. + * @param {string} expectedSha Expected immutable Gutenberg source SHA. + * @param {{ root?: string, destination?: string }} [paths] Paths used for extraction. + * @return {Promise} Resolves after extraction completes. + */ +async function extractVerifiedArchive( archivePath, expectedSha, paths = {} ) { + const extractionRoot = paths.root || rootDir; + const destination = paths.destination || gutenbergDir; + const stagingDir = path.join( + extractionRoot, + `.gutenberg-download-${ process.pid }` + ); + const backupDir = path.join( + extractionRoot, + `.gutenberg-backup-${ process.pid }` + ); + let preserveBackup = false; + fs.rmSync( stagingDir, { recursive: true, force: true } ); + fs.mkdirSync( stagingDir, { recursive: true } ); + + try { + const tar = spawn( 'tar', [ '-xzf', archivePath, '-C', stagingDir ], { + stdio: [ 'ignore', 'inherit', 'inherit' ], } ); - /** @type {Promise} */ - const tarDone = new Promise( ( resolve, reject ) => { + await new Promise( ( resolve, reject ) => { tar.on( 'close', ( code ) => { if ( code !== 0 ) { reject( new Error( `tar exited with code ${ code }` ) ); - } else { - resolve(); + return; } + resolve( undefined ); } ); tar.on( 'error', reject ); } ); - /* - * Pipe: fetch body → gunzip → tar stdin. - * Decompressing in Node keeps the pipeline error handling - * consistent and means tar only sees plain tar data on stdin. - */ - await pipeline( - Readable.fromWeb( - /** @type {import('stream/web').ReadableStream} */ ( response.body ) - ), - zlib.createGunzip(), - tar.stdin, + const extractedHashPath = path.join( stagingDir, '.gutenberg-hash' ); + const extractedSha = fs + .readFileSync( extractedHashPath, 'utf8' ) + .trim(); + if ( extractedSha !== expectedSha ) { + throw new Error( + `Extracted Gutenberg SHA mismatch: expected ${ expectedSha } but found ${ extractedSha }` + ); + } + + if ( fs.existsSync( destination ) ) { + fs.rmSync( backupDir, { recursive: true, force: true } ); + fs.renameSync( destination, backupDir ); + } + + try { + fs.renameSync( stagingDir, destination ); + } catch ( error ) { + if ( + fs.existsSync( backupDir ) && + ! fs.existsSync( destination ) + ) { + try { + fs.renameSync( backupDir, destination ); + } catch ( restoreError ) { + preserveBackup = true; + throw new Error( + `Could not replace Gutenberg directory: ${ + /** @type {Error} */ ( error ).message + }; ` + + `could not restore the backup at ${ backupDir }: ${ + /** @type {Error} */ ( restoreError ).message + }` + ); + } + } + + throw error; + } + + fs.rmSync( backupDir, { recursive: true, force: true } ); + } finally { + fs.rmSync( stagingDir, { recursive: true, force: true } ); + if ( ! preserveBackup ) { + fs.rmSync( backupDir, { recursive: true, force: true } ); + } + } +} + +/** + * Validate an existing archive or fetch one, then extract it. + * + * @param {{ archivePath: string, metadata: { sourceSha: string, blobSha256: string, blobSize: number }, fetchArchive?: () => Promise, extractionPaths?: { root?: string, destination?: string } }} options Archive preparation options. + * @return {Promise} Resolves after the verified archive is extracted. + */ +async function prepareArchive( options ) { + const metadata = normalizeMetadata( options.metadata ); + if ( ! fs.existsSync( options.archivePath ) ) { + if ( ! options.fetchArchive ) { + throw new Error( + `Archive does not exist: ${ options.archivePath }` + ); + } + await options.fetchArchive(); + } + + verifyArchive( options.archivePath, metadata ); + await extractVerifiedArchive( + options.archivePath, + metadata.sourceSha, + options.extractionPaths + ); +} + +/** + * Download and extract a build using a caller-provided archive path. + * + * @param {{ archivePath: string, metadata: { sourceSha: string, blobSha256: string, blobSize: number }, config?: { ref: string, ghcrRepo: string, isMutable: boolean }, fetchToken?: (ghcrRepo: string) => Promise, blobUrl?: string, extractionPaths?: { root?: string, destination?: string }, delayFn?: (milliseconds: number) => Promise }} options Download options. + * @return {Promise} Resolves after extraction completes. + */ +async function downloadAndExtract( options ) { + const metadata = normalizeMetadata( options.metadata ); + const config = options.config || readGutenbergConfig(); + await prepareArchive( { + archivePath: options.archivePath, + metadata, + extractionPaths: options.extractionPaths, + fetchArchive: async () => { + const token = await ( options.fetchToken || fetchGhcrToken )( + config.ghcrRepo + ); + const url = + options.blobUrl || + `https://ghcr.io/v2/${ config.ghcrRepo }/blobs/sha256:${ metadata.blobSha256 }`; + await downloadBlobWithRetries( + url, + token, + metadata, + options.archivePath, + options.delayFn + ); + }, + } ); +} + +/** + * Return the temporary archive path used by no-argument downloads. + * + * @return {string} Temporary archive path. + */ +function getTemporaryArchivePath() { + return path.join( + os.tmpdir(), + `wordpress-gutenberg-${ process.pid }.tar.gz` + ); +} + +/** + * Return the only archive path accepted by cache-mode commands. + * + * @param {string} archivePath Archive path supplied on the command line. + * @return {string} Stable archive path in the checkout. + */ +function getCacheArchivePath( archivePath ) { + const stableArchivePath = path.join( + rootDir, + '.ci', + 'gutenberg', + 'archive.tar.gz' + ); + const resolvedArchivePath = path.resolve( rootDir, archivePath ); + if ( resolvedArchivePath !== stableArchivePath ) { + throw new Error( + 'Archive mode requires the checkout path .ci/gutenberg/archive.tar.gz' ); + } - await tarDone; + return stableArchivePath; +} - console.log( 'āœ… Download and extraction complete' ); - } catch ( error ) { - console.error( 'āŒ Download/extraction failed:', /** @type {Error} */ ( error ).message ); - process.exit( 1 ); +/** + * Preserve the no-argument downloader behavior, including temporary archive + * cleanup after extraction or failure. + * + * @param {{ config?: { ref: string, ghcrRepo: string, isMutable: boolean }, fetchToken?: (ghcrRepo: string) => Promise, fetchManifestFn?: (ref: string, ghcrRepo: string, token: string) => Promise>, blobUrl?: string, archivePath?: string, extractionPaths?: { root?: string, destination?: string }, delayFn?: (milliseconds: number) => Promise }} [options] Download options. + * @return {Promise} Resolves after extraction completes. + */ +async function downloadDefault( options = {} ) { + const config = options.config || readGutenbergConfig(); + const token = await ( options.fetchToken || fetchGhcrToken )( + config.ghcrRepo + ); + const metadata = await resolveDownloadMetadata( + config, + token, + options.fetchManifestFn + ); + const archivePath = options.archivePath || getTemporaryArchivePath(); + + try { + await downloadAndExtract( { + archivePath, + metadata, + config, + fetchToken: async () => token, + blobUrl: options.blobUrl, + extractionPaths: options.extractionPaths, + delayFn: options.delayFn, + } ); + } finally { + fs.rmSync( archivePath, { force: true } ); + } +} + +/** + * Parse command-line arguments for the metadata and stable archive modes. + * + * @param {string[]} argv Command-line arguments. + * @return {{ metadata: boolean, archivePath?: string, sourceSha?: string, blobSha256?: string, blobSize?: string }} Parsed arguments. + */ +function parseArguments( argv ) { + const parsed = { metadata: false }; + for ( let index = 0; index < argv.length; index++ ) { + const argument = argv[ index ]; + if ( argument === '--metadata' ) { + parsed.metadata = true; + continue; + } + + const values = { + '--archive': 'archivePath', + '--source-sha': 'sourceSha', + '--blob-sha256': 'blobSha256', + '--blob-size': 'blobSize', + }; + const property = values[ argument ]; + if ( ! property || ! argv[ index + 1 ] ) { + throw new Error( `Unknown or incomplete argument: ${ argument }` ); + } + parsed[ property ] = argv[ ++index ]; } - console.log( '\nāœ… Gutenberg download complete!' ); + return parsed; } -// Run main function. -main().catch( ( error ) => { - console.error( 'āŒ Unexpected error:', error ); - process.exit( 1 ); -} ); +/** + * Main execution function. + * + * @param {string[]} [argv] Command-line arguments. + * @return {Promise} Resolves after the requested operation completes. + */ +async function main( argv = process.argv.slice( 2 ) ) { + const args = parseArguments( argv ); + if ( args.metadata ) { + if ( + args.archivePath || + args.sourceSha || + args.blobSha256 || + args.blobSize + ) { + throw new Error( + '--metadata cannot be combined with archive arguments' + ); + } + + const config = readGutenbergConfig(); + const token = await fetchGhcrToken( config.ghcrRepo ); + const metadata = await resolveDownloadMetadata( config, token ); + writeMetadataOutput( metadata, process.env.GITHUB_OUTPUT ); + return; + } + + if ( + ! args.archivePath && + ! args.sourceSha && + ! args.blobSha256 && + ! args.blobSize + ) { + await downloadDefault(); + return; + } + + if ( + ! args.archivePath || + ! args.sourceSha || + ! args.blobSha256 || + ! args.blobSize + ) { + throw new Error( + 'Archive mode requires --archive, --source-sha, --blob-sha256, and --blob-size' + ); + } + + await downloadAndExtract( { + archivePath: getCacheArchivePath( args.archivePath ), + metadata: { + sourceSha: args.sourceSha, + blobSha256: args.blobSha256, + blobSize: Number( args.blobSize ), + }, + } ); +} + +module.exports = { + downloadAndExtract, + downloadBlobWithRetries, + downloadDefault, + extractVerifiedArchive, + getCacheArchivePath, + getManifestMetadata, + getTemporaryArchivePath, + isRetryableDownloadError, + main, + normalizeMetadata, + parseArguments, + prepareArchive, + resolveDownloadManifest, + resolveDownloadMetadata, + removePartialArchives, + verifyArchive, + writeMetadataOutput, +}; + +if ( require.main === module ) { + main().catch( ( error ) => { + console.error( 'āŒ Gutenberg download failed:', error.message ); + process.exit( 1 ); + } ); +} diff --git a/tools/gutenberg/tests/download.test.js b/tools/gutenberg/tests/download.test.js new file mode 100644 index 0000000000000..a27b01cc268e3 --- /dev/null +++ b/tools/gutenberg/tests/download.test.js @@ -0,0 +1,565 @@ +#!/usr/bin/env node + +/** + * Tests for the Gutenberg archive downloader. + * + * @package + */ + +const { test, describe } = require( 'node:test' ); +const assert = require( 'node:assert/strict' ); +const crypto = require( 'node:crypto' ); +const fs = require( 'node:fs' ); +const http = require( 'node:http' ); +const os = require( 'node:os' ); +const path = require( 'node:path' ); +const { spawnSync } = require( 'node:child_process' ); +const { + downloadAndExtract, + downloadBlobWithRetries, + downloadDefault, + getManifestMetadata, + main, + resolveDownloadManifest, + writeMetadataOutput, +} = require( '../download.js' ); + +const SOURCE_SHA = 'a'.repeat( 40 ); +const IMMUTABLE_SHA = 'b'.repeat( 40 ); + +/** + * Create a disposable directory. + * + * @return {string} Fixture directory. + */ +function createFixture() { + return fs.mkdtempSync( + path.join( os.tmpdir(), 'gutenberg-download-test-' ) + ); +} + +/** + * Calculate a SHA-256 digest. + * + * @param {Buffer} value Content to hash. + * @return {string} SHA-256 hex digest. + */ +function sha256( value ) { + return crypto.createHash( 'sha256' ).update( value ).digest( 'hex' ); +} + +/** + * Create an archive whose extraction has the expected Gutenberg SHA. + * + * @param {string} fixture Fixture directory. + * @param {string} sourceSha Expected source SHA. + * @return {{ archivePath: string, body: Buffer, metadata: { sourceSha: string, blobSha256: string, blobSize: number } }} Archive fixture. + */ +function createArchive( fixture, sourceSha = SOURCE_SHA ) { + const sourceDir = path.join( fixture, 'source' ); + const archivePath = path.join( fixture, 'build.tar.gz' ); + fs.mkdirSync( sourceDir ); + fs.writeFileSync( + path.join( sourceDir, '.gutenberg-hash' ), + `${ sourceSha }\n` + ); + fs.writeFileSync( + path.join( sourceDir, 'build.txt' ), + 'Gutenberg build fixture\n' + ); + + const result = spawnSync( 'tar', [ + '-czf', + archivePath, + '-C', + sourceDir, + '.', + ] ); + assert.equal( result.status, 0, result.stderr.toString() ); + + const body = fs.readFileSync( archivePath ); + return { + archivePath, + body, + metadata: { + sourceSha, + blobSha256: sha256( body ), + blobSize: body.length, + }, + }; +} + +/** + * Start a local HTTP server with scripted responses. + * + * @param {Array<(res: import('http').ServerResponse) => void>} responses Response handlers. + * @return {Promise<{ url: string, requests: () => number, close: () => Promise }>} Server controls. + */ +function createServer( responses ) { + let count = 0; + const server = http.createServer( ( _request, response ) => { + const handler = responses[ count++ ]; + if ( ! handler ) { + response.writeHead( 599 ); + response.end(); + return; + } + handler( response ); + } ); + + return new Promise( ( resolve, reject ) => { + server.on( 'error', reject ); + server.listen( 0, '127.0.0.1', () => { + const address = /** @type {import('net').AddressInfo} */ ( + server.address() + ); + resolve( { + url: `http://127.0.0.1:${ address.port }/blob`, + requests: () => count, + close: () => new Promise( ( done ) => server.close( done ) ), + } ); + } ); + } ); +} + +/** + * Send an archive response. + * + * @param {Buffer} body Archive bytes. + * @return {(res: import('http').ServerResponse) => void} Response handler. + */ +function archiveResponse( body ) { + return ( response ) => { + response.writeHead( 200, { 'Content-Length': String( body.length ) } ); + response.end( body ); + }; +} + +describe( 'metadata resolution', { concurrency: false }, () => { + test( 'writes strict immutable metadata to the step output file', async () => { + const fixture = createFixture(); + const outputPath = path.join( fixture, 'github-output' ); + try { + const manifest = { + layers: [ + { digest: `sha256:${ 'C'.repeat( 64 ) }`, size: 123 }, + ], + }; + const metadata = getManifestMetadata( + manifest, + SOURCE_SHA.toUpperCase() + ); + writeMetadataOutput( metadata, outputPath ); + + assert.equal( + fs.readFileSync( outputPath, 'utf8' ), + `source-sha=${ SOURCE_SHA }\nblob-sha256=${ 'c'.repeat( + 64 + ) }\nblob-size=123\n` + ); + } finally { + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); + + test( 'uses immutable refs directly and resolves mutable refs again by revision', async () => { + const calls = []; + const fetchManifest = async ( ref ) => { + calls.push( ref ); + if ( ref === 'trunk' ) { + return { + annotations: { + 'org.opencontainers.image.revision': + IMMUTABLE_SHA.toUpperCase(), + }, + }; + } + return { layers: [] }; + }; + + const immutable = await resolveDownloadManifest( + { ref: SOURCE_SHA, ghcrRepo: 'owner/package', isMutable: false }, + 'token', + fetchManifest + ); + assert.equal( immutable.sourceSha, SOURCE_SHA ); + assert.deepEqual( calls, [ SOURCE_SHA ] ); + + calls.length = 0; + const mutable = await resolveDownloadManifest( + { ref: 'trunk', ghcrRepo: 'owner/package', isMutable: true }, + 'token', + fetchManifest + ); + assert.equal( mutable.sourceSha, IMMUTABLE_SHA ); + assert.deepEqual( calls, [ 'trunk', IMMUTABLE_SHA ] ); + } ); + + test( 'rejects a mutable ref without a valid immutable manifest', async () => { + await assert.rejects( + () => + resolveDownloadManifest( + { + ref: 'trunk', + ghcrRepo: 'owner/package', + isMutable: true, + }, + 'token', + async ( ref ) => { + if ( ref === 'trunk' ) { + return { + annotations: { + 'org.opencontainers.image.revision': + IMMUTABLE_SHA, + }, + }; + } + const error = new Error( + 'immutable manifest unavailable' + ); + error.status = 404; + throw error; + } + ), + /immutable manifest unavailable/ + ); + } ); +} ); + +describe( 'archive verification', { concurrency: false }, () => { + test( 'rejects a poisoned supplied archive before extraction', async () => { + const fixture = createFixture(); + const archive = createArchive( fixture ); + const destination = path.join( fixture, 'gutenberg' ); + const markerPath = path.join( destination, 'marker' ); + try { + fs.mkdirSync( destination ); + fs.writeFileSync( markerPath, 'existing Gutenberg directory' ); + await assert.rejects( + () => + downloadAndExtract( { + archivePath: archive.archivePath, + metadata: { + ...archive.metadata, + blobSha256: '0'.repeat( 64 ), + }, + config: { + ref: SOURCE_SHA, + ghcrRepo: 'owner/package', + isMutable: false, + }, + extractionPaths: { root: fixture, destination }, + } ), + /Archive SHA-256 mismatch/ + ); + assert.equal( + fs.readFileSync( markerPath, 'utf8' ), + 'existing Gutenberg directory' + ); + + await assert.rejects( + () => + downloadAndExtract( { + archivePath: archive.archivePath, + metadata: { + ...archive.metadata, + blobSize: archive.metadata.blobSize + 1, + }, + config: { + ref: SOURCE_SHA, + ghcrRepo: 'owner/package', + isMutable: false, + }, + extractionPaths: { root: fixture, destination }, + } ), + /Archive size mismatch/ + ); + assert.equal( + fs.readFileSync( markerPath, 'utf8' ), + 'existing Gutenberg directory' + ); + } finally { + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); +} ); + +describe( 'archive path validation', { concurrency: false }, () => { + test( 'rejects an archive path outside the checkout', async () => { + await assert.rejects( + () => + main( [ + '--archive', + '../archive.tar.gz', + '--source-sha', + SOURCE_SHA, + '--blob-sha256', + '0'.repeat( 64 ), + '--blob-size', + '1', + ] ), + /Archive mode requires the checkout path/ + ); + } ); +} ); + +describe( 'archive preparation', { concurrency: false }, () => { + test( 'fetches a cold archive at the stable path, verifies it, and extracts it', async () => { + const fixture = createFixture(); + const archive = createArchive( fixture ); + const stablePath = path.join( + fixture, + '.ci', + 'gutenberg', + 'archive.tar.gz' + ); + const destination = path.join( fixture, 'gutenberg' ); + const server = await createServer( [ + archiveResponse( archive.body ), + ] ); + try { + fs.mkdirSync( path.dirname( stablePath ), { recursive: true } ); + fs.writeFileSync( + `${ stablePath }.partial-interrupted`, + 'stale partial archive' + ); + await downloadAndExtract( { + archivePath: stablePath, + metadata: archive.metadata, + config: { + ref: SOURCE_SHA, + ghcrRepo: 'owner/package', + isMutable: false, + }, + fetchToken: async () => 'token', + blobUrl: server.url, + extractionPaths: { root: fixture, destination }, + delayFn: async () => {}, + } ); + + assert.equal( server.requests(), 1 ); + assert.equal( fs.existsSync( stablePath ), true ); + assert.equal( + fs.existsSync( `${ stablePath }.partial-interrupted` ), + false + ); + assert.equal( + fs + .readFileSync( + path.join( destination, '.gutenberg-hash' ), + 'utf8' + ) + .trim(), + SOURCE_SHA + ); + } finally { + await server.close(); + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); + + test( 'validates and extracts a supplied archive without requesting a token', async () => { + const fixture = createFixture(); + const archive = createArchive( fixture ); + const destination = path.join( fixture, 'gutenberg' ); + try { + await downloadAndExtract( { + archivePath: archive.archivePath, + metadata: archive.metadata, + config: { + ref: SOURCE_SHA, + ghcrRepo: 'owner/package', + isMutable: false, + }, + fetchToken: async () => { + throw new Error( 'a supplied archive must not fetch' ); + }, + extractionPaths: { root: fixture, destination }, + } ); + + assert.equal( fs.existsSync( archive.archivePath ), true ); + assert.equal( + fs + .readFileSync( + path.join( destination, '.gutenberg-hash' ), + 'utf8' + ) + .trim(), + SOURCE_SHA + ); + } finally { + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); +} ); + +describe( 'download retries and cleanup', { concurrency: false }, () => { + test( 'retries a transient failure and leaves no partial archive', async () => { + const fixture = createFixture(); + const archive = createArchive( fixture ); + const stablePath = path.join( + fixture, + '.ci', + 'gutenberg', + 'archive.tar.gz' + ); + const server = await createServer( [ + ( response ) => { + response.writeHead( 503 ); + response.end(); + }, + archiveResponse( archive.body ), + ] ); + try { + await downloadBlobWithRetries( + server.url, + 'token', + archive.metadata, + stablePath, + async () => {} + ); + + assert.equal( server.requests(), 2 ); + assert.equal( fs.existsSync( stablePath ), true ); + assert.deepEqual( + fs + .readdirSync( path.dirname( stablePath ) ) + .filter( ( file ) => file.includes( '.partial-' ) ), + [] + ); + } finally { + await server.close(); + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); + + test( 'removes partial and stable paths after retry exhaustion', async () => { + const fixture = createFixture(); + const stablePath = path.join( + fixture, + '.ci', + 'gutenberg', + 'archive.tar.gz' + ); + const server = await createServer( + Array.from( { length: 3 }, () => ( response ) => { + response.writeHead( 503 ); + response.end(); + } ) + ); + try { + await assert.rejects( + () => + downloadBlobWithRetries( + server.url, + 'token', + { + sourceSha: SOURCE_SHA, + blobSha256: '0'.repeat( 64 ), + blobSize: 1, + }, + stablePath, + async () => {} + ), + /503/ + ); + assert.equal( server.requests(), 3 ); + assert.equal( fs.existsSync( stablePath ), false ); + assert.deepEqual( + fs + .readdirSync( path.dirname( stablePath ) ) + .filter( ( file ) => file.includes( '.partial-' ) ), + [] + ); + } finally { + await server.close(); + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); + + test( 'removes partial and stable paths after digest mismatches', async () => { + const fixture = createFixture(); + const archive = createArchive( fixture ); + const stablePath = path.join( + fixture, + '.ci', + 'gutenberg', + 'archive.tar.gz' + ); + const server = await createServer( [ + archiveResponse( archive.body ), + archiveResponse( archive.body ), + archiveResponse( archive.body ), + ] ); + try { + await assert.rejects( + () => + downloadBlobWithRetries( + server.url, + 'token', + { + ...archive.metadata, + blobSha256: '0'.repeat( 64 ), + }, + stablePath, + async () => {} + ), + /SHA-256 mismatch/ + ); + assert.equal( server.requests(), 3 ); + assert.equal( fs.existsSync( stablePath ), false ); + assert.deepEqual( + fs + .readdirSync( path.dirname( stablePath ) ) + .filter( ( file ) => file.includes( '.partial-' ) ), + [] + ); + } finally { + await server.close(); + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); +} ); + +describe( 'default downloader', { concurrency: false }, () => { + test( 'removes its temporary archive after a successful extraction', async () => { + const fixture = createFixture(); + const archive = createArchive( fixture ); + const temporaryPath = path.join( fixture, 'temporary.tar.gz' ); + const destination = path.join( fixture, 'gutenberg' ); + const server = await createServer( [ + archiveResponse( archive.body ), + ] ); + try { + await downloadDefault( { + config: { + ref: SOURCE_SHA, + ghcrRepo: 'owner/package', + isMutable: false, + }, + fetchToken: async () => 'token', + fetchManifestFn: async () => ( { + layers: [ + { + digest: `sha256:${ archive.metadata.blobSha256 }`, + size: archive.metadata.blobSize, + }, + ], + } ), + blobUrl: server.url, + archivePath: temporaryPath, + extractionPaths: { root: fixture, destination }, + delayFn: async () => {}, + } ); + + assert.equal( fs.existsSync( temporaryPath ), false ); + assert.equal( + fs.existsSync( path.join( destination, '.gutenberg-hash' ) ), + true + ); + } finally { + await server.close(); + fs.rmSync( fixture, { recursive: true, force: true } ); + } + } ); +} ); diff --git a/tools/gutenberg/utils.js b/tools/gutenberg/utils.js index 3ba95199578b4..6820a888bfe2c 100644 --- a/tools/gutenberg/utils.js +++ b/tools/gutenberg/utils.js @@ -121,6 +121,17 @@ async function fetchManifest( ref, ghcrRepo, token ) { * @return {Promise} The expected SHA. */ async function resolveExpectedSha( { ref, ghcrRepo, isMutable } ) { + const workflowSha = process.env.GUTENBERG_EXPECTED_SHA; + if ( workflowSha ) { + if ( ! SHA_PATTERN.test( workflowSha ) ) { + throw new Error( + `GUTENBERG_EXPECTED_SHA must be a 40-character Git SHA, received "${ workflowSha }"` + ); + } + + return workflowSha; + } + if ( ! isMutable ) { return ref; } @@ -157,11 +168,11 @@ function downloadGutenberg() { /** * Verify that the installed Gutenberg version matches the expected SHA. * - * For SHA refs, the expected SHA is the configured value. For mutable refs, - * the expected SHA is whatever the mutable tag currently points to in GHCR - * (read from the manifest's image.revision annotation). The installed - * `.gutenberg-hash` is compared against the expected SHA; on mismatch, a - * fresh download is triggered. + * A calling workflow may supply GUTENBERG_EXPECTED_SHA after resolving a build + * once. This avoids re-resolving a mutable tag in every matrix job. Otherwise, + * SHA refs use the configured value and mutable refs resolve their current + * image.revision annotation. The installed `.gutenberg-hash` is compared + * against the expected SHA; on mismatch, a fresh download is triggered. */ async function verifyGutenbergVersion() { console.log( '\nšŸ” Verifying Gutenberg version...' );