diff --git a/.github/workflows/cloudflare-verification.yml b/.github/workflows/cloudflare-verification.yml new file mode 100644 index 0000000..09d94de --- /dev/null +++ b/.github/workflows/cloudflare-verification.yml @@ -0,0 +1,194 @@ +name: Reusable Cloudflare verification + +on: + workflow_call: + inputs: + factory_ref: + description: Factory commit to verify + required: false + type: string + default: main + infra_repository: + description: Dedicated private verification-infra repository + required: false + type: string + default: AgentWorkforce/factory-test-infra + infra_ref: + description: Verification-infra commit to use + required: false + type: string + default: main + run_real: + description: Provision a real dispatch namespace and Container + required: false + type: boolean + default: true + secrets: + infra_repository_token: + description: Read token for the private factory-test-infra repository + required: true + cloudflare_resource_json: + description: JSON Resource object containing accountId and apiToken + required: false + +permissions: + contents: read + +concurrency: + group: cloudflare-verification-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + miniflare: + name: Miniflare Track-C suite + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout Factory + uses: actions/checkout@v6 + with: + ref: ${{ inputs.factory_ref }} + path: factory + persist-credentials: false + + - name: Checkout factory-test-infra + uses: actions/checkout@v6 + with: + repository: ${{ inputs.infra_repository }} + ref: ${{ inputs.infra_ref }} + token: ${{ secrets.infra_repository_token }} + path: factory-test-infra + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + cache-dependency-path: | + factory/package-lock.json + factory-test-infra/package-lock.json + + - name: Install Factory + working-directory: factory + run: npm ci + + - name: Install verification infrastructure + working-directory: factory-test-infra + run: npm ci + + - name: Start isolated Miniflare environment + working-directory: factory-test-infra + env: + FACTORY_REPO_PATH: ${{ github.workspace }}/factory + FACTORY_VERIFICATION_MODE: miniflare + run: bash ci/verification-env/up.sh + + - name: Run Track-C suites and guardrail assertions + working-directory: factory-test-infra + env: + FACTORY_REPO_PATH: ${{ github.workspace }}/factory + FACTORY_VERIFICATION_MODE: miniflare + run: bash ci/verification-env/run-suite.sh + + - name: Tear down Miniflare environment + if: always() + working-directory: factory-test-infra + env: + FACTORY_REPO_PATH: ${{ github.workspace }}/factory + FACTORY_VERIFICATION_MODE: miniflare + run: bash ci/verification-env/down.sh + + - name: Upload Miniflare evidence + if: always() + uses: actions/upload-artifact@v6 + with: + name: factory-cloudflare-miniflare-${{ github.sha }} + path: factory-test-infra/artifacts/verification-env/ + if-no-files-found: error + + cloudflare: + name: Real dispatch namespace and Container + if: inputs.run_real + needs: miniflare + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Require real Cloudflare resource secret + env: + FACTORY_TEST_INFRA_RESOURCE_JSON: ${{ secrets.cloudflare_resource_json }} + run: test -n "$FACTORY_TEST_INFRA_RESOURCE_JSON" + + - name: Checkout Factory + uses: actions/checkout@v6 + with: + ref: ${{ inputs.factory_ref }} + path: factory + persist-credentials: false + + - name: Checkout factory-test-infra + uses: actions/checkout@v6 + with: + repository: ${{ inputs.infra_repository }} + ref: ${{ inputs.infra_ref }} + token: ${{ secrets.infra_repository_token }} + path: factory-test-infra + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + cache-dependency-path: | + factory/package-lock.json + factory-test-infra/package-lock.json + + - name: Install dependencies + run: | + npm --prefix factory ci + npm --prefix factory-test-infra ci + + - name: Materialize linked Cloudflare resource + env: + FACTORY_TEST_INFRA_RESOURCE_JSON: ${{ secrets.cloudflare_resource_json }} + run: | + install -m 600 /dev/null "$RUNNER_TEMP/factory-test-infra-resource.json" + printf '%s' "$FACTORY_TEST_INFRA_RESOURCE_JSON" > "$RUNNER_TEMP/factory-test-infra-resource.json" + + - name: Provision real verification environment + working-directory: factory-test-infra + env: + FACTORY_REPO_PATH: ${{ github.workspace }}/factory + FACTORY_TEST_INFRA_RESOURCE_FILE: ${{ runner.temp }}/factory-test-infra-resource.json + FACTORY_VERIFICATION_MODE: cloudflare + run: bash ci/verification-env/up.sh + + - name: Run Track-C suites and guardrail assertions + working-directory: factory-test-infra + env: + FACTORY_REPO_PATH: ${{ github.workspace }}/factory + FACTORY_TEST_INFRA_RESOURCE_FILE: ${{ runner.temp }}/factory-test-infra-resource.json + FACTORY_VERIFICATION_MODE: cloudflare + run: bash ci/verification-env/run-suite.sh + + - name: Tear down real verification environment + if: always() + working-directory: factory-test-infra + env: + FACTORY_REPO_PATH: ${{ github.workspace }}/factory + FACTORY_TEST_INFRA_RESOURCE_FILE: ${{ runner.temp }}/factory-test-infra-resource.json + FACTORY_VERIFICATION_MODE: cloudflare + run: bash ci/verification-env/down.sh + + - name: Remove linked Cloudflare resource + if: always() + run: rm -f "$RUNNER_TEMP/factory-test-infra-resource.json" + + - name: Upload real-environment evidence + if: always() + uses: actions/upload-artifact@v6 + with: + name: factory-cloudflare-real-${{ github.sha }} + path: factory-test-infra/artifacts/verification-env/ + if-no-files-found: error diff --git a/docs/cloudflare-environment-provider.md b/docs/cloudflare-environment-provider.md new file mode 100644 index 0000000..6792b0d --- /dev/null +++ b/docs/cloudflare-environment-provider.md @@ -0,0 +1,123 @@ +# Cloudflare verification environments + +`CloudflareEnvironmentProvider` is Factory's Cloudflare-first implementation of +the substrate-neutral `EnvironmentProvider` port. It deliberately owns only the +Factory-side lifecycle. The dedicated private `AgentWorkforce/factory-test-infra` +repository owns account resources, the dispatch Worker, Container definitions, +usage monitoring, the scheduled reaper, and real-environment CI scripts. + +## Credential boundary + +Cloudflare credentials must not be placed in `factory.config.json` or read from +`process.env`. Configuration contains only a logical resource name and +source-control-safe policy: + +```json +{ + "environments": { + "cloudflare": { + "resource": "FactoryTestInfra", + "dispatchNamespacePrefix": "factory-verification", + "maxConcurrentEnvironments": 2, + "defaultTtlMs": 900000, + "maxTtlMs": 3600000, + "maxRunCostUsd": 1, + "workerLimits": { "cpuMs": 100, "subrequests": 100 }, + "container": { "instanceType": "lite", "maxInstances": 3 } + } + } +} +``` + +At runtime, pass the linked SST `Resource.*`-style object explicitly: + +```ts +import { CloudflareEnvironmentProvider } from '@agent-relay/factory/environments' + +const provider = CloudflareEnvironmentProvider.fromResource({ + config: factoryConfig.environments.cloudflare, + resources: { + FactoryTestInfra: Resource.FactoryTestInfra, + }, +}) +``` + +The linked resource has this shape: + +```ts +{ + accountId: string + apiToken: string + dispatcherUrlTemplate?: 'https://.../{namespace}/{script}' +} +``` + +The API token needs the narrow Workers Scripts read/write permissions used by +the Workers for Platforms dispatch namespace API. Never log, persist, return in +an `Environment`, or copy that object into Factory configuration. + +## Lifecycle and isolation + +Each provision call: + +1. Validates TTL, requested cost reservation, Container instance count, and the + max-concurrent namespace cap before creating anything. +2. Creates a new dispatch namespace. A namespace reported as `trusted` is + rejected and immediately removed. +3. Uploads an inert `factory-environment` Worker containing ownership, creation, + expiration, and guardrail bindings. User workloads never receive the account + token or bindings belonging to another namespace. +4. Returns the namespace identity and non-secret limits through `Environment`. + +Destroy and reaper paths read the metadata Worker's binding lease and require +its `FACTORY_ENVIRONMENT_ID` to equal the namespace name before deletion. A +missing, incomplete, expired in the wrong direction, or mismatched lease fails +closed. The provider reaper handles TTL and optional owner-liveness checks; the +separate infra Cron Trigger performs the same check independently. + +Cloudflare documents that Workers for Platforms user Workers are untrusted by +default, which isolates their cache and request metadata. Binding isolation is +still a deployment responsibility: the infra renderer must create or select +per-environment D1/KV/R2/Queue/Hyperdrive resources and attach only that +environment's identifiers to scripts in its dispatch namespace. + +## Guardrail contract + +The sentinel Worker exposes these non-secret bindings for the infra control +plane and scheduled reaper: + +- `FACTORY_ENVIRONMENT_ID`, `FACTORY_OWNER_ID`, `FACTORY_CUSTOMER_ID` +- `FACTORY_REPOSITORY`, `FACTORY_CREATED_AT`, `FACTORY_EXPIRES_AT` +- `FACTORY_MAX_RUN_COST_USD`, `FACTORY_CONTAINER_MAX_INSTANCES` +- `FACTORY_WORKER_CPU_MS`, `FACTORY_WORKER_SUBREQUESTS` + +Worker CPU and subrequest limits are also attached to the metadata Worker at +upload. Cloudflare Container deployments must copy `maxInstances` to Wrangler's +production-enforced `containers[].max_instances`. The infra usage monitor must +abort and tear down a run when elapsed time or metered cost reaches its binding; +the Factory provider's TTL reaper and the Cron Trigger remain independent +backstops. + +## Verification workflow + +`.github/workflows/cloudflare-verification.yml` is reusable through +`workflow_call`. It checks out Factory and the private infra repository, runs the +infra repository's exact `ci/verification-env/up.sh`, `run-suite.sh`, and +`down.sh` contract against Miniflare and a real Cloudflare account, and always +runs teardown. The real job consumes a single JSON resource secret through a +temporary mode-0600 file and uploads guardrail evidence even when an assertion +fails. + +The real job is intentionally not part of ordinary fork PR execution. A trusted +caller supplies the private-repository checkout token and Cloudflare resource +secret. The scripts must fail loudly for isolation, quota/cost refusal, +concurrency refusal, orphan reaping, and k6 load absorption; a successful HTTP +smoke test alone is not an acceptable result. + +Primary Cloudflare references: + +- +- +- +- +- diff --git a/factory-test-infra/README.md b/factory-test-infra/README.md index 97ef804..115e9c3 100644 --- a/factory-test-infra/README.md +++ b/factory-test-infra/README.md @@ -1,13 +1,36 @@ -# Factory Kubernetes test infrastructure +# factory-test-infra handoff -`kubernetes-connections.example.yaml` is the source-control-safe shape for the -customer-to-cluster registry. Real credentials belong in the configured secret -manager. Only their references belong in Factory config. +This directory is a source-control-safe handoff note, not the environment data +plane. `@agent-relay/factory` is a library and must not own Cloudflare account +resources or operate verification workloads. -The provider creates the namespace RBAC, quota, limits, Pod Security labels, and -NetworkPolicies for every run; there is no mutable shared base chart to drift. -Managed EKS should provide a dedicated, tainted verification node group matching -the example selector/toleration and must run a NetworkPolicy-enforcing CNI. +The private `AgentWorkforce/factory-test-infra` repository is the data-plane +home. Its companion implementation for this change owns: -The shared managed connection is a fallback, not an EKS-fidelity claim. Its -mandatory caveat is surfaced on every resulting `Environment`. +```text +infra/ Terraform/SST account resources +containers/ bounded Container service definitions +workers/dispatcher/ dynamic Workers for Platforms dispatch Worker +workers/reaper/ Cron Trigger and leak alerting +templates/ Worker/Container + D1/KV/R2/Queue/Hyperdrive inputs +ci/verification-env/up.sh Miniflare or real environment provision +ci/verification-env/run-suite.sh +ci/verification-env/down.sh identity-checked teardown +.github/workflows/verification.yml +``` + +Its linked runtime resource must expose `accountId`, `apiToken`, and optionally +`dispatcherUrlTemplate`; see `docs/cloudflare-environment-provider.md`. Factory +configuration stores only the resource name and guardrail policy. + +The CI scripts are a strict contract. `run-suite.sh` must make isolation, +Container quota, elapsed-time/cost budget, max-concurrency, orphan reaping, and +k6 load assertions fail loudly if a guardrail is removed. `down.sh` must be +idempotent and verify the namespace's sentinel Worker identity before deletion. +`.github/workflows/cloudflare-verification.yml` in this repository and the +dedicated repository's `verification.yml` invoke that contract against both +Miniflare and a real Cloudflare account. + +`kubernetes-connections.example.yaml` remains the optional escape-hatch provider +configuration for stacks Cloudflare Containers cannot faithfully run. It is not +the default verification substrate and contains no credentials. diff --git a/src/__tests__/dist-entrypoints.test.ts b/src/__tests__/dist-entrypoints.test.ts index 7395c6a..bda5947 100644 --- a/src/__tests__/dist-entrypoints.test.ts +++ b/src/__tests__/dist-entrypoints.test.ts @@ -23,9 +23,11 @@ describe('published dist entrypoints', () => { expect(main.generateFeatureMap).toBeTypeOf('function') expect(main.VerificationStackDescriptorSchema).toBeDefined() expect(main.VerificationStackDeployer).toBeTypeOf('function') + expect(main.CloudflareEnvironmentProvider).toBeTypeOf('function') expect(main.KubernetesEnvironmentProvider).toBeTypeOf('function') expect(hosted.createHostedFactory).toBeTypeOf('function') expect(hosted.DurableObjectHostedFactoryStateStore).toBeTypeOf('function') + expect(environments.CloudflareEnvironmentProvider).toBeTypeOf('function') expect(environments.KubernetesEnvironmentProvider).toBeTypeOf('function') expect(environments.KubectlEnvironmentProvider).toBeTypeOf('function') expect(environments.VerificationPipeline).toBeTypeOf('function') diff --git a/src/config/schema.test.ts b/src/config/schema.test.ts index d99450a..2d0bd60 100644 --- a/src/config/schema.test.ts +++ b/src/config/schema.test.ts @@ -129,6 +129,50 @@ describe('FactoryConfigSchema', () => { })).toThrow(/never inline kubeconfig/iu) }) + it('accepts source-control-safe Cloudflare policy with only a Resource-style credential reference', () => { + const parsed = FactoryConfigSchema.parse({ + repos: { default: 'AgentWorkforce/factory' }, + environments: { + cloudflare: { + resource: 'FactoryVerificationInfra', + dispatchNamespacePrefix: 'factory-ci', + maxConcurrentEnvironments: 4, + defaultTtlMs: 10 * 60_000, + maxTtlMs: 30 * 60_000, + maxRunCostUsd: 0.5, + workerLimits: { cpuMs: 50, subrequests: 75 }, + container: { instanceType: 'basic', maxInstances: 2 }, + }, + }, + }) + + expect(parsed.environments.cloudflare).toEqual({ + resource: 'FactoryVerificationInfra', + dispatchNamespacePrefix: 'factory-ci', + maxConcurrentEnvironments: 4, + defaultTtlMs: 10 * 60_000, + maxTtlMs: 30 * 60_000, + maxRunCostUsd: 0.5, + workerLimits: { cpuMs: 50, subrequests: 75 }, + container: { instanceType: 'basic', maxInstances: 2 }, + }) + expect(() => FactoryConfigSchema.parse({ + repos: {}, + environments: { + cloudflare: { + accountId: 'must-not-live-in-config', + apiToken: 'must-not-live-in-config', + }, + }, + })).toThrow() + expect(() => FactoryConfigSchema.parse({ + repos: {}, + environments: { + cloudflare: { defaultTtlMs: 120_000, maxTtlMs: 60_000 }, + }, + })).toThrow('must be less than or equal to maxTtlMs') + }) + it('preserves explicit model overrides', () => { const parsed = FactoryConfigSchema.parse({ workspaceId: 'ws_123', diff --git a/src/config/schema.ts b/src/config/schema.ts index 81c2dbf..ee5d501 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -3,6 +3,7 @@ import { join } from 'node:path' import { z } from 'zod' +import { CloudflareEnvironmentConfigSchema } from '../environments/cloudflare-provider.js' import { KubernetesEnvironmentConfigSchema } from '../environments/connection-registry.js' // The five workflow-state roles the factory drives an issue through. Each is @@ -225,6 +226,7 @@ const safetySchema = z.object({ }).default({}) const environmentsSchema = z.object({ + cloudflare: CloudflareEnvironmentConfigSchema.optional(), kubernetes: KubernetesEnvironmentConfigSchema.optional(), }).strict().default({}) diff --git a/src/environments/cloudflare-provider.test.ts b/src/environments/cloudflare-provider.test.ts new file mode 100644 index 0000000..a8ccf62 --- /dev/null +++ b/src/environments/cloudflare-provider.test.ts @@ -0,0 +1,405 @@ +import { describe, expect, it, vi } from 'vitest' + +import { + CLOUDFLARE_ENVIRONMENT_BINDINGS, + CLOUDFLARE_METADATA_SCRIPT, + CloudflareApiError, + CloudflareEnvironmentProvider, + CloudflareEnvironmentQuotaError, + CloudflareEnvironmentResourceSchema, + HttpCloudflareEnvironmentClient, + cloudflareEnvironmentName, + type CloudflareDispatchNamespace, + type CloudflareEnvironmentClient, + type CloudflareWorkerBinding, + type UploadCloudflareWorkerInput, +} from './cloudflare-provider' + +const resource = CloudflareEnvironmentResourceSchema.parse({ + accountId: '0123456789abcdef0123456789abcdef', + apiToken: 'resource-secret-token', + dispatcherUrlTemplate: 'https://verification.example.test/{namespace}/{script}', +}) + +class FakeCloudflareEnvironmentClient implements CloudflareEnvironmentClient { + readonly namespaces = new Map() + readonly bindings = new Map() + readonly uploads: UploadCloudflareWorkerInput[] = [] + readonly deletes: string[] = [] + failUpload = false + createTrustedNamespace = false + + async listDispatchNamespaces(): Promise { + return [...this.namespaces.values()].map((namespace) => ({ ...namespace })) + } + + async getDispatchNamespace(name: string): Promise { + const namespace = this.namespaces.get(name) + return namespace ? { ...namespace } : undefined + } + + async createDispatchNamespace(name: string): Promise { + if (this.namespaces.has(name)) throw new Error('AlreadyExists') + const namespace = { + namespace_name: name, + namespace_id: `namespace-${name}`, + trusted_workers: this.createTrustedNamespace, + } + this.namespaces.set(name, namespace) + return { ...namespace } + } + + async deleteDispatchNamespace(name: string): Promise { + this.deletes.push(name) + this.namespaces.delete(name) + for (const key of [...this.bindings.keys()]) { + if (key.startsWith(`${name}/`)) this.bindings.delete(key) + } + } + + async uploadDispatchWorker(input: UploadCloudflareWorkerInput): Promise { + if (this.failUpload) throw new Error('upload failed') + this.uploads.push(structuredClone(input)) + this.bindings.set(`${input.namespace}/${input.scriptName}`, structuredClone(input.bindings)) + } + + async getDispatchWorkerBindings( + namespace: string, + scriptName: string, + ): Promise { + return structuredClone(this.bindings.get(`${namespace}/${scriptName}`)) + } +} + +describe('CloudflareEnvironmentProvider', () => { + it('resolves credentials only from the configured Resource-style object', () => { + const provider = CloudflareEnvironmentProvider.fromResource({ + config: { resource: 'VerificationInfra' }, + resources: { VerificationInfra: resource }, + client: new FakeCloudflareEnvironmentClient(), + }) + + expect(provider.config.resource).toBe('VerificationInfra') + expect(() => CloudflareEnvironmentProvider.fromResource({ + config: { resource: 'MissingInfra' }, + resources: {}, + })).toThrow('resource "MissingInfra" is unavailable') + expect(() => CloudflareEnvironmentResourceSchema.parse({ + accountId: resource.accountId, + apiToken: '', + })).toThrow() + }) + + it('creates one untrusted namespace with a durable identity lease and guardrail metadata', async () => { + const client = new FakeCloudflareEnvironmentClient() + const provider = new CloudflareEnvironmentProvider({ + resource, + client, + config: { + maxRunCostUsd: 0.5, + workerLimits: { cpuMs: 75, subrequests: 80 }, + container: { instanceType: 'basic', maxInstances: 3 }, + }, + now: () => new Date('2026-07-21T10:00:00.000Z'), + randomId: () => 'abc123', + }) + + const environment = await provider.provision({ + customerId: 'customer-a', + repository: 'AgentWorkforce/factory', + ownerId: 'run/146', + ttl: 120_000, + stack: { containerInstances: 2, runCostBudgetUsd: 0.25 }, + }) + + expect(environment).toMatchObject({ + id: 'factory-verification-factory-abc123', + provider: 'cloudflare', + dispatchNamespace: 'factory-verification-factory-abc123', + status: 'ready', + createdAt: '2026-07-21T10:00:00.000Z', + ttl: 120_000, + endpoints: {}, + bindings: { + 'cloudflare.resource': 'FactoryTestInfra', + 'cloudflare.metadataScript': CLOUDFLARE_METADATA_SCRIPT, + 'cloudflare.expiresAt': '2026-07-21T10:02:00.000Z', + 'cloudflare.maxRunCostUsd': '0.25', + 'cloudflare.container.instanceType': 'basic', + 'cloudflare.container.maxInstances': '2', + 'cloudflare.worker.cpuMs': '75', + 'cloudflare.worker.subrequests': '80', + }, + }) + expect(client.namespaces.get(environment.id)?.trusted_workers).toBe(false) + expect(client.uploads).toHaveLength(1) + expect(client.uploads[0]).toMatchObject({ + namespace: environment.id, + scriptName: CLOUDFLARE_METADATA_SCRIPT, + compatibilityDate: '2026-07-21', + limits: { cpu_ms: 75, subrequests: 80 }, + }) + const metadata = plainTextBindings(client.uploads[0].bindings) + expect(metadata).toMatchObject({ + [CLOUDFLARE_ENVIRONMENT_BINDINGS.environmentId]: environment.id, + [CLOUDFLARE_ENVIRONMENT_BINDINGS.ownerId]: 'run/146', + [CLOUDFLARE_ENVIRONMENT_BINDINGS.customerId]: 'customer-a', + [CLOUDFLARE_ENVIRONMENT_BINDINGS.repository]: 'AgentWorkforce/factory', + [CLOUDFLARE_ENVIRONMENT_BINDINGS.expiresAt]: '2026-07-21T10:02:00.000Z', + [CLOUDFLARE_ENVIRONMENT_BINDINGS.maxRunCostUsd]: '0.25', + [CLOUDFLARE_ENVIRONMENT_BINDINGS.containerMaxInstances]: '2', + }) + + await provider.destroy(environment.id) + await provider.destroy(environment.id) + expect(client.deletes).toEqual([environment.id]) + expect(await provider.status(environment.id)).toBe('destroyed') + }) + + it.each([ + { + label: 'Container instances', + config: { container: { maxInstances: 2 } }, + spec: { stack: { containerInstances: 3 } }, + message: /exceeds per-environment cap/iu, + }, + { + label: 'run cost', + config: { maxRunCostUsd: 0.1 }, + spec: { stack: { runCostBudgetUsd: 0.11 } }, + message: /exceeds per-run cap/iu, + }, + { + label: 'ttl', + config: { maxTtlMs: 60_000, defaultTtlMs: 60_000 }, + spec: { ttl: 60_001 }, + message: /ttl must be between/iu, + }, + ])('refuses a $label request before creating Cloudflare resources', async ({ config, spec, message }) => { + const client = new FakeCloudflareEnvironmentClient() + const provider = new CloudflareEnvironmentProvider({ resource, client, config }) + + await expect(provider.provision({ + customerId: 'customer', repository: 'org/repo', ownerId: 'run', ...spec, + })).rejects.toThrow(message) + expect(client.namespaces.size).toBe(0) + expect(client.uploads).toHaveLength(0) + }) + + it('serializes provisioning so concurrent calls cannot pass the max-concurrent cap', async () => { + const client = new FakeCloudflareEnvironmentClient() + let random = 0 + const provider = new CloudflareEnvironmentProvider({ + resource, + client, + config: { maxConcurrentEnvironments: 1 }, + randomId: () => `run${++random}`, + }) + const spec = { customerId: 'customer', repository: 'org/repo', ownerId: 'run' } + + const results = await Promise.allSettled([provider.provision(spec), provider.provision(spec)]) + + expect(results.filter((result) => result.status === 'fulfilled')).toHaveLength(1) + const rejected = results.find((result) => result.status === 'rejected') as PromiseRejectedResult + expect(rejected.reason).toBeInstanceOf(CloudflareEnvironmentQuotaError) + expect(String(rejected.reason)).toContain('max-concurrent') + expect(client.namespaces.size).toBe(1) + }) + + it('reconciles concurrent provisioning across provider instances', async () => { + const client = new FakeCloudflareEnvironmentClient() + client.namespaces.set('factory-verification-existing-z', { + namespace_name: 'factory-verification-existing-z', trusted_workers: false, + }) + const first = new CloudflareEnvironmentProvider({ + resource, client, config: { maxConcurrentEnvironments: 2 }, randomId: () => 'run-a', + }) + const second = new CloudflareEnvironmentProvider({ + resource, client, config: { maxConcurrentEnvironments: 2 }, randomId: () => 'run-b', + }) + const spec = { customerId: 'customer', repository: 'org/repo', ownerId: 'run' } + + const results = await Promise.allSettled([first.provision(spec), second.provision(spec)]) + + expect(results.filter((result) => result.status === 'fulfilled')).toHaveLength(1) + const rejected = results.find((result) => result.status === 'rejected') as PromiseRejectedResult + expect(rejected.reason).toBeInstanceOf(CloudflareEnvironmentQuotaError) + expect(String(rejected.reason)).toContain('concurrent provision') + expect(client.namespaces.size).toBe(2) + expect(client.namespaces.has('factory-verification-existing-z')).toBe(true) + }) + + it('isolates environments by namespace and refuses teardown after ownership metadata is changed', async () => { + const client = new FakeCloudflareEnvironmentClient() + let random = 0 + const provider = new CloudflareEnvironmentProvider({ + resource, + client, + config: { maxConcurrentEnvironments: 2 }, + randomId: () => `run${++random}`, + }) + const first = await provider.provision({ + customerId: 'customer-a', repository: 'org/repo', ownerId: 'owner-a', + }) + const second = await provider.provision({ + customerId: 'customer-b', repository: 'org/repo', ownerId: 'owner-b', + }) + + expect(first.dispatchNamespace).not.toBe(second.dispatchNamespace) + expect(plainTextBindings(client.bindings.get(`${first.id}/${CLOUDFLARE_METADATA_SCRIPT}`) ?? [])) + .toMatchObject({ FACTORY_OWNER_ID: 'owner-a' }) + expect(plainTextBindings(client.bindings.get(`${second.id}/${CLOUDFLARE_METADATA_SCRIPT}`) ?? [])) + .toMatchObject({ FACTORY_OWNER_ID: 'owner-b' }) + + const firstBindings = client.bindings.get(`${first.id}/${CLOUDFLARE_METADATA_SCRIPT}`) + const identity = firstBindings?.find((binding) => binding.name === CLOUDFLARE_ENVIRONMENT_BINDINGS.environmentId) + if (!identity) throw new Error('fixture identity binding is missing') + identity.text = second.id + + await expect(provider.destroy(first.id)).rejects.toThrow('ownership identity mismatch') + expect(client.namespaces.has(first.id)).toBe(true) + expect(client.namespaces.has(second.id)).toBe(true) + }) + + it('cleans up a newly-created namespace if its metadata Worker cannot be uploaded', async () => { + const client = new FakeCloudflareEnvironmentClient() + client.failUpload = true + const provider = new CloudflareEnvironmentProvider({ + resource, client, randomId: () => 'cleanup', + }) + + await expect(provider.provision({ + customerId: 'customer', repository: 'org/repo', ownerId: 'owner', + })).rejects.toThrow('upload failed') + expect(client.namespaces.size).toBe(0) + expect(client.deletes).toEqual(['factory-verification-repo-cleanup']) + }) + + it('rejects trusted dispatch namespaces and removes the unsafe allocation', async () => { + const client = new FakeCloudflareEnvironmentClient() + client.createTrustedNamespace = true + const provider = new CloudflareEnvironmentProvider({ + resource, client, randomId: () => 'trusted', + }) + + await expect(provider.provision({ + customerId: 'customer', repository: 'org/repo', ownerId: 'owner', + })).rejects.toThrow('must be untrusted') + expect(client.namespaces.size).toBe(0) + }) + + it('reaps expired and dead-owner environments while preserving invalid or live leases', async () => { + const client = new FakeCloudflareEnvironmentClient() + let now = new Date('2026-07-21T10:00:00.000Z') + let random = 0 + const provider = new CloudflareEnvironmentProvider({ + resource, + client, + config: { maxConcurrentEnvironments: 4, defaultTtlMs: 60_000 }, + now: () => now, + randomId: () => `reap${++random}`, + ownerIsAlive: async (ownerId) => ownerId !== 'dead-owner', + }) + const expired = await provider.provision({ + customerId: 'customer', repository: 'org/repo', ownerId: 'live-owner', + }) + const deadOwner = await provider.provision({ + customerId: 'customer', repository: 'org/repo', ownerId: 'dead-owner', ttl: 120_000, + }) + const live = await provider.provision({ + customerId: 'customer', repository: 'org/repo', ownerId: 'live-owner', ttl: 120_000, + }) + client.namespaces.set('factory-verification-spoof-invalid', { + namespace_name: 'factory-verification-spoof-invalid', trusted_workers: false, + }) + now = new Date('2026-07-21T10:01:01.000Z') + + const report = await provider.reap() + + expect(report.reaped).toEqual(expect.arrayContaining([ + { id: expired.id, reason: 'ttl-expired' }, + { id: deadOwner.id, reason: 'owner-gone' }, + ])) + expect(report.skipped).toContainEqual({ + id: 'factory-verification-spoof-invalid', + reason: 'Refusing Cloudflare namespace factory-verification-spoof-invalid: ownership metadata Worker is missing', + }) + expect(client.namespaces.has(expired.id)).toBe(false) + expect(client.namespaces.has(deadOwner.id)).toBe(false) + expect(client.namespaces.has(live.id)).toBe(true) + expect(client.namespaces.has('factory-verification-spoof-invalid')).toBe(true) + }) +}) + +describe('HttpCloudflareEnvironmentClient', () => { + it('uses the documented account dispatch endpoint and bearer resource token', async () => { + const calls: Array<{ url: string; init?: RequestInit }> = [] + const fetch = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(input), init }) + return new Response(JSON.stringify({ + success: true, + result: { namespace_name: 'factory-verification-repo-api' }, + }), { status: 200, headers: { 'content-type': 'application/json' } }) + }) as typeof globalThis.fetch + const client = new HttpCloudflareEnvironmentClient({ resource, fetch }) + + await client.createDispatchNamespace('factory-verification-repo-api') + + expect(calls).toHaveLength(1) + expect(calls[0].url).toBe( + `https://api.cloudflare.com/client/v4/accounts/${resource.accountId}/workers/dispatch/namespaces`, + ) + expect(calls[0].init).toMatchObject({ + method: 'POST', + headers: { + authorization: 'Bearer resource-secret-token', + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'factory-verification-repo-api' }), + }) + }) + + it('treats GET 404 as absence and does not leak the token in API errors', async () => { + const notFound = new HttpCloudflareEnvironmentClient({ + resource, + fetch: vi.fn(async () => new Response(JSON.stringify({ success: false, errors: [] }), { + status: 404, + })) as typeof globalThis.fetch, + }) + expect(await notFound.getDispatchNamespace('missing')).toBeUndefined() + + const failing = new HttpCloudflareEnvironmentClient({ + resource, + fetch: vi.fn(async () => new Response(JSON.stringify({ + success: false, + errors: [{ code: 10001, message: 'authentication failed' }], + }), { status: 403 })) as typeof globalThis.fetch, + }) + const error = await failing.listDispatchNamespaces().catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(CloudflareApiError) + expect(String(error)).toContain('authentication failed') + expect(String(error)).not.toContain(resource.apiToken) + }) +}) + +describe('cloudflareEnvironmentName', () => { + it('produces a stable DNS label within the Cloudflare namespace bound', () => { + const name = cloudflareEnvironmentName( + 'factory-verification', + `AgentWorkforce/${'Very_Long_Repository_'.repeat(5)}`, + 'A-B_C!123', + ) + + expect(name).toMatch(/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u) + expect(name.length).toBeLessThanOrEqual(63) + expect(name.endsWith('-abc123')).toBe(true) + }) +}) + +function plainTextBindings(bindings: CloudflareWorkerBinding[]): Record { + return Object.fromEntries(bindings.flatMap((binding) => ( + binding.type === 'plain_text' && typeof binding.text === 'string' + ? [[binding.name, binding.text]] + : [] + ))) +} diff --git a/src/environments/cloudflare-provider.ts b/src/environments/cloudflare-provider.ts new file mode 100644 index 0000000..570b4f9 --- /dev/null +++ b/src/environments/cloudflare-provider.ts @@ -0,0 +1,758 @@ +import { randomUUID } from 'node:crypto' + +import { z } from 'zod' + +import { normalizeLogger } from '../logging.js' +import type { + Environment, + EnvironmentProvider, + EnvironmentStatus, + ProvisionEnvironmentSpec, +} from '../ports/environment.js' +import type { Logger } from '../ports/system.js' + +export const CLOUDFLARE_METADATA_SCRIPT = 'factory-environment' +export const CLOUDFLARE_MANAGED_TAG = 'factory-managed' +export const CLOUDFLARE_ENVIRONMENT_TAG = 'factory-verification-environment' + +export const CLOUDFLARE_ENVIRONMENT_BINDINGS = { + environmentId: 'FACTORY_ENVIRONMENT_ID', + ownerId: 'FACTORY_OWNER_ID', + customerId: 'FACTORY_CUSTOMER_ID', + repository: 'FACTORY_REPOSITORY', + createdAt: 'FACTORY_CREATED_AT', + expiresAt: 'FACTORY_EXPIRES_AT', + maxRunCostUsd: 'FACTORY_MAX_RUN_COST_USD', + containerMaxInstances: 'FACTORY_CONTAINER_MAX_INSTANCES', + workerCpuMs: 'FACTORY_WORKER_CPU_MS', + workerSubrequests: 'FACTORY_WORKER_SUBREQUESTS', +} as const + +const dnsLabelSchema = z.string().trim().min(1).max(40).regex( + /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u, + 'must be a lowercase DNS label', +) + +const containerInstanceTypeSchema = z.enum([ + 'lite', + 'basic', + 'standard-1', + 'standard-2', + 'standard-3', + 'standard-4', +]) + +/** + * Source-control-safe Cloudflare policy. Account credentials deliberately do + * not belong here; `resource` names the SST Resource.*-style object supplied + * to `CloudflareEnvironmentProvider.fromResource` at runtime. + */ +export const CloudflareEnvironmentConfigSchema = z.object({ + resource: z.string().trim().min(1).default('FactoryTestInfra'), + dispatchNamespacePrefix: dnsLabelSchema.default('factory-verification'), + maxConcurrentEnvironments: z.number().int().min(1).max(100).default(2), + defaultTtlMs: z.number().int().min(60_000).max(24 * 60 * 60_000).default(15 * 60_000), + maxTtlMs: z.number().int().min(60_000).max(24 * 60 * 60_000).default(60 * 60_000), + maxRunCostUsd: z.number().positive().max(1_000).default(1), + workerLimits: z.object({ + cpuMs: z.number().int().min(1).max(300_000).default(100), + subrequests: z.number().int().min(1).max(10_000_000).default(100), + }).strict().default({}), + container: z.object({ + instanceType: containerInstanceTypeSchema.default('lite'), + maxInstances: z.number().int().min(1).max(100).default(3), + }).strict().default({}), +}).strict().superRefine((config, context) => { + if (config.defaultTtlMs > config.maxTtlMs) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['defaultTtlMs'], + message: 'must be less than or equal to maxTtlMs', + }) + } +}).default({}) + +/** Runtime secret/output object linked by factory-test-infra. */ +export const CloudflareEnvironmentResourceSchema = z.object({ + accountId: z.string().trim().min(1).max(32), + apiToken: z.string().min(1), + /** Optional path template exposed by the infra dispatch Worker. */ + dispatcherUrlTemplate: z.string().url().refine( + (value) => value.includes('{namespace}') && value.includes('{script}'), + 'must contain {namespace} and {script} placeholders', + ).optional(), +}).strict() + +export const CloudflareProvisionStackSchema = z.object({ + /** Requested concurrent live Container instances for this run. */ + containerInstances: z.number().int().min(0).default(0), + /** Hard upper cost reservation for the infra usage monitor. */ + runCostBudgetUsd: z.number().positive().optional(), +}).strict().default({}) + +export type CloudflareEnvironmentConfig = z.output +export type CloudflareEnvironmentConfigInput = z.input +export type CloudflareEnvironmentResource = z.output +export type CloudflareProvisionStack = z.output + +export interface CloudflareProvisionSpec extends Omit { + stack?: z.input +} + +export interface CloudflareDispatchNamespace { + namespace_name: string + namespace_id?: string + created_on?: string + modified_on?: string + script_count?: number + trusted_workers?: boolean +} + +export interface CloudflareWorkerBinding { + name: string + type: string + text?: string + [key: string]: unknown +} + +export interface UploadCloudflareWorkerInput { + namespace: string + scriptName: string + source: string + compatibilityDate: string + bindings: CloudflareWorkerBinding[] + tags: string[] + limits: { + cpu_ms: number + subrequests: number + } +} + +/** Small API seam so lifecycle behavior can be tested without Cloudflare. */ +export interface CloudflareEnvironmentClient { + listDispatchNamespaces(): Promise + getDispatchNamespace(name: string): Promise + createDispatchNamespace(name: string): Promise + deleteDispatchNamespace(name: string): Promise + uploadDispatchWorker(input: UploadCloudflareWorkerInput): Promise + getDispatchWorkerBindings(namespace: string, scriptName: string): Promise +} + +interface CloudflareApiEnvelope { + success: boolean + result?: T + errors?: Array<{ code?: number; message?: string }> + messages?: Array<{ code?: number; message?: string }> +} + +export interface HttpCloudflareEnvironmentClientOptions { + resource: CloudflareEnvironmentResource + fetch?: typeof globalThis.fetch + apiBaseUrl?: string +} + +export class CloudflareApiError extends Error { + constructor( + message: string, + public readonly status: number, + public readonly method: string, + public readonly path: string, + ) { + super(message) + this.name = 'CloudflareApiError' + } +} + +/** Minimal Workers for Platforms client; no credential is read from process.env. */ +export class HttpCloudflareEnvironmentClient implements CloudflareEnvironmentClient { + readonly #resource: CloudflareEnvironmentResource + readonly #fetch: typeof globalThis.fetch + readonly #apiBaseUrl: string + + constructor(options: HttpCloudflareEnvironmentClientOptions) { + this.#resource = CloudflareEnvironmentResourceSchema.parse(options.resource) + this.#fetch = options.fetch ?? globalThis.fetch + this.#apiBaseUrl = (options.apiBaseUrl ?? 'https://api.cloudflare.com/client/v4').replace(/\/+$/u, '') + } + + async listDispatchNamespaces(): Promise { + return await this.#request('GET', this.#namespacePath()) ?? [] + } + + async getDispatchNamespace(name: string): Promise { + return await this.#request( + 'GET', + `${this.#namespacePath()}/${encodeURIComponent(name)}`, + undefined, + { notFoundIsUndefined: true }, + ) + } + + async createDispatchNamespace(name: string): Promise { + const created = await this.#request('POST', this.#namespacePath(), { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name }), + }) + if (!created?.namespace_name) throw new Error('Cloudflare create dispatch namespace returned no namespace identity') + return created + } + + async deleteDispatchNamespace(name: string): Promise { + await this.#request('DELETE', `${this.#namespacePath()}/${encodeURIComponent(name)}`) + } + + async uploadDispatchWorker(input: UploadCloudflareWorkerInput): Promise { + const moduleName = `${input.scriptName}.mjs` + const metadata = { + main_module: moduleName, + compatibility_date: input.compatibilityDate, + bindings: input.bindings, + tags: input.tags, + limits: input.limits, + } + const form = new FormData() + form.append('metadata', new Blob([JSON.stringify(metadata)], { type: 'application/json' })) + form.append(moduleName, new Blob([input.source], { type: 'application/javascript+module' }), moduleName) + await this.#request( + 'PUT', + `${this.#namespacePath()}/${encodeURIComponent(input.namespace)}/scripts/${encodeURIComponent(input.scriptName)}`, + { body: form }, + ) + } + + async getDispatchWorkerBindings( + namespace: string, + scriptName: string, + ): Promise { + return await this.#request( + 'GET', + `${this.#namespacePath()}/${encodeURIComponent(namespace)}/scripts/${encodeURIComponent(scriptName)}/bindings`, + undefined, + { notFoundIsUndefined: true }, + ) + } + + #namespacePath(): string { + return `/accounts/${encodeURIComponent(this.#resource.accountId)}/workers/dispatch/namespaces` + } + + async #request( + method: string, + path: string, + init: Omit = {}, + options: { notFoundIsUndefined?: boolean } = {}, + ): Promise { + const response = await this.#fetch(`${this.#apiBaseUrl}${path}`, { + ...init, + method, + headers: { + authorization: `Bearer ${this.#resource.apiToken}`, + ...init.headers, + }, + }) + if (response.status === 404 && options.notFoundIsUndefined) return undefined + + const envelope = await response.json().catch(() => undefined) as CloudflareApiEnvelope | undefined + if (!response.ok || envelope?.success !== true) { + const details = [...(envelope?.errors ?? []), ...(envelope?.messages ?? [])] + .map((entry) => entry.message || (entry.code === undefined ? '' : String(entry.code))) + .filter(Boolean) + .join('; ') + throw new CloudflareApiError( + `Cloudflare API ${method} ${path} failed with HTTP ${response.status}${details ? `: ${details}` : ''}`, + response.status, + method, + path, + ) + } + return envelope.result + } +} + +interface CloudflareEnvironmentRecord { + environment: Environment + ownerId: string + expiresAt: string +} + +export interface CloudflareEnvironmentProviderOptions { + config?: CloudflareEnvironmentConfigInput + resource: CloudflareEnvironmentResource + client?: CloudflareEnvironmentClient + now?: () => Date + randomId?: () => string + logger?: Logger + ownerIsAlive?: (ownerId: string) => Promise +} + +export interface CloudflareEnvironmentProviderResourceOptions extends Omit< + CloudflareEnvironmentProviderOptions, + 'resource' | 'config' +> { + config?: CloudflareEnvironmentConfigInput + resources: Record +} + +export interface CloudflareReapReport { + reaped: Array<{ id: string; reason: 'ttl-expired' | 'owner-gone' }> + skipped: Array<{ id?: string; reason: string }> +} + +export class CloudflareEnvironmentQuotaError extends Error { + constructor(message: string) { + super(message) + this.name = 'CloudflareEnvironmentQuotaError' + } +} + +/** + * One-untrusted-dispatch-namespace-per-run Cloudflare provider. + * + * The metadata Worker is intentionally inert. Its bindings are the durable, + * identity-checked lease that both Factory and factory-test-infra's Cron reaper + * can inspect before deleting a namespace. + */ +export class CloudflareEnvironmentProvider implements EnvironmentProvider { + readonly config: CloudflareEnvironmentConfig + readonly #resource: CloudflareEnvironmentResource + readonly #client: CloudflareEnvironmentClient + readonly #now: () => Date + readonly #randomId: () => string + readonly #logger?: Logger + readonly #ownerIsAlive?: (ownerId: string) => Promise + readonly #records = new Map() + #provisionLock: Promise = Promise.resolve() + + constructor(options: CloudflareEnvironmentProviderOptions) { + this.config = CloudflareEnvironmentConfigSchema.parse(options.config ?? {}) + this.#resource = CloudflareEnvironmentResourceSchema.parse(options.resource) + this.#client = options.client ?? new HttpCloudflareEnvironmentClient({ resource: this.#resource }) + this.#now = options.now ?? (() => new Date()) + this.#randomId = options.randomId ?? (() => randomUUID().replaceAll('-', '').slice(0, 10)) + this.#logger = options.logger ? normalizeLogger(options.logger) : undefined + this.#ownerIsAlive = options.ownerIsAlive + } + + static fromResource(options: CloudflareEnvironmentProviderResourceOptions): CloudflareEnvironmentProvider { + const config = CloudflareEnvironmentConfigSchema.parse(options.config ?? {}) + const resource = options.resources[config.resource] + if (resource === undefined) { + throw new Error(`Cloudflare environment resource ${JSON.stringify(config.resource)} is unavailable`) + } + const { resources: _resources, ...providerOptions } = options + return new CloudflareEnvironmentProvider({ + ...providerOptions, + config, + resource: CloudflareEnvironmentResourceSchema.parse(resource), + }) + } + + async provision(spec: CloudflareProvisionSpec): Promise { + return await this.#withProvisionLock(async () => { + const stack = CloudflareProvisionStackSchema.parse(spec.stack ?? {}) + const ttl = spec.ttl ?? this.config.defaultTtlMs + this.#assertRequestWithinGuardrails(ttl, stack) + + const active = (await this.#client.listDispatchNamespaces()) + .filter((namespace) => isManagedNamespaceName(namespace.namespace_name, this.config.dispatchNamespacePrefix)) + if (active.length >= this.config.maxConcurrentEnvironments) { + throw new CloudflareEnvironmentQuotaError( + `Cloudflare max-concurrent environment cap of ${this.config.maxConcurrentEnvironments} is exhausted`, + ) + } + const preexistingNames = new Set(active.map((namespace) => namespace.namespace_name)) + + const createdAt = this.#now() + const expiresAt = new Date(createdAt.getTime() + ttl) + const id = cloudflareEnvironmentName( + this.config.dispatchNamespacePrefix, + spec.repository, + this.#randomId(), + ) + let namespaceCreated = false + try { + const namespace = await this.#client.createDispatchNamespace(id) + namespaceCreated = true + if (namespace.namespace_name !== id) { + throw new Error(`Cloudflare returned dispatch namespace ${namespace.namespace_name} for requested identity ${id}`) + } + if (namespace.trusted_workers === true) { + throw new Error(`Cloudflare dispatch namespace ${id} is trusted; verification namespaces must be untrusted`) + } + + // The in-process lock closes races within one Factory instance. This + // reconciliation also closes the common multi-run race where two + // providers both observe one free slot before either creates it. + const reconciled = (await this.#client.listDispatchNamespaces()) + .filter((candidate) => isManagedNamespaceName( + candidate.namespace_name, + this.config.dispatchNamespacePrefix, + )) + .sort((left, right) => ( + Number(preexistingNames.has(right.namespace_name)) - Number(preexistingNames.has(left.namespace_name)) || + compareDispatchNamespaces(left, right) + )) + if (reconciled.length > this.config.maxConcurrentEnvironments && + reconciled.findIndex((candidate) => candidate.namespace_name === id) >= + this.config.maxConcurrentEnvironments) { + throw new CloudflareEnvironmentQuotaError( + `Cloudflare max-concurrent environment cap of ${this.config.maxConcurrentEnvironments} ` + + 'was exceeded by a concurrent provision', + ) + } + + const costBudget = stack.runCostBudgetUsd ?? this.config.maxRunCostUsd + await this.#client.uploadDispatchWorker({ + namespace: id, + scriptName: CLOUDFLARE_METADATA_SCRIPT, + source: metadataWorkerSource(), + compatibilityDate: createdAt.toISOString().slice(0, 10), + bindings: environmentMetadataBindings({ + id, + spec, + createdAt, + expiresAt, + costBudget, + containerMaxInstances: stack.containerInstances || this.config.container.maxInstances, + workerCpuMs: this.config.workerLimits.cpuMs, + workerSubrequests: this.config.workerLimits.subrequests, + }), + tags: [CLOUDFLARE_MANAGED_TAG, CLOUDFLARE_ENVIRONMENT_TAG, `factory-environment-${id}`], + limits: { + cpu_ms: this.config.workerLimits.cpuMs, + subrequests: this.config.workerLimits.subrequests, + }, + }) + + const environment = this.#environmentFromMetadata({ + id, + ownerId: spec.ownerId, + createdAt: createdAt.toISOString(), + expiresAt: expiresAt.toISOString(), + ttl, + containerMaxInstances: stack.containerInstances || this.config.container.maxInstances, + costBudget, + }) + this.#records.set(id, { environment, ownerId: spec.ownerId, expiresAt: expiresAt.toISOString() }) + this.#logger?.info?.('[cloudflare-environment] provisioned environment', { + id, + expiresAt: expiresAt.toISOString(), + containerMaxInstances: environment.bindings['cloudflare.container.maxInstances'], + maxRunCostUsd: environment.bindings['cloudflare.maxRunCostUsd'], + }) + return cloneEnvironment(environment) + } catch (error) { + if (!namespaceCreated) throw error + try { + await this.#client.deleteDispatchNamespace(id) + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `Cloudflare provision failed and cleanup also failed for ${id}`, + ) + } + throw error + } + }) + } + + async status(id: string): Promise { + const namespace = await this.#client.getDispatchNamespace(id) + if (!namespace) { + this.#records.delete(id) + return 'destroyed' + } + await this.#readOwnedMetadata(id) + return this.#records.get(id)?.environment.status ?? 'ready' + } + + async endpoints(id: string): Promise> { + const namespace = await this.#client.getDispatchNamespace(id) + if (!namespace) throw new Error(`Cloudflare environment ${id} does not exist`) + await this.#readOwnedMetadata(id) + return {} + } + + async destroy(id: string): Promise { + const namespace = await this.#client.getDispatchNamespace(id) + if (!namespace) { + this.#records.delete(id) + return + } + await this.#readOwnedMetadata(id) + const record = this.#records.get(id) + if (record) record.environment.status = 'destroying' + await this.#client.deleteDispatchNamespace(id) + if (record) record.environment.status = 'destroyed' + this.#records.delete(id) + this.#logger?.info?.('[cloudflare-environment] destroyed environment', { id }) + } + + async reap(): Promise { + const report: CloudflareReapReport = { reaped: [], skipped: [] } + const nowMs = this.#now().getTime() + let namespaces: CloudflareDispatchNamespace[] + try { + namespaces = (await this.#client.listDispatchNamespaces()) + .filter((namespace) => isManagedNamespaceName(namespace.namespace_name, this.config.dispatchNamespacePrefix)) + } catch (error) { + return { reaped: [], skipped: [{ reason: `list failed: ${errorMessage(error)}` }] } + } + + for (const namespace of namespaces) { + const id = namespace.namespace_name + let metadata: Map + try { + metadata = await this.#readOwnedMetadata(id) + } catch (error) { + report.skipped.push({ id, reason: errorMessage(error) }) + continue + } + const expiresAt = Date.parse(metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.expiresAt) ?? '') + const ownerId = metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.ownerId) + let reason: CloudflareReapReport['reaped'][number]['reason'] | undefined + if (Number.isFinite(expiresAt) && expiresAt <= nowMs) { + reason = 'ttl-expired' + } else if (ownerId && this.#ownerIsAlive) { + try { + if (await this.#ownerIsAlive(ownerId) === false) reason = 'owner-gone' + } catch (error) { + report.skipped.push({ id, reason: `owner check failed: ${errorMessage(error)}` }) + continue + } + } + if (!reason) continue + + try { + await this.#client.deleteDispatchNamespace(id) + } catch (error) { + report.skipped.push({ id, reason: `namespace deletion failed: ${errorMessage(error)}` }) + continue + } + const record = this.#records.get(id) + if (record) record.environment.status = 'destroyed' + this.#records.delete(id) + report.reaped.push({ id, reason }) + this.#logger?.warn?.('[cloudflare-environment] reaped environment', { id, reason }) + } + return report + } + + #assertRequestWithinGuardrails(ttl: number, stack: CloudflareProvisionStack): void { + if (!Number.isInteger(ttl) || ttl < 60_000 || ttl > this.config.maxTtlMs) { + throw new CloudflareEnvironmentQuotaError( + `Cloudflare environment ttl must be between 60000 and ${this.config.maxTtlMs} milliseconds`, + ) + } + if (stack.containerInstances > this.config.container.maxInstances) { + throw new CloudflareEnvironmentQuotaError( + `Cloudflare Container request of ${stack.containerInstances} exceeds per-environment cap of ${this.config.container.maxInstances}`, + ) + } + if ((stack.runCostBudgetUsd ?? 0) > this.config.maxRunCostUsd) { + throw new CloudflareEnvironmentQuotaError( + `Cloudflare run cost budget $${stack.runCostBudgetUsd} exceeds per-run cap of $${this.config.maxRunCostUsd}`, + ) + } + } + + async #readOwnedMetadata(id: string): Promise> { + if (!isManagedNamespaceName(id, this.config.dispatchNamespacePrefix)) { + throw new Error(`Refusing Cloudflare namespace ${id}: name is outside the managed prefix`) + } + const bindings = await this.#client.getDispatchWorkerBindings(id, CLOUDFLARE_METADATA_SCRIPT) + if (!bindings) { + throw new Error(`Refusing Cloudflare namespace ${id}: ownership metadata Worker is missing`) + } + const metadata = new Map(bindings.flatMap((binding) => ( + binding.type === 'plain_text' && typeof binding.text === 'string' + ? [[binding.name, binding.text] as const] + : [] + ))) + if (metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.environmentId) !== id) { + throw new Error(`Refusing Cloudflare namespace ${id}: ownership identity mismatch`) + } + const createdAt = metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.createdAt) + const expiresAt = metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.expiresAt) + const ownerId = metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.ownerId) + const createdAtMs = Date.parse(createdAt ?? '') + const expiresAtMs = Date.parse(expiresAt ?? '') + if (!ownerId || !Number.isFinite(createdAtMs) || !Number.isFinite(expiresAtMs) || expiresAtMs <= createdAtMs) { + throw new Error(`Refusing Cloudflare namespace ${id}: ownership lease is incomplete or invalid`) + } + if (!this.#records.has(id)) { + const ttl = expiresAtMs - createdAtMs + const costBudget = parseFiniteNumber(metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.maxRunCostUsd)) + const containerMaxInstances = parseFiniteNumber( + metadata.get(CLOUDFLARE_ENVIRONMENT_BINDINGS.containerMaxInstances), + ) + this.#records.set(id, { + environment: this.#environmentFromMetadata({ + id, + ownerId, + createdAt: new Date(createdAtMs).toISOString(), + expiresAt: new Date(expiresAtMs).toISOString(), + ttl, + costBudget, + containerMaxInstances, + }), + ownerId, + expiresAt: new Date(expiresAtMs).toISOString(), + }) + } + return metadata + } + + #environmentFromMetadata(input: { + id: string + ownerId: string + createdAt: string + expiresAt: string + ttl: number + costBudget: number + containerMaxInstances: number + }): Environment { + return { + id: input.id, + provider: 'cloudflare', + dispatchNamespace: input.id, + endpoints: {}, + bindings: { + 'cloudflare.resource': this.config.resource, + 'cloudflare.metadataScript': CLOUDFLARE_METADATA_SCRIPT, + 'cloudflare.expiresAt': input.expiresAt, + 'cloudflare.maxRunCostUsd': String(input.costBudget), + 'cloudflare.container.instanceType': this.config.container.instanceType, + 'cloudflare.container.maxInstances': String(input.containerMaxInstances), + 'cloudflare.worker.cpuMs': String(this.config.workerLimits.cpuMs), + 'cloudflare.worker.subrequests': String(this.config.workerLimits.subrequests), + }, + status: 'ready', + createdAt: input.createdAt, + ttl: input.ttl, + } + } + + async #withProvisionLock(work: () => Promise): Promise { + const previous = this.#provisionLock + let release = (): void => undefined + this.#provisionLock = new Promise((resolve) => { release = resolve }) + await previous + try { + return await work() + } finally { + release() + } + } +} + +export class CloudflareEnvironmentReaper { + readonly #provider: CloudflareEnvironmentProvider + readonly #intervalMs: number + #timer: ReturnType | undefined + #running = false + #stopped = false + + constructor(provider: CloudflareEnvironmentProvider, options: { intervalMs?: number } = {}) { + this.#provider = provider + this.#intervalMs = options.intervalMs ?? 60_000 + } + + start(): void { + if (this.#timer || this.#stopped) return + this.#schedule(0) + } + + async stop(): Promise { + this.#stopped = true + if (this.#timer) clearTimeout(this.#timer) + this.#timer = undefined + } + + #schedule(delayMs: number): void { + this.#timer = setTimeout(() => { + this.#timer = undefined + void this.#tick() + }, delayMs) + } + + async #tick(): Promise { + if (this.#stopped || this.#running) return + this.#running = true + try { + await this.#provider.reap() + } finally { + this.#running = false + if (!this.#stopped) this.#schedule(this.#intervalMs) + } + } +} + +export function cloudflareEnvironmentName(prefix: string, repository: string, randomId: string): string { + const suffix = randomId.toLowerCase().replace(/[^a-z0-9]+/gu, '').slice(0, 12) + if (!suffix) throw new Error('Cloudflare environment random identity must contain a letter or number') + const repositoryName = (repository.split('/').pop() ?? repository) + .toLowerCase() + .replace(/[^a-z0-9]+/gu, '-') + .replace(/^-+|-+$/gu, '') || 'repo' + const available = 63 - prefix.length - suffix.length - 2 + if (available < 1) throw new Error('Cloudflare environment prefix and random identity leave no room for a repository name') + const repo = repositoryName.slice(0, Math.max(1, available)).replace(/-+$/gu, '') || 'r' + return `${prefix}-${repo}-${suffix}` +} + +function isManagedNamespaceName(name: string, prefix: string): boolean { + return name.startsWith(`${prefix}-`) && name.length <= 63 +} + +function compareDispatchNamespaces(left: CloudflareDispatchNamespace, right: CloudflareDispatchNamespace): number { + const created = (left.created_on ?? '').localeCompare(right.created_on ?? '') + return created || left.namespace_name.localeCompare(right.namespace_name) +} + +function metadataWorkerSource(): string { + return `export default { async fetch() { return new Response('not found', { status: 404 }); } };\n` +} + +function environmentMetadataBindings(input: { + id: string + spec: CloudflareProvisionSpec + createdAt: Date + expiresAt: Date + costBudget: number + containerMaxInstances: number + workerCpuMs: number + workerSubrequests: number +}): CloudflareWorkerBinding[] { + const values: Array<[string, string]> = [ + [CLOUDFLARE_ENVIRONMENT_BINDINGS.environmentId, input.id], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.ownerId, input.spec.ownerId], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.customerId, input.spec.customerId], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.repository, input.spec.repository], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.createdAt, input.createdAt.toISOString()], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.expiresAt, input.expiresAt.toISOString()], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.maxRunCostUsd, String(input.costBudget)], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.containerMaxInstances, String(input.containerMaxInstances)], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.workerCpuMs, String(input.workerCpuMs)], + [CLOUDFLARE_ENVIRONMENT_BINDINGS.workerSubrequests, String(input.workerSubrequests)], + ] + return values.map(([name, text]) => ({ name, type: 'plain_text', text })) +} + +function parseFiniteNumber(value: string | undefined): number { + const parsed = Number(value) + if (!Number.isFinite(parsed)) throw new Error(`Cloudflare environment metadata number is invalid: ${value}`) + return parsed +} + +function cloneEnvironment(environment: Environment): Environment { + return { + ...environment, + endpoints: { ...environment.endpoints }, + bindings: { ...environment.bindings }, + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/environments/cloudflare-workflow.test.ts b/src/environments/cloudflare-workflow.test.ts new file mode 100644 index 0000000..3f0bd1f --- /dev/null +++ b/src/environments/cloudflare-workflow.test.ts @@ -0,0 +1,39 @@ +import { readFile } from 'node:fs/promises' + +import { parse } from 'yaml' +import { describe, expect, it } from 'vitest' + +describe('reusable Cloudflare verification workflow', () => { + it('runs the infra up/suite/down contract in Miniflare and real Cloudflare modes', async () => { + const text = await readFile('.github/workflows/cloudflare-verification.yml', 'utf8') + const workflow = parse(text) as { + on: { workflow_call: unknown } + jobs: Record> }> + } + + expect(workflow.on.workflow_call).toBeDefined() + expect(Object.keys(workflow.jobs)).toEqual(['miniflare', 'cloudflare']) + expect(workflow.jobs.cloudflare.if).toBe('inputs.run_real') + + for (const [jobName, mode] of [['miniflare', 'miniflare'], ['cloudflare', 'cloudflare']] as const) { + const job = workflow.jobs[jobName] + const commands = job.steps.flatMap((step) => typeof step.run === 'string' ? [step.run] : []) + expect(commands).toContain('bash ci/verification-env/up.sh') + expect(commands).toContain('bash ci/verification-env/run-suite.sh') + expect(commands).toContain('bash ci/verification-env/down.sh') + + const lifecycle = job.steps.filter((step) => ( + typeof step.run === 'string' && step.run.startsWith('bash ci/verification-env/') + )) + expect(lifecycle).toHaveLength(3) + expect(lifecycle.every((step) => ( + (step.env as Record).FACTORY_VERIFICATION_MODE === mode + ))).toBe(true) + expect(lifecycle.find((step) => step.run === 'bash ci/verification-env/down.sh')?.if).toBe('always()') + } + + const realSteps = workflow.jobs.cloudflare.steps + expect(realSteps.some((step) => step.name === 'Require real Cloudflare resource secret')).toBe(true) + expect(realSteps.some((step) => step.name === 'Upload real-environment evidence' && step.if === 'always()')).toBe(true) + }) +}) diff --git a/src/environments/index.ts b/src/environments/index.ts index 16a364f..71677a5 100644 --- a/src/environments/index.ts +++ b/src/environments/index.ts @@ -1,3 +1,33 @@ +export { + CLOUDFLARE_ENVIRONMENT_BINDINGS, + CLOUDFLARE_ENVIRONMENT_TAG, + CLOUDFLARE_MANAGED_TAG, + CLOUDFLARE_METADATA_SCRIPT, + CloudflareApiError, + CloudflareEnvironmentConfigSchema, + CloudflareEnvironmentProvider, + CloudflareEnvironmentQuotaError, + CloudflareEnvironmentReaper, + CloudflareEnvironmentResourceSchema, + CloudflareProvisionStackSchema, + HttpCloudflareEnvironmentClient, + cloudflareEnvironmentName, +} from './cloudflare-provider.js' +export type { + CloudflareDispatchNamespace, + CloudflareEnvironmentClient, + CloudflareEnvironmentConfig, + CloudflareEnvironmentConfigInput, + CloudflareEnvironmentProviderOptions, + CloudflareEnvironmentProviderResourceOptions, + CloudflareEnvironmentResource, + CloudflareProvisionSpec, + CloudflareProvisionStack, + CloudflareReapReport, + CloudflareWorkerBinding, + HttpCloudflareEnvironmentClientOptions, + UploadCloudflareWorkerInput, +} from './cloudflare-provider.js' export { DEFAULT_MANAGED_FIDELITY_CAVEAT, EnvironmentKubernetesCredentialResolver, diff --git a/src/index.ts b/src/index.ts index 5fd4cb3..65b5cfc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -366,6 +366,36 @@ export type { } from './environments/verification-stack-deployer' export { KubernetesEnvironmentProvider } from './environments/kubernetes-provider' export type { KubernetesEnvironmentProviderOptions } from './environments/kubernetes-provider' +export { + CLOUDFLARE_ENVIRONMENT_BINDINGS, + CLOUDFLARE_ENVIRONMENT_TAG, + CLOUDFLARE_MANAGED_TAG, + CLOUDFLARE_METADATA_SCRIPT, + CloudflareApiError, + CloudflareEnvironmentConfigSchema, + CloudflareEnvironmentProvider, + CloudflareEnvironmentQuotaError, + CloudflareEnvironmentReaper, + CloudflareEnvironmentResourceSchema, + CloudflareProvisionStackSchema, + HttpCloudflareEnvironmentClient, + cloudflareEnvironmentName, +} from './environments/cloudflare-provider' +export type { + CloudflareDispatchNamespace, + CloudflareEnvironmentClient, + CloudflareEnvironmentConfig, + CloudflareEnvironmentConfigInput, + CloudflareEnvironmentProviderOptions, + CloudflareEnvironmentProviderResourceOptions, + CloudflareEnvironmentResource, + CloudflareProvisionSpec, + CloudflareProvisionStack, + CloudflareReapReport, + CloudflareWorkerBinding, + HttpCloudflareEnvironmentClientOptions, + UploadCloudflareWorkerInput, +} from './environments/cloudflare-provider' export { CommandExecutionError, ProcessCommandRunner } from './environments/kubernetes-command' export type { CommandResult,