From 9ad8f50d32bc8f380b549046a671835fc108fa36 Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Sat, 4 Jul 2026 22:47:53 +0800 Subject: [PATCH 1/2] feat(tls): support multiple server certificates --- deploy/rustfs-operator/crds/tenant-crd.yaml | 224 ++++- deploy/rustfs-operator/crds/tenant.yaml | 224 ++++- docs/operator-user-guide.md | 43 +- docs/operator-user-guide.zh-CN.md | 43 +- examples/README.md | 1 + examples/multi-cert-tls-tenant.yaml | 65 ++ src/reconcile/tls.rs | 983 ++++++++++++++++---- src/types/v1alpha1/status/certificate.rs | 26 + src/types/v1alpha1/tenant/workloads.rs | 37 +- src/types/v1alpha1/tls.rs | 296 ++++-- 10 files changed, 1657 insertions(+), 285 deletions(-) create mode 100644 examples/multi-cert-tls-tenant.yaml diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index 888c15c..e35e7eb 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -1400,6 +1400,45 @@ spec: tls: nullable: true properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object certManager: nullable: true properties: @@ -1506,6 +1545,128 @@ spec: type: string type: array type: object + certificates: + items: + properties: + certManager: + properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object + certificateName: + nullable: true + type: string + commonName: + nullable: true + type: string + dnsNames: + items: + type: string + type: array + duration: + nullable: true + type: string + includeGeneratedDnsNames: + default: true + type: boolean + issuerRef: + nullable: true + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + type: string + name: + type: string + required: + - name + type: object + manageCertificate: + default: false + type: boolean + privateKey: + nullable: true + properties: + algorithm: + nullable: true + type: string + encoding: + nullable: true + type: string + rotationPolicy: + nullable: true + type: string + size: + format: int32 + nullable: true + type: integer + type: object + renewBefore: + nullable: true + type: string + secretName: + nullable: true + type: string + secretType: + nullable: true + type: string + usages: + items: + type: string + type: array + type: object + default: + default: false + type: boolean + hosts: + items: + type: string + type: array + name: + type: string + required: + - certManager + - name + type: object + type: array enableInternodeHttps: default: false type: boolean @@ -1616,6 +1777,67 @@ spec: - kind - name type: object + certificates: + items: + properties: + certificateRef: + nullable: true + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + observedGeneration: + format: int64 + nullable: true + type: integer + ready: + nullable: true + type: boolean + reason: + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + default: + default: false + type: boolean + dnsNames: + items: + type: string + type: array + hosts: + items: + type: string + type: array + managedCertificate: + nullable: true + type: boolean + name: + type: string + serverSecretRef: + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + required: + - name + - serverSecretRef + type: object + type: array clientCaSecretRef: nullable: true properties: @@ -1725,7 +1947,7 @@ spec: description: Status of the condition (True, False, Unknown) type: string type: - description: Type of condition (Ready, Progressing, Degraded) + description: Type of condition (Ready, Reconciling, Degraded) type: string required: - message diff --git a/deploy/rustfs-operator/crds/tenant.yaml b/deploy/rustfs-operator/crds/tenant.yaml index 888c15c..e35e7eb 100755 --- a/deploy/rustfs-operator/crds/tenant.yaml +++ b/deploy/rustfs-operator/crds/tenant.yaml @@ -1400,6 +1400,45 @@ spec: tls: nullable: true properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object certManager: nullable: true properties: @@ -1506,6 +1545,128 @@ spec: type: string type: array type: object + certificates: + items: + properties: + certManager: + properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object + certificateName: + nullable: true + type: string + commonName: + nullable: true + type: string + dnsNames: + items: + type: string + type: array + duration: + nullable: true + type: string + includeGeneratedDnsNames: + default: true + type: boolean + issuerRef: + nullable: true + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + type: string + name: + type: string + required: + - name + type: object + manageCertificate: + default: false + type: boolean + privateKey: + nullable: true + properties: + algorithm: + nullable: true + type: string + encoding: + nullable: true + type: string + rotationPolicy: + nullable: true + type: string + size: + format: int32 + nullable: true + type: integer + type: object + renewBefore: + nullable: true + type: string + secretName: + nullable: true + type: string + secretType: + nullable: true + type: string + usages: + items: + type: string + type: array + type: object + default: + default: false + type: boolean + hosts: + items: + type: string + type: array + name: + type: string + required: + - certManager + - name + type: object + type: array enableInternodeHttps: default: false type: boolean @@ -1616,6 +1777,67 @@ spec: - kind - name type: object + certificates: + items: + properties: + certificateRef: + nullable: true + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + observedGeneration: + format: int64 + nullable: true + type: integer + ready: + nullable: true + type: boolean + reason: + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + default: + default: false + type: boolean + dnsNames: + items: + type: string + type: array + hosts: + items: + type: string + type: array + managedCertificate: + nullable: true + type: boolean + name: + type: string + serverSecretRef: + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + required: + - name + - serverSecretRef + type: object + type: array clientCaSecretRef: nullable: true properties: @@ -1725,7 +1947,7 @@ spec: description: Status of the condition (True, False, Unknown) type: string type: - description: Type of condition (Ready, Progressing, Degraded) + description: Type of condition (Ready, Reconciling, Degraded) type: string required: - message diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index f8252e9..ec04c6b 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -373,7 +373,9 @@ Important fields: | `rotationStrategy` | `Rollout` is supported. `HotReload` is accepted by the CRD but currently blocks reconciliation. | | `enableInternodeHttps` | Use HTTPS for RustFS peer communication. | | `requireSanMatch` | Require generated DNS names to match certificate SANs. Defaults to `true`. | -| `certManager` | Certificate settings when using cert-manager. `secretName` is required for `mode: certManager`. | +| `certManager` | Backward-compatible single certificate settings. `secretName` is required for `mode: certManager` when `certificates` is empty. | +| `certificates` | Multiple server certificate entries rendered into the RustFS TLS directory for SNI. Exactly one entry must set `default: true`; non-default entries must set `hosts`. | +| `caTrust` | Process-wide RustFS trust settings. This controls `ca.crt`, `client_ca.crt`, `RUSTFS_TRUST_SYSTEM_CA`, and server mTLS; it is not selected per SNI host. | For cert-manager managed certificates: @@ -395,6 +397,45 @@ spec: When `manageCertificate: true`, `issuerRef` is also required. The operator creates or reconciles the cert-manager `Certificate`, waits for the referenced Secret, validates `tls.crt` and `tls.key`, and uses `ca.crt` unless another CA trust source is configured. +For separate public and internal certificates: + +```yaml +spec: + tls: + mode: certManager + rotationStrategy: Rollout + enableInternodeHttps: true + caTrust: + source: CertificateSecretCa + certificates: + - name: internal + default: true + hosts: + - rustfs.internal.example.local + certManager: + manageCertificate: true + secretName: rustfs-internal-tls + issuerRef: + group: cert-manager.io + kind: Issuer + name: private-ca + includeGeneratedDnsNames: true + - name: public + hosts: + - s3.example.com + certManager: + manageCertificate: true + secretName: rustfs-public-tls + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: letsencrypt-prod + includeGeneratedDnsNames: false +``` + +The default certificate is projected to `rustfs_cert.pem` and `rustfs_key.pem` at `mountPath`, so RustFS can use it as the fallback certificate and for internode HTTPS. Each `hosts` value is projected as a RustFS SNI directory, for example `s3.example.com/rustfs_cert.pem` and `s3.example.com/rustfs_key.pem`. +When `certificates` is set, configure process-wide trust with top-level `caTrust` or the `caTrust` on the `default: true` certificate. The legacy `certManager.caTrust` field is only used by the single-certificate form. + ### 7.6 Logging Tenant logging is configured under `spec.logging`. diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index 8f7ee2b..d5d5e2a 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -375,7 +375,9 @@ Tenant TLS 通过 `spec.tls` 配置。 | `rotationStrategy` | 当前支持 `Rollout`。`HotReload` 会被 CRD 接受，但目前会阻塞 reconcile。 | | `enableInternodeHttps` | RustFS 节点间通信是否使用 HTTPS。 | | `requireSanMatch` | 是否要求证书 SAN 匹配生成的 DNS 名称。默认 `true`。 | -| `certManager` | 使用 cert-manager 时的证书配置。`mode: certManager` 必须设置 `secretName`。 | +| `certManager` | 兼容旧版本的单证书配置。`certificates` 为空且 `mode: certManager` 时必须设置 `secretName`。 | +| `certificates` | 多个服务端证书配置，会被渲染到 RustFS TLS 目录供 SNI 使用。必须且只能有一个条目设置 `default: true`，非默认条目必须设置 `hosts`。 | +| `caTrust` | RustFS 进程级信任配置。它控制 `ca.crt`、`client_ca.crt`、`RUSTFS_TRUST_SYSTEM_CA` 和服务端 mTLS，不会按 SNI host 分别生效。 | cert-manager 证书示例： @@ -397,6 +399,45 @@ spec: 当 `manageCertificate: true` 时，`issuerRef` 也是必填项。Operator 会创建或更新 cert-manager `Certificate`，等待引用的 Secret 就绪，校验 `tls.crt` 和 `tls.key`，并在未配置其它 CA trust source 时使用 `ca.crt`。 +公有域名和内部域名使用不同证书时： + +```yaml +spec: + tls: + mode: certManager + rotationStrategy: Rollout + enableInternodeHttps: true + caTrust: + source: CertificateSecretCa + certificates: + - name: internal + default: true + hosts: + - rustfs.internal.example.local + certManager: + manageCertificate: true + secretName: rustfs-internal-tls + issuerRef: + group: cert-manager.io + kind: Issuer + name: private-ca + includeGeneratedDnsNames: true + - name: public + hosts: + - s3.example.com + certManager: + manageCertificate: true + secretName: rustfs-public-tls + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: letsencrypt-prod + includeGeneratedDnsNames: false +``` + +默认条目会被投影到 `mountPath` 根目录下的 `rustfs_cert.pem` 和 `rustfs_key.pem`，供 RustFS 作为 fallback 证书和节点间 HTTPS 证书使用。每个 `hosts` 值会被投影为 RustFS SNI 子目录，例如 `s3.example.com/rustfs_cert.pem` 和 `s3.example.com/rustfs_key.pem`。 +配置了 `certificates` 时，进程级 trust 应放在顶层 `caTrust` 或 `default: true` 证书条目的 `caTrust` 中。旧的 `certManager.caTrust` 只对单证书写法生效。 + ### 7.6 日志配置 Tenant 日志通过 `spec.logging` 配置。 diff --git a/examples/README.md b/examples/README.md index 0215429..9d5aecc 100755 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,7 @@ This directory contains example Tenant configurations for the RustFS Kubernetes | [simple-tenant.yaml](./simple-tenant.yaml) | Documentation Reference | ⭐⭐ Moderate | Configurable | Learning all options | | [secret-credentials-tenant.yaml](./secret-credentials-tenant.yaml) | Secret-based Credentials | ⭐ Simple | Configurable | **Production credential security** | | [provisioning-tenant.yaml](./provisioning-tenant.yaml) | Policy/User/Bucket Provisioning | ⭐⭐ Moderate | 40Gi | Tenant bootstrap automation | +| [multi-cert-tls-tenant.yaml](./multi-cert-tls-tenant.yaml) | Public/Internal TLS | ⭐⭐ Moderate | 40Gi | Separate SNI certificates | | [multi-pool-tenant.yaml](./multi-pool-tenant.yaml) | Multiple Pools | ⭐⭐ Moderate | ~160Gi | Multi-pool setups | | [production-ha-tenant.yaml](./production-ha-tenant.yaml) | Production HA | ⭐⭐⭐ Advanced | 6.4TB | HA with zone distribution | | [cluster-expansion-tenant.yaml](./cluster-expansion-tenant.yaml) | Capacity Expansion | ⭐⭐⭐ Advanced | 384TB | Growing cluster capacity | diff --git a/examples/multi-cert-tls-tenant.yaml b/examples/multi-cert-tls-tenant.yaml new file mode 100644 index 0000000..ceb45ee --- /dev/null +++ b/examples/multi-cert-tls-tenant.yaml @@ -0,0 +1,65 @@ +# RustFS Tenant example for separate internal and public TLS certificates. +# +# This manifest does not create cert-manager Issuers. Before applying it, create +# the private Issuer and public ClusterIssuer referenced below, or adjust the +# issuerRef values to match your cluster. +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: multi-cert-tls + namespace: rustfs-system +spec: + image: rustfs/rustfs:latest + + tls: + mode: certManager + mountPath: /var/run/rustfs/tls + rotationStrategy: Rollout + enableInternodeHttps: true + requireSanMatch: true + caTrust: + source: CertificateSecretCa + trustSystemCa: false + trustLeafCertificateAsCa: false + certificates: + - name: internal + default: true + hosts: + - rustfs.internal.example.local + certManager: + manageCertificate: true + certificateName: multi-cert-tls-internal + secretName: multi-cert-tls-internal + issuerRef: + group: cert-manager.io + kind: Issuer + name: rustfs-private-ca + includeGeneratedDnsNames: true + usages: + - server auth + - name: public + hosts: + - s3.example.com + certManager: + manageCertificate: true + certificateName: multi-cert-tls-public + secretName: multi-cert-tls-public + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: letsencrypt-prod + includeGeneratedDnsNames: false + usages: + - server auth + + pools: + - name: primary + servers: 1 + persistence: + volumesPerServer: 4 + volumeClaimTemplate: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/src/reconcile/tls.rs b/src/reconcile/tls.rs index 47e984b..a19ec76 100644 --- a/src/reconcile/tls.rs +++ b/src/reconcile/tls.rs @@ -17,18 +17,19 @@ use crate::context::{self, Context}; use crate::status::{StatusBuilder, StatusError}; use crate::types::v1alpha1::status::Reason; use crate::types::v1alpha1::status::certificate::{ - CertificateObjectRef, SecretStatusRef, TlsCertificateStatus, + CertificateObjectRef, SecretStatusRef, TlsCertificateStatus, TlsServerCertificateStatus, }; use crate::types::v1alpha1::tenant::Tenant; use crate::types::v1alpha1::tls::{ - CaTrustSource, CertManagerIssuerRef, CertManagerTlsConfig, SecretKeyReference, TlsConfig, - TlsMode, TlsPlan, TlsRotationStrategy, + CaTrustSource, CertManagerIssuerRef, CertManagerTlsConfig, SecretKeyReference, + TlsCertificateConfig, TlsConfig, TlsMode, TlsPlan, TlsRotationStrategy, + TlsServerCertificateMount, }; use k8s_openapi::api::core::v1::Secret; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::api::{Api, Patch, PatchParams}; use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; -use rustls::pki_types::{CertificateDer, ServerName}; +use rustls::pki_types::{CertificateDer, DnsName, ServerName}; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; @@ -79,6 +80,24 @@ struct CertManagerCertificateObservation { message: Option, } +#[derive(Clone, Debug)] +struct TlsCertificateEntry { + name: String, + default: bool, + hosts: Vec, + cert_manager: CertManagerTlsConfig, + legacy: bool, +} + +#[derive(Clone, Debug)] +struct ObservedTlsCertificate { + entry: TlsCertificateEntry, + secret_name: String, + secret: Secret, + certificate_ref: Option, + san_dns_names: Vec, +} + impl CertManagerCertificateObservation { fn status_ref(&self) -> CertificateObjectRef { CertificateObjectRef { @@ -151,189 +170,218 @@ async fn reconcile_cert_manager_tls( namespace: &str, config: &TlsConfig, ) -> Result { - let Some(cert_manager) = config.cert_manager.as_ref() else { - return tls_blocked( - ctx, - tenant, - config, - Reason::CertificateSecretNotFound, - "spec.tls.certManager.secretName is required for certManager TLS mode".to_string(), - ) - .await; + let entries = match certificate_entries(config) { + Ok(entries) => entries, + Err(failure) => return tls_validation_blocked(ctx, tenant, config, failure).await, }; - let Some(secret_name) = cert_manager - .secret_name - .as_deref() - .filter(|name| !name.is_empty()) - else { - return tls_blocked( + if entries + .iter() + .any(|entry| entry.cert_manager.manage_certificate) + && let Err(error) = ensure_cert_manager_certificate_crd(ctx).await + { + return cert_manager_prerequisite_failed( ctx, tenant, config, - Reason::CertificateSecretNotFound, - "spec.tls.certManager.secretName is required for certManager TLS mode".to_string(), + CertManagerPrerequisite::CertificateCrd, + error, + format!( + "cert-manager Certificate CRD '{}' is not installed", + CERT_MANAGER_CERTIFICATE_CRD + ), ) .await; - }; + } - let mut certificate_ref = None; - if cert_manager.manage_certificate { - let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() else { + let mut observed = Vec::with_capacity(entries.len()); + for entry in entries { + let cert_manager = &entry.cert_manager; + let Some(secret_name) = cert_manager + .secret_name + .as_deref() + .filter(|name| !name.is_empty()) + .map(ToString::to_string) + else { return tls_blocked( ctx, tenant, config, - Reason::CertManagerIssuerNotFound, - "spec.tls.certManager.issuerRef is required when manageCertificate=true" - .to_string(), - ) - .await; - }; - let certificate_name = certificate_name(tenant, cert_manager); - - if let Err(error) = ensure_cert_manager_certificate_crd(ctx).await { - return cert_manager_prerequisite_failed( - ctx, - tenant, - config, - CertManagerPrerequisite::CertificateCrd, - error, - format!( - "cert-manager Certificate CRD '{}' is not installed", - CERT_MANAGER_CERTIFICATE_CRD - ), - ) - .await; - } - - if let Err(error) = ensure_cert_manager_issuer(ctx, namespace, issuer_ref).await { - return cert_manager_prerequisite_failed( - ctx, - tenant, - config, - issuer_prerequisite(issuer_ref), - error, + Reason::CertificateSecretNotFound, format!( - "cert-manager {} '{}' was not found", - issuer_ref.kind, issuer_ref.name + "spec.tls certificate '{}' requires certManager.secretName", + entry.name ), ) .await; - } + }; - let desired_certificate = build_cert_manager_certificate( - tenant, - namespace, - config, - cert_manager, - secret_name, - &certificate_name, - ); - let observed_certificate = match apply_cert_manager_certificate( - ctx, - namespace, - &certificate_name, - &desired_certificate, - ) - .await - { - Ok(certificate) => certificate, - Err(error) if context::is_kube_not_found(&error) => { + let mut certificate_ref = None; + if cert_manager.manage_certificate { + let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() else { return tls_blocked( ctx, tenant, config, - Reason::CertManagerCrdMissing, + Reason::CertManagerIssuerNotFound, format!( - "cert-manager Certificate API was not found while applying '{}'", - certificate_name + "spec.tls certificate '{}' requires certManager.issuerRef when manageCertificate=true", + entry.name ), ) .await; - } - Err(error) => { - return tls_blocked( + }; + let certificate_name = certificate_name(tenant, &entry); + + if let Err(error) = ensure_cert_manager_issuer(ctx, namespace, issuer_ref).await { + return cert_manager_prerequisite_failed( ctx, tenant, config, - Reason::CertManagerCertificateApplyFailed, + issuer_prerequisite(issuer_ref), + error, format!( - "failed to apply cert-manager Certificate '{}': {}", - certificate_name, - sanitize_status_message(&error.to_string()) + "cert-manager {} '{}' was not found", + issuer_ref.kind, issuer_ref.name ), ) .await; } - }; - let observation = observe_cert_manager_certificate(&observed_certificate); - certificate_ref = Some(observation.status_ref()); - if !observation.ready { - return tls_pending_with_certificate_ref( - ctx, + + let desired_certificate = build_cert_manager_certificate( tenant, - config, - Reason::CertManagerCertificateNotReady, - certificate_not_ready_message(&certificate_name, &observation), - certificate_ref.clone(), + namespace, + cert_manager, + &entry.hosts, + &secret_name, + &certificate_name, + ); + let observed_certificate = match apply_cert_manager_certificate( + ctx, + namespace, + &certificate_name, + &desired_certificate, ) - .await; + .await + { + Ok(certificate) => certificate, + Err(error) if context::is_kube_not_found(&error) => { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerCrdMissing, + format!( + "cert-manager Certificate API was not found while applying '{}'", + certificate_name + ), + ) + .await; + } + Err(error) => { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerCertificateApplyFailed, + format!( + "failed to apply cert-manager Certificate '{}': {}", + certificate_name, + sanitize_status_message(&error.to_string()) + ), + ) + .await; + } + }; + let observation = observe_cert_manager_certificate(&observed_certificate); + certificate_ref = Some(observation.status_ref()); + if !observation.ready { + return tls_pending_with_certificate_ref( + ctx, + tenant, + config, + Reason::CertManagerCertificateNotReady, + certificate_not_ready_message(&certificate_name, &observation), + certificate_ref.clone(), + ) + .await; + } } - } - let secret = get_server_secret_or_tls_error( - ctx, - tenant, - config, - namespace, - secret_name, - cert_manager.manage_certificate, - certificate_ref.clone(), - ) - .await?; + let secret = get_server_secret_or_tls_error( + ctx, + tenant, + config, + namespace, + &secret_name, + cert_manager.manage_certificate, + certificate_ref.clone(), + ) + .await?; - if let Err(failure) = validate_tls_secret_type( - &secret, - secret_name, - cert_manager - .secret_type - .as_deref() - .filter(|secret_type| !secret_type.is_empty()), - ) { - return tls_validation_blocked(ctx, tenant, config, failure).await; - } + if let Err(failure) = validate_tls_secret_type( + &secret, + &secret_name, + cert_manager + .secret_type + .as_deref() + .filter(|secret_type| !secret_type.is_empty()), + ) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } - let cert_bytes = require_secret_key( - ctx, - tenant, - config, - &secret, - secret_name, - TLS_CERT_KEY, - Reason::CertificateSecretMissingKey, - ) - .await?; - require_secret_key( - ctx, - tenant, - config, - &secret, - secret_name, - TLS_KEY_KEY, - Reason::CertificateSecretMissingKey, - ) - .await?; + let cert_bytes = require_secret_key( + ctx, + tenant, + config, + &secret, + &secret_name, + TLS_CERT_KEY, + Reason::CertificateSecretMissingKey, + ) + .await?; + require_secret_key( + ctx, + tenant, + config, + &secret, + &secret_name, + TLS_KEY_KEY, + Reason::CertificateSecretMissingKey, + ) + .await?; - if config.require_san_match && config.enable_internode_https { - let expected_dns_names = certificate_dns_names(tenant, namespace, cert_manager); - if let Err(failure) = - validate_tls_secret_san_match(secret_name, &cert_bytes, &expected_dns_names) + let san_dns_names = san_validation_dns_names(tenant, namespace, config, &entry); + if config.require_san_match + && let Err(failure) = + validate_tls_secret_san_match(&secret_name, &cert_bytes, &san_dns_names) { return tls_validation_blocked(ctx, tenant, config, failure).await; } + + observed.push(ObservedTlsCertificate { + entry, + secret_name, + secret, + certificate_ref, + san_dns_names, + }); } + let Some(default_certificate) = observed + .iter() + .find(|certificate| certificate.entry.default) + else { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertificateInvalid, + "spec.tls.certificates must contain exactly one default=true certificate".to_string(), + ) + .await; + }; + let ca_trust = config.ca_trust(); let trust_system_ca = ca_trust.trust_system_ca || ca_trust.source == CaTrustSource::SystemCa; let mut server_ca_key = None; @@ -343,8 +391,8 @@ async fn reconcile_cert_manager_tls( match ca_trust.source { CaTrustSource::CertificateSecretCa => match certificate_secret_ca_material( - &secret, - secret_name, + &default_certificate.secret, + &default_certificate.secret_name, config.enable_internode_https, trust_system_ca, ) { @@ -442,7 +490,7 @@ async fn reconcile_cert_manager_tls( let hash = tls_hash( config, - &secret, + &observed, explicit_ca.as_ref(), explicit_ca_bytes.as_deref(), client_ca.as_ref(), @@ -451,19 +499,28 @@ async fn reconcile_cert_manager_tls( ); let status = cert_manager_tls_status( config, - secret_name, - &secret, + &observed, explicit_ca.as_ref().zip(explicit_ca_secret.as_ref()), client_ca.as_ref().zip(client_ca_secret.as_ref()), &hash, - certificate_ref, ); + let server_certificates = observed + .iter() + .map(|certificate| TlsServerCertificateMount { + secret_name: certificate.secret_name.clone(), + domains: rustfs_certificate_domains(certificate), + ca_key: certificate + .entry + .default + .then(|| server_ca_key.clone()) + .flatten(), + }) + .collect(); - Ok(TlsPlan::rollout( + Ok(TlsPlan::rollout_certificates( config.mount_path.clone(), hash, - secret_name.to_string(), - server_ca_key, + server_certificates, explicit_ca, client_ca, config.enable_internode_https, @@ -516,6 +573,143 @@ async fn get_server_secret_or_tls_error( } } +fn certificate_entries( + config: &TlsConfig, +) -> Result, TlsValidationFailure> { + let entries = if config.certificates.is_empty() { + let Some(cert_manager) = config.cert_manager.clone() else { + return Err(TlsValidationFailure { + reason: Reason::CertificateSecretNotFound, + message: "spec.tls.certManager.secretName or spec.tls.certificates is required for certManager TLS mode".to_string(), + }); + }; + vec![TlsCertificateEntry { + name: "default".to_string(), + default: true, + hosts: Vec::new(), + cert_manager, + legacy: true, + }] + } else { + config.certificates.iter().map(certificate_entry).collect() + }; + + validate_certificate_entries(entries) +} + +fn certificate_entry(config: &TlsCertificateConfig) -> TlsCertificateEntry { + TlsCertificateEntry { + name: config.name.clone(), + default: config.default, + hosts: config.hosts.clone(), + cert_manager: config.cert_manager.clone(), + legacy: false, + } +} + +fn validate_certificate_entries( + entries: Vec, +) -> Result, TlsValidationFailure> { + if entries.is_empty() { + return Err(TlsValidationFailure { + reason: Reason::CertificateSecretNotFound, + message: "spec.tls.certificates must contain at least one certificate".to_string(), + }); + } + + let mut names = BTreeSet::new(); + let mut hosts = BTreeSet::new(); + let mut default_count = 0; + for entry in &entries { + validate_certificate_entry_name(&entry.name)?; + if !names.insert(entry.name.clone()) { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls.certificates contains duplicate name '{}'", + entry.name + ), + }); + } + if entry.default { + default_count += 1; + } else if entry.hosts.is_empty() { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate '{}' must set hosts unless default=true", + entry.name + ), + }); + } + for host in &entry.hosts { + validate_rustfs_sni_host(host)?; + if !hosts.insert(host.clone()) { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!("spec.tls.certificates contains duplicate host '{}'", host), + }); + } + } + } + + if default_count != 1 { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: "spec.tls.certificates must contain exactly one default=true certificate" + .to_string(), + }); + } + + Ok(entries) +} + +fn validate_certificate_entry_name(name: &str) -> Result<(), TlsValidationFailure> { + let valid = !name.is_empty() + && name.len() <= 63 + && name + .chars() + .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '-') + && name + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && name + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric); + if valid { + return Ok(()); + } + Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate name '{}' must be a DNS label (lowercase alphanumeric or '-', 1-63 chars)", + name + ), + }) +} + +fn validate_rustfs_sni_host(host: &str) -> Result<(), TlsValidationFailure> { + let valid_path = !host.is_empty() + && host != "." + && host != ".." + && !host.starts_with('.') + && !host.contains('/') + && !host.contains('\\') + && !host.contains('*'); + if valid_path && DnsName::try_from(host).is_ok() { + return Ok(()); + } + Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate host '{}' must be a concrete DNS name usable as a RustFS SNI directory", + host + ), + }) +} + async fn ensure_cert_manager_certificate_crd(ctx: &Context) -> Result<(), context::Error> { let api: Api = Api::all(ctx.client.clone()); api.get(CERT_MANAGER_CERTIFICATE_CRD) @@ -590,8 +784,8 @@ async fn cert_manager_prerequisite_failed( fn build_cert_manager_certificate( tenant: &Tenant, namespace: &str, - _config: &TlsConfig, cert_manager: &CertManagerTlsConfig, + hosts: &[String], secret_name: &str, certificate_name: &str, ) -> DynamicObject { @@ -609,7 +803,12 @@ fn build_cert_manager_certificate( } spec.insert( "dnsNames".to_string(), - json!(certificate_dns_names(tenant, namespace, cert_manager)), + json!(certificate_dns_names( + tenant, + namespace, + cert_manager, + hosts + )), ); spec.insert( "usages".to_string(), @@ -768,13 +967,20 @@ fn missing_cert_manager_prerequisite_reason(prerequisite: CertManagerPrerequisit } } -fn certificate_name(tenant: &Tenant, cert_manager: &CertManagerTlsConfig) -> String { - cert_manager +fn certificate_name(tenant: &Tenant, entry: &TlsCertificateEntry) -> String { + entry + .cert_manager .certificate_name .as_deref() .filter(|name| !name.is_empty()) .map(ToString::to_string) - .unwrap_or_else(|| format!("{}-server-tls", tenant.name())) + .unwrap_or_else(|| { + if entry.legacy { + format!("{}-server-tls", tenant.name()) + } else { + format!("{}-{}-tls", tenant.name(), entry.name) + } + }) } fn issuer_ref_value(issuer_ref: &CertManagerIssuerRef) -> Value { @@ -789,8 +995,10 @@ fn certificate_dns_names( tenant: &Tenant, namespace: &str, cert_manager: &CertManagerTlsConfig, + hosts: &[String], ) -> Vec { let mut names = BTreeSet::new(); + names.extend(hosts.iter().filter(|name| !name.is_empty()).cloned()); names.extend( cert_manager .dns_names @@ -818,6 +1026,57 @@ fn certificate_dns_names( names.into_iter().collect() } +fn san_validation_dns_names( + tenant: &Tenant, + namespace: &str, + config: &TlsConfig, + entry: &TlsCertificateEntry, +) -> Vec { + let mut names = BTreeSet::new(); + names.extend(entry.hosts.iter().filter(|name| !name.is_empty()).cloned()); + names.extend( + entry + .cert_manager + .dns_names + .iter() + .filter(|name| !name.is_empty()) + .cloned(), + ); + if entry.default && config.enable_internode_https { + names.extend(generated_dns_names(tenant, namespace)); + } + names.into_iter().collect() +} + +fn generated_dns_names(tenant: &Tenant, namespace: &str) -> Vec { + let mut names = BTreeSet::new(); + let tenant_name = tenant.name(); + let io_service = format!("{tenant_name}-io"); + let headless_service = tenant.headless_service_name(); + names.insert(format!("{io_service}.{namespace}.svc")); + names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(format!("{headless_service}.{namespace}.svc")); + names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + for pool in &tenant.spec.pools { + for ordinal in 0..pool.servers.max(0) { + names.insert(format!( + "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", + pool.name + )); + } + } + names.into_iter().collect() +} + +fn rustfs_certificate_domains(certificate: &ObservedTlsCertificate) -> Vec> { + let mut domains = Vec::new(); + if certificate.entry.default { + domains.push(None); + } + domains.extend(certificate.entry.hosts.iter().cloned().map(Some)); + domains +} + fn certificate_usages(cert_manager: &CertManagerTlsConfig) -> Vec { if cert_manager.usages.is_empty() { vec!["server auth".to_string()] @@ -1182,41 +1441,57 @@ async fn patch_tls_error_with_certificate_ref( fn cert_manager_tls_status( config: &TlsConfig, - secret_name: &str, - secret: &Secret, + certificates: &[ObservedTlsCertificate], explicit_ca: Option<(&SecretKeyReference, &Secret)>, client_ca: Option<(&SecretKeyReference, &Secret)>, hash: &str, - certificate_ref: Option, ) -> TlsCertificateStatus { let ca_trust = config.ca_trust(); + let Some(default_certificate) = certificates + .iter() + .find(|certificate| certificate.entry.default) + .or_else(|| certificates.first()) + else { + return TlsCertificateStatus { + mode: tls_mode_name(config.mode).to_string(), + ready: false, + rotation_strategy: Some(rotation_strategy_name(config.rotation_strategy).to_string()), + mount_path: Some(config.mount_path.clone()), + trust_source: Some(ca_trust_source_name(ca_trust.source).to_string()), + last_error_reason: Some(Reason::CertificateInvalid.as_str().to_string()), + last_error_message: Some("no TLS server certificates were observed".to_string()), + ..Default::default() + }; + }; + let server_secret_ref = SecretStatusRef { + name: default_certificate.secret_name.clone(), + key: None, + resource_version: default_certificate.secret.metadata.resource_version.clone(), + }; TlsCertificateStatus { mode: tls_mode_name(config.mode).to_string(), ready: true, - managed_certificate: config - .cert_manager - .as_ref() - .map(|cert_manager| cert_manager.manage_certificate), + managed_certificate: Some(default_certificate.entry.cert_manager.manage_certificate), rotation_strategy: Some(rotation_strategy_name(config.rotation_strategy).to_string()), mount_path: Some(config.mount_path.clone()), - certificate_ref, - server_secret_ref: Some(SecretStatusRef { - name: secret_name.to_string(), - key: None, - resource_version: secret.metadata.resource_version.clone(), - }), - ca_secret_ref: ca_status_ref(secret_name, secret, explicit_ca), + certificate_ref: default_certificate.certificate_ref.clone(), + server_secret_ref: Some(server_secret_ref.clone()), + certificates: certificates + .iter() + .map(tls_server_certificate_status) + .collect(), + ca_secret_ref: ca_status_ref( + &default_certificate.secret_name, + &default_certificate.secret, + explicit_ca, + ), client_ca_secret_ref: client_ca.map(|(secret_ref, ca_secret)| SecretStatusRef { name: secret_ref.name.clone(), key: Some(secret_ref.key.clone()), resource_version: ca_secret.metadata.resource_version.clone(), }), observed_hash: Some(hash.to_string()), - dns_names: config - .cert_manager - .as_ref() - .map(|cert_manager| cert_manager.dns_names.clone()) - .unwrap_or_default(), + dns_names: default_certificate.san_dns_names.clone(), trust_source: Some(ca_trust_source_name(ca_trust.source).to_string()), last_validated_time: Some( chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), @@ -1225,6 +1500,24 @@ fn cert_manager_tls_status( } } +fn tls_server_certificate_status( + certificate: &ObservedTlsCertificate, +) -> TlsServerCertificateStatus { + TlsServerCertificateStatus { + name: certificate.entry.name.clone(), + default: certificate.entry.default, + hosts: certificate.entry.hosts.clone(), + managed_certificate: Some(certificate.entry.cert_manager.manage_certificate), + certificate_ref: certificate.certificate_ref.clone(), + server_secret_ref: SecretStatusRef { + name: certificate.secret_name.clone(), + key: None, + resource_version: certificate.secret.metadata.resource_version.clone(), + }, + dns_names: certificate.san_dns_names.clone(), + } +} + fn ca_status_ref( secret_name: &str, secret: &Secret, @@ -1274,7 +1567,7 @@ fn error_tls_status_with_certificate_ref( fn tls_hash( config: &TlsConfig, - secret: &Secret, + certificates: &[ObservedTlsCertificate], explicit_ca: Option<&SecretKeyReference>, explicit_ca_bytes: Option<&[u8]>, client_ca: Option<&SecretKeyReference>, @@ -1302,17 +1595,44 @@ fn tls_hash( "trustSystemCa", if trust_system_ca { "true" } else { "false" }, ); - hash_str( - &mut hasher, - "serverSecret.resourceVersion", - secret.metadata.resource_version.as_deref().unwrap_or(""), - ); - hash_bytes(&mut hasher, "tls.crt", secret_bytes(secret, TLS_CERT_KEY)); - hash_bytes( - &mut hasher, - "secret.ca.crt", - secret_bytes(secret, CA_CERT_KEY), - ); + for certificate in certificates { + hash_str(&mut hasher, "certificate.name", &certificate.entry.name); + hash_str( + &mut hasher, + "certificate.default", + if certificate.entry.default { + "true" + } else { + "false" + }, + ); + for host in &certificate.entry.hosts { + hash_str(&mut hasher, "certificate.host", host); + } + hash_str(&mut hasher, "serverSecret.name", &certificate.secret_name); + hash_str( + &mut hasher, + "serverSecret.resourceVersion", + certificate + .secret + .metadata + .resource_version + .as_deref() + .unwrap_or(""), + ); + hash_bytes( + &mut hasher, + "tls.crt", + secret_bytes(&certificate.secret, TLS_CERT_KEY), + ); + if certificate.entry.default { + hash_bytes( + &mut hasher, + "secret.ca.crt", + secret_bytes(&certificate.secret, CA_CERT_KEY), + ); + } + } if let Some(secret_ref) = explicit_ca { hash_str(&mut hasher, "explicitCa.name", &secret_ref.name); hash_str(&mut hasher, "explicitCa.key", &secret_ref.key); @@ -1382,7 +1702,7 @@ const fn ca_trust_source_name(source: CaTrustSource) -> &'static str { mod tests { use super::*; use crate::types::v1alpha1::tls::{ - CaTrustConfig, CertManagerPrivateKeyConfig, CertManagerTlsConfig, + CaTrustConfig, CertManagerPrivateKeyConfig, CertManagerTlsConfig, TlsCertificateConfig, }; use k8s_openapi::ByteString; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; @@ -1635,19 +1955,56 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj Some(b"-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----\n"), ); - let baseline = tls_hash(&config, &first, None, None, None, None, false); + let baseline_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + first, + CertManagerTlsConfig::default(), + vec![], + )]; + let private_key_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + changed_private_key, + CertManagerTlsConfig::default(), + vec![], + )]; + let resource_version_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + changed_resource_version, + CertManagerTlsConfig::default(), + vec![], + )]; + let ca_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + changed_ca, + CertManagerTlsConfig::default(), + vec![], + )]; + + let baseline = tls_hash(&config, &baseline_certs, None, None, None, None, false); let private_key_changed = - tls_hash(&config, &changed_private_key, None, None, None, None, false); + tls_hash(&config, &private_key_certs, None, None, None, None, false); let resource_version_changed = tls_hash( &config, - &changed_resource_version, + &resource_version_certs, None, None, None, None, false, ); - let ca_changed = tls_hash(&config, &changed_ca, None, None, None, None, false); + let ca_changed = tls_hash(&config, &ca_certs, None, None, None, None, false); assert_eq!(baseline, private_key_changed); assert_ne!(baseline, resource_version_changed); @@ -1742,14 +2099,21 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj Some(PUBLIC_CERT_PEM), ); + let observed = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + server, + config.cert_manager.as_ref().unwrap().clone(), + vec![], + )]; let status = cert_manager_tls_status( &config, - "server-tls", - &server, + &observed, Some((&secret_ref("server-ca", "ca.crt"), &ca)), Some((&secret_ref("client-ca", "client_ca.crt"), &client_ca)), "sha256:test", - None, ); assert_eq!( @@ -1806,8 +2170,8 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj let certificate = build_cert_manager_certificate( &tenant, "storage", - &config, cert_manager, + &[], "tenant-a-server-tls", "tenant-a-server", ); @@ -1883,6 +2247,209 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj ); } + #[test] + fn managed_certificate_manifest_includes_sni_hosts() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + let cert_manager = CertManagerTlsConfig { + secret_name: Some("tenant-a-public-tls".to_string()), + dns_names: vec!["custom.example.com".to_string()], + include_generated_dns_names: false, + ..Default::default() + }; + + let certificate = build_cert_manager_certificate( + &tenant, + "storage", + &cert_manager, + &["s3.example.com".to_string()], + "tenant-a-public-tls", + "tenant-a-public-tls", + ); + let dns_names = certificate + .data + .pointer("/spec/dnsNames") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect::>() + }); + + assert_eq!( + dns_names, + Some(vec![ + "custom.example.com".to_string(), + "s3.example.com".to_string() + ]) + ); + } + + #[test] + fn multi_certificate_entries_validate_default_and_hosts() { + let valid = TlsConfig { + mode: TlsMode::CertManager, + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig { + secret_name: Some("internal-tls".to_string()), + ..Default::default() + }, + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + secret_name: Some("public-tls".to_string()), + ..Default::default() + }, + }, + ], + ..Default::default() + }; + + let entries = certificate_entries(&valid).expect("valid multi-cert config should pass"); + + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].name, "internal"); + assert!(entries[0].default); + assert_eq!(entries[1].hosts, vec!["s3.example.com".to_string()]); + + let ip_host = TlsConfig { + mode: TlsMode::CertManager, + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["127.0.0.1".to_string()], + cert_manager: CertManagerTlsConfig::default(), + }, + ], + ..Default::default() + }; + let failure = certificate_entries(&ip_host).expect_err("SNI host should reject IP values"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!(failure.message.contains("concrete DNS name")); + + let missing_host = TlsConfig { + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + ], + ..valid + }; + let failure = + certificate_entries(&missing_host).expect_err("non-default cert requires hosts"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!(failure.message.contains("must set hosts")); + } + + #[test] + fn multi_certificate_status_keeps_default_fields_and_lists_all_certs() { + let config = TlsConfig { + mode: TlsMode::CertManager, + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig { + secret_name: Some("internal-tls".to_string()), + ..Default::default() + }, + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + secret_name: Some("public-tls".to_string()), + ..Default::default() + }, + }, + ], + ..Default::default() + }; + let observed = vec![ + observed_tls_certificate( + "internal", + true, + vec![], + "internal-tls", + tls_secret( + "internal-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + Some(PUBLIC_CERT_PEM), + ), + config.certificates[0].cert_manager.clone(), + vec!["tenant-a-io.storage.svc"], + ), + observed_tls_certificate( + "public", + false, + vec!["s3.example.com"], + "public-tls", + tls_secret( + "public-tls", + "9", + Some("kubernetes.io/tls"), + true, + true, + None, + ), + config.certificates[1].cert_manager.clone(), + vec!["s3.example.com"], + ), + ]; + + let status = cert_manager_tls_status(&config, &observed, None, None, "sha256:multi"); + + assert_eq!( + status + .server_secret_ref + .as_ref() + .map(|secret| secret.name.as_str()), + Some("internal-tls") + ); + assert_eq!(status.certificates.len(), 2); + assert!(status.certificates[0].default); + assert_eq!(status.certificates[1].hosts, vec!["s3.example.com"]); + assert_eq!( + status.certificates[1] + .server_secret_ref + .resource_version + .as_deref(), + Some("9") + ); + } + #[test] fn certificate_observation_requires_ready_condition_for_current_generation() { let ready = certificate_object( @@ -2036,6 +2603,30 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj } } + fn observed_tls_certificate( + name: &str, + default: bool, + hosts: Vec<&str>, + secret_name: &str, + secret: Secret, + cert_manager: CertManagerTlsConfig, + san_dns_names: Vec<&str>, + ) -> ObservedTlsCertificate { + ObservedTlsCertificate { + entry: TlsCertificateEntry { + name: name.to_string(), + default, + hosts: hosts.into_iter().map(ToString::to_string).collect(), + cert_manager, + legacy: false, + }, + secret_name: secret_name.to_string(), + secret, + certificate_ref: None, + san_dns_names: san_dns_names.into_iter().map(ToString::to_string).collect(), + } + } + fn certificate_object(name: &str, generation: Option, data: Value) -> DynamicObject { let mut object = DynamicObject::new(name, &certificate_api_resource()).data(data); object.metadata.generation = generation; diff --git a/src/types/v1alpha1/status/certificate.rs b/src/types/v1alpha1/status/certificate.rs index eb62f18..836fac0 100755 --- a/src/types/v1alpha1/status/certificate.rs +++ b/src/types/v1alpha1/status/certificate.rs @@ -50,6 +50,9 @@ pub struct TlsCertificateStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub server_secret_ref: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub certificates: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] pub ca_secret_ref: Option, @@ -93,6 +96,29 @@ pub struct TlsCertificateStatus { pub last_error_message: Option, } +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TlsServerCertificateStatus { + pub name: String, + + #[serde(default)] + pub default: bool, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hosts: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub managed_certificate: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub certificate_ref: Option, + + pub server_secret_ref: SecretStatusRef, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dns_names: Vec, +} + #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub struct CertificateObjectRef { diff --git a/src/types/v1alpha1/tenant/workloads.rs b/src/types/v1alpha1/tenant/workloads.rs index 89f9a85..43ad03e 100755 --- a/src/types/v1alpha1/tenant/workloads.rs +++ b/src/types/v1alpha1/tenant/workloads.rs @@ -1052,6 +1052,29 @@ mod tests { .iter() .any(|volume| volume.name == "rustfs-tls-server") ); + let server_volume = volumes + .iter() + .find(|volume| volume.name == "rustfs-tls-server") + .expect("TLS server volume should exist"); + let projected_items = server_volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .expect("TLS server volume should be projected") + .iter() + .flat_map(|source| { + source + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .into_iter() + .flatten() + }) + .map(|item| (item.key.as_str(), item.path.as_str())) + .collect::>(); + assert!(projected_items.contains(&("tls.crt", "rustfs_cert.pem"))); + assert!(projected_items.contains(&("tls.key", "rustfs_key.pem"))); + assert!(projected_items.contains(&("ca.crt", "ca.crt"))); let container = &pod_spec.containers[0]; let env = container.env.as_ref().expect("TLS env should be present"); @@ -1072,18 +1095,8 @@ mod tests { .expect("TLS volume mounts should be present"); assert!(mounts.iter().any(|mount| { mount.name == "rustfs-tls-server" - && mount.mount_path == "/var/run/rustfs/tls/rustfs_cert.pem" - && mount.sub_path.as_deref() == Some("rustfs_cert.pem") - })); - assert!(mounts.iter().any(|mount| { - mount.name == "rustfs-tls-server" - && mount.mount_path == "/var/run/rustfs/tls/rustfs_key.pem" - && mount.sub_path.as_deref() == Some("rustfs_key.pem") - })); - assert!(mounts.iter().any(|mount| { - mount.name == "rustfs-tls-server" - && mount.mount_path == "/var/run/rustfs/tls/ca.crt" - && mount.sub_path.as_deref() == Some("ca.crt") + && mount.mount_path == "/var/run/rustfs/tls" + && mount.sub_path.is_none() })); assert_eq!( diff --git a/src/types/v1alpha1/tls.rs b/src/types/v1alpha1/tls.rs index d8c603d..ab368d1 100644 --- a/src/types/v1alpha1/tls.rs +++ b/src/types/v1alpha1/tls.rs @@ -166,6 +166,20 @@ pub struct CertManagerTlsConfig { pub ca_trust: Option, } +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TlsCertificateConfig { + pub name: String, + + #[serde(default)] + pub default: bool, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hosts: Vec, + + pub cert_manager: CertManagerTlsConfig, +} + fn default_include_generated_dns_names() -> bool { true } @@ -210,6 +224,12 @@ pub struct TlsConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub cert_manager: Option, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub certificates: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_trust: Option, } fn default_tls_mount_path() -> String { @@ -229,6 +249,8 @@ impl Default for TlsConfig { enable_internode_https: false, require_san_match: default_require_san_match(), cert_manager: None, + certificates: Vec::new(), + ca_trust: None, } } } @@ -239,9 +261,22 @@ impl TlsConfig { } pub fn ca_trust(&self) -> CaTrustConfig { - self.cert_manager - .as_ref() - .and_then(|cert_manager| cert_manager.ca_trust.clone()) + if let Some(ca_trust) = self.ca_trust.clone() { + return ca_trust; + } + + if self.certificates.is_empty() { + return self + .cert_manager + .as_ref() + .and_then(|cert_manager| cert_manager.ca_trust.clone()) + .unwrap_or_default(); + } + + self.certificates + .iter() + .find(|certificate| certificate.default) + .and_then(|certificate| certificate.cert_manager.ca_trust.clone()) .unwrap_or_default() } } @@ -260,7 +295,120 @@ mod tests { assert!(!config.enable_internode_https); assert!(config.require_san_match); assert!(config.cert_manager.is_none()); + assert!(config.certificates.is_empty()); + assert!(config.ca_trust.is_none()); } + + #[test] + fn multi_certificate_ca_trust_ignores_legacy_cert_manager() { + let config = TlsConfig { + cert_manager: Some(CertManagerTlsConfig { + ca_trust: Some(CaTrustConfig { + source: CaTrustSource::SystemCa, + ..Default::default() + }), + ..Default::default() + }), + certificates: vec![TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig { + ca_trust: Some(CaTrustConfig { + source: CaTrustSource::SecretRef, + ca_secret_ref: Some(SecretKeyReference { + name: "internal-ca".to_string(), + key: "bundle.pem".to_string(), + }), + ..Default::default() + }), + ..Default::default() + }, + }], + ..Default::default() + }; + + let ca_trust = config.ca_trust(); + + assert_eq!(ca_trust.source, CaTrustSource::SecretRef); + assert_eq!( + ca_trust + .ca_secret_ref + .as_ref() + .map(|secret| { (secret.name.as_str(), secret.key.as_str()) }), + Some(("internal-ca", "bundle.pem")) + ); + } + + #[test] + fn tls_plan_projects_multiple_certificates_into_rustfs_sni_layout() { + let plan = TlsPlan::rollout_certificates( + DEFAULT_TLS_MOUNT_PATH.to_string(), + "sha256:multi".to_string(), + vec![ + TlsServerCertificateMount { + secret_name: "internal-tls".to_string(), + domains: vec![None, Some("rustfs.internal.example.local".to_string())], + ca_key: Some("ca.crt".to_string()), + }, + TlsServerCertificateMount { + secret_name: "public-tls".to_string(), + domains: vec![Some("s3.example.com".to_string())], + ca_key: None, + }, + ], + None, + None, + true, + false, + false, + None, + ); + + let volume = plan + .volumes + .iter() + .find(|volume| volume.name == "rustfs-tls-server") + .expect("TLS volume should exist"); + let paths = volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .expect("TLS volume should use projected sources") + .iter() + .flat_map(|source| { + source + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .into_iter() + .flatten() + }) + .map(|item| item.path.as_str()) + .collect::>(); + + assert!(paths.contains(&"rustfs_cert.pem")); + assert!(paths.contains(&"rustfs_key.pem")); + assert!(paths.contains(&"ca.crt")); + assert!(paths.contains(&"rustfs.internal.example.local/rustfs_cert.pem")); + assert!(paths.contains(&"rustfs.internal.example.local/rustfs_key.pem")); + assert!(paths.contains(&"s3.example.com/rustfs_cert.pem")); + assert!(paths.contains(&"s3.example.com/rustfs_key.pem")); + assert_eq!( + plan.volume_mounts + .iter() + .find(|mount| mount.name == "rustfs-tls-server") + .map(|mount| (mount.mount_path.as_str(), mount.sub_path.as_deref())), + Some((DEFAULT_TLS_MOUNT_PATH, None)) + ); + } +} + +#[derive(Clone, Debug, Default)] +pub struct TlsServerCertificateMount { + pub secret_name: String, + pub domains: Vec>, + pub ca_key: Option, } #[derive(Clone, Debug, Default)] @@ -303,6 +451,35 @@ impl TlsPlan { trust_system_ca: bool, trust_leaf_certificate_as_ca: bool, status: Option, + ) -> Self { + Self::rollout_certificates( + mount_path, + hash, + vec![TlsServerCertificateMount { + secret_name: server_secret_name, + domains: vec![None], + ca_key: server_ca_key, + }], + explicit_ca, + client_ca, + enable_internode_https, + trust_system_ca, + trust_leaf_certificate_as_ca, + status, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn rollout_certificates( + mount_path: String, + hash: String, + server_certificates: Vec, + explicit_ca: Option, + client_ca: Option, + enable_internode_https: bool, + trust_system_ca: bool, + trust_leaf_certificate_as_ca: bool, + status: Option, ) -> Self { let mut annotations = BTreeMap::new(); annotations.insert(TLS_HASH_ANNOTATION.to_string(), hash); @@ -334,63 +511,32 @@ impl TlsPlan { }); } - let has_server_ca = server_ca_key.is_some(); - let mut server_items = vec![ - key_to_path("tls.crt", RUSTFS_TLS_CERT_FILE), - key_to_path("tls.key", RUSTFS_TLS_KEY_FILE), - ]; - if let Some(ca_key) = server_ca_key.as_deref() { - server_items.push(key_to_path(ca_key, RUSTFS_CA_FILE)); + let mut sources = server_certificates + .iter() + .map(|certificate| { + secret_projection( + &certificate.secret_name, + server_certificate_items(certificate), + ) + }) + .collect::>(); + + if let Some(explicit_ca) = &explicit_ca { + sources.push(secret_projection( + &explicit_ca.name, + vec![key_to_path(&explicit_ca.key, RUSTFS_CA_FILE)], + )); } - - let (mut volumes, mut volume_mounts) = if let Some(explicit_ca) = &explicit_ca { - ( - vec![projected_volume( - "rustfs-tls-server", - vec![ - secret_projection(&server_secret_name, server_items), - secret_projection( - &explicit_ca.name, - vec![key_to_path(&explicit_ca.key, RUSTFS_CA_FILE)], - ), - ], - )], - vec![directory_mount("rustfs-tls-server", &mount_path)], - ) - } else { - let mut volume_mounts = vec![ - file_mount("rustfs-tls-server", &mount_path, RUSTFS_TLS_CERT_FILE), - file_mount("rustfs-tls-server", &mount_path, RUSTFS_TLS_KEY_FILE), - ]; - if has_server_ca { - volume_mounts.push(file_mount("rustfs-tls-server", &mount_path, RUSTFS_CA_FILE)); - } - ( - vec![secret_volume( - "rustfs-tls-server", - &server_secret_name, - server_items, - )], - volume_mounts, - ) - }; - if let Some(client_ca) = &client_ca { - volumes.push(secret_volume( - "rustfs-tls-client-ca", + sources.push(secret_projection( &client_ca.name, vec![key_to_path(&client_ca.key, RUSTFS_CLIENT_CA_FILE)], )); - volume_mounts.push(file_mount( - "rustfs-tls-client-ca", - &mount_path, - RUSTFS_CLIENT_CA_FILE, - )); } Self { enabled: true, - mount_path, + mount_path: mount_path.clone(), internode_scheme: if enable_internode_https { "https" } else { @@ -399,8 +545,8 @@ impl TlsPlan { probe_scheme: "HTTPS", pod_template_annotations: annotations, env, - volumes, - volume_mounts, + volumes: vec![projected_volume("rustfs-tls-server", sources)], + volume_mounts: vec![directory_mount("rustfs-tls-server", &mount_path)], status, } } @@ -422,17 +568,31 @@ impl TlsPlan { } } -fn secret_volume(name: &str, secret_name: &str, items: Vec) -> corev1::Volume { - corev1::Volume { - name: name.to_string(), - secret: Some(corev1::SecretVolumeSource { - secret_name: Some(secret_name.to_string()), - items: Some(items), - optional: Some(false), - ..Default::default() - }), - ..Default::default() +fn server_certificate_items(certificate: &TlsServerCertificateMount) -> Vec { + let mut items = Vec::new(); + for domain in &certificate.domains { + items.push(key_to_path("tls.crt", &server_cert_path(domain.as_deref()))); + items.push(key_to_path("tls.key", &server_key_path(domain.as_deref()))); + } + if let Some(ca_key) = certificate.ca_key.as_deref() { + items.push(key_to_path(ca_key, RUSTFS_CA_FILE)); } + items +} + +fn server_cert_path(domain: Option<&str>) -> String { + rustfs_tls_path(domain, RUSTFS_TLS_CERT_FILE) +} + +fn server_key_path(domain: Option<&str>) -> String { + rustfs_tls_path(domain, RUSTFS_TLS_KEY_FILE) +} + +fn rustfs_tls_path(domain: Option<&str>, file: &str) -> String { + domain + .filter(|domain| !domain.is_empty()) + .map(|domain| format!("{domain}/{file}")) + .unwrap_or_else(|| file.to_string()) } fn projected_volume(name: &str, sources: Vec) -> corev1::Volume { @@ -465,16 +625,6 @@ fn key_to_path(key: &str, path: &str) -> corev1::KeyToPath { } } -fn file_mount(volume_name: &str, mount_path: &str, file_name: &str) -> corev1::VolumeMount { - corev1::VolumeMount { - name: volume_name.to_string(), - mount_path: format!("{}/{}", mount_path.trim_end_matches('/'), file_name), - sub_path: Some(file_name.to_string()), - read_only: Some(true), - ..Default::default() - } -} - fn directory_mount(volume_name: &str, mount_path: &str) -> corev1::VolumeMount { corev1::VolumeMount { name: volume_name.to_string(), From 08f9642c3f0140f1d7c1b50fb63804d708be27cd Mon Sep 17 00:00:00 2001 From: GatewayJ <18332154+GatewayJ@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:15:48 +0800 Subject: [PATCH 2/2] fix(tls): align multi-cert DNS validation --- deploy/k8s-dev/operator-rbac.yaml | 2 +- deploy/rustfs-operator/crds/tenant-crd.yaml | 4 +- deploy/rustfs-operator/crds/tenant.yaml | 4 +- .../templates/clusterrole.yaml | 2 +- docs/operator-user-guide.md | 5 +- docs/operator-user-guide.zh-CN.md | 5 +- e2e/src/framework/cert_manager_tls.rs | 2 +- e2e/tests/cert_manager_tls.rs | 2 +- src/reconcile/tls.rs | 448 +++++++++++++++++- src/types/v1alpha1/tls.rs | 30 +- 10 files changed, 448 insertions(+), 56 deletions(-) diff --git a/deploy/k8s-dev/operator-rbac.yaml b/deploy/k8s-dev/operator-rbac.yaml index e6a34d7..af447b2 100755 --- a/deploy/k8s-dev/operator-rbac.yaml +++ b/deploy/k8s-dev/operator-rbac.yaml @@ -46,7 +46,7 @@ rules: verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["cert-manager.io"] resources: ["certificates"] - verbs: ["get", "list", "watch", "create", "patch", "update"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] - apiGroups: ["cert-manager.io"] resources: ["issuers", "clusterissuers"] verbs: ["get", "list", "watch"] diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index e35e7eb..28299d3 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -1495,7 +1495,7 @@ spec: nullable: true type: string includeGeneratedDnsNames: - default: true + nullable: true type: boolean issuerRef: nullable: true @@ -1603,7 +1603,7 @@ spec: nullable: true type: string includeGeneratedDnsNames: - default: true + nullable: true type: boolean issuerRef: nullable: true diff --git a/deploy/rustfs-operator/crds/tenant.yaml b/deploy/rustfs-operator/crds/tenant.yaml index e35e7eb..28299d3 100755 --- a/deploy/rustfs-operator/crds/tenant.yaml +++ b/deploy/rustfs-operator/crds/tenant.yaml @@ -1495,7 +1495,7 @@ spec: nullable: true type: string includeGeneratedDnsNames: - default: true + nullable: true type: boolean issuerRef: nullable: true @@ -1603,7 +1603,7 @@ spec: nullable: true type: string includeGeneratedDnsNames: - default: true + nullable: true type: boolean issuerRef: nullable: true diff --git a/deploy/rustfs-operator/templates/clusterrole.yaml b/deploy/rustfs-operator/templates/clusterrole.yaml index bb2bcc9..3d100c2 100755 --- a/deploy/rustfs-operator/templates/clusterrole.yaml +++ b/deploy/rustfs-operator/templates/clusterrole.yaml @@ -48,7 +48,7 @@ rules: # cert-manager Certificate orchestration and readiness watches - apiGroups: ["cert-manager.io"] resources: ["certificates"] - verbs: ["get", "list", "watch", "create", "patch", "update"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] - apiGroups: ["cert-manager.io"] resources: ["issuers", "clusterissuers"] verbs: ["get", "list", "watch"] diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index ec04c6b..b42ea55 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -396,6 +396,7 @@ spec: ``` When `manageCertificate: true`, `issuerRef` is also required. The operator creates or reconciles the cert-manager `Certificate`, waits for the referenced Secret, validates `tls.crt` and `tls.key`, and uses `ca.crt` unless another CA trust source is configured. +For the backward-compatible single-certificate form, omitted `includeGeneratedDnsNames` behaves as `true`. For separate public and internal certificates: @@ -434,7 +435,9 @@ spec: ``` The default certificate is projected to `rustfs_cert.pem` and `rustfs_key.pem` at `mountPath`, so RustFS can use it as the fallback certificate and for internode HTTPS. Each `hosts` value is projected as a RustFS SNI directory, for example `s3.example.com/rustfs_cert.pem` and `s3.example.com/rustfs_key.pem`. -When `certificates` is set, configure process-wide trust with top-level `caTrust` or the `caTrust` on the `default: true` certificate. The legacy `certManager.caTrust` field is only used by the single-certificate form. +When `certificates` is set, omitted `includeGeneratedDnsNames` is treated as `true` only on the `default: true` certificate. Non-default entries include only `hosts` and `certManager.dnsNames` unless they explicitly set `includeGeneratedDnsNames: true`. +When `enableInternodeHttps: true`, the default managed certificate must cover the generated RustFS peer DNS names. Keep `includeGeneratedDnsNames` enabled, or list the generated peer names explicitly in `hosts` or `certManager.dnsNames`. +When `certificates` is set, configure process-wide trust with top-level `caTrust` or the `caTrust` on the `default: true` certificate. The legacy `certManager.caTrust` field is only used by the single-certificate form, and `certManager.caTrust` on non-default entries is rejected. ### 7.6 Logging diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index d5d5e2a..80efb44 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -398,6 +398,7 @@ spec: ``` 当 `manageCertificate: true` 时，`issuerRef` 也是必填项。Operator 会创建或更新 cert-manager `Certificate`，等待引用的 Secret 就绪，校验 `tls.crt` 和 `tls.key`，并在未配置其它 CA trust source 时使用 `ca.crt`。 +兼容旧版本的单证书写法中，省略 `includeGeneratedDnsNames` 时按 `true` 处理。 公有域名和内部域名使用不同证书时： @@ -436,7 +437,9 @@ spec: ``` 默认条目会被投影到 `mountPath` 根目录下的 `rustfs_cert.pem` 和 `rustfs_key.pem`，供 RustFS 作为 fallback 证书和节点间 HTTPS 证书使用。每个 `hosts` 值会被投影为 RustFS SNI 子目录，例如 `s3.example.com/rustfs_cert.pem` 和 `s3.example.com/rustfs_key.pem`。 -配置了 `certificates` 时，进程级 trust 应放在顶层 `caTrust` 或 `default: true` 证书条目的 `caTrust` 中。旧的 `certManager.caTrust` 只对单证书写法生效。 +配置了 `certificates` 时，省略 `includeGeneratedDnsNames` 只有在 `default: true` 证书条目中才按 `true` 处理。非默认条目只包含 `hosts` 和 `certManager.dnsNames`，除非显式设置 `includeGeneratedDnsNames: true`。 +当 `enableInternodeHttps: true` 时，默认的托管证书必须覆盖 RustFS 自动生成的节点间 DNS 名称。应保持 `includeGeneratedDnsNames` 启用，或在 `hosts` / `certManager.dnsNames` 中显式列出这些节点间名称。 +配置了 `certificates` 时，进程级 trust 应放在顶层 `caTrust` 或 `default: true` 证书条目的 `caTrust` 中。旧的 `certManager.caTrust` 只对单证书写法生效，非默认条目上的 `certManager.caTrust` 会被拒绝。 ### 7.6 日志配置 diff --git a/e2e/src/framework/cert_manager_tls.rs b/e2e/src/framework/cert_manager_tls.rs index 78acfab..bbf4e69 100644 --- a/e2e/src/framework/cert_manager_tls.rs +++ b/e2e/src/framework/cert_manager_tls.rs @@ -168,7 +168,7 @@ fn tls_certificate_dns_names(config: &E2eConfig, tenant: &Tenant) -> Vec .cloned(), ); - if cert_manager.include_generated_dns_names { + if cert_manager.include_generated_dns_names.unwrap_or(true) { let tenant_name = &config.tenant_name; let namespace = &config.test_namespace; let io_service = format!("{tenant_name}-io"); diff --git a/e2e/tests/cert_manager_tls.rs b/e2e/tests/cert_manager_tls.rs index 00ba9f5..9777b03 100644 --- a/e2e/tests/cert_manager_tls.rs +++ b/e2e/tests/cert_manager_tls.rs @@ -1255,7 +1255,7 @@ fn expected_tls_dns_names(config: &E2eConfig, tenant: &Tenant) -> BTreeSet, +) -> Result<(), context::Error> { + let resource = certificate_api_resource(); + let api: Api = Api::namespaced_with(ctx.client.clone(), namespace, &resource); + let selector = format!("rustfs.tenant={}", tenant.name()); + let certificates = match api.list(&ListParams::default().labels(&selector)).await { + Ok(certificates) => certificates, + Err(source) => { + let error = context::Error::Kube { source }; + if desired_names.is_empty() && context::is_kube_not_found(&error) { + return Ok(()); + } + return Err(error); + } + }; + + for name in stale_cert_manager_certificate_names(tenant, &certificates.items, desired_names) { + if let Err(source) = api.delete(name, &DeleteParams::default()).await { + let error = context::Error::Kube { source }; + if !context::is_kube_not_found(&error) { + return Err(error); + } + } + } + + Ok(()) +} + +fn stale_cert_manager_certificate_names<'a>( + tenant: &Tenant, + certificates: &'a [DynamicObject], + desired_names: &BTreeSet, +) -> Vec<&'a str> { + certificates + .iter() + .filter_map(|certificate| { + let name = certificate.metadata.name.as_deref()?; + if desired_names.contains(name) + || !cert_manager_certificate_owned_by_tenant(certificate, tenant) + { + return None; + } + Some(name) + }) + .collect() +} + +fn cert_manager_certificate_owned_by_tenant(certificate: &DynamicObject, tenant: &Tenant) -> bool { + let tenant_uid = tenant.metadata.uid.as_deref().unwrap_or(""); + certificate + .metadata + .owner_references + .as_ref() + .is_some_and(|refs| { + refs.iter().any(|owner| { + owner.kind == "Tenant" + && owner.name == tenant.name() + && (tenant_uid.is_empty() || owner.uid == tenant_uid) + }) + }) +} + async fn cert_manager_prerequisite_failed( ctx: &Context, tenant: &Tenant, @@ -786,6 +891,7 @@ fn build_cert_manager_certificate( namespace: &str, cert_manager: &CertManagerTlsConfig, hosts: &[String], + include_generated_dns_names: bool, secret_name: &str, certificate_name: &str, ) -> DynamicObject { @@ -807,7 +913,8 @@ fn build_cert_manager_certificate( tenant, namespace, cert_manager, - hosts + hosts, + include_generated_dns_names, )), ); spec.insert( @@ -996,6 +1103,7 @@ fn certificate_dns_names( namespace: &str, cert_manager: &CertManagerTlsConfig, hosts: &[String], + include_generated_dns_names: bool, ) -> Vec { let mut names = BTreeSet::new(); names.extend(hosts.iter().filter(|name| !name.is_empty()).cloned()); @@ -1006,7 +1114,7 @@ fn certificate_dns_names( .filter(|name| !name.is_empty()) .cloned(), ); - if cert_manager.include_generated_dns_names { + if include_generated_dns_names { let tenant_name = tenant.name(); let io_service = format!("{tenant_name}-io"); let headless_service = tenant.headless_service_name(); @@ -1026,26 +1134,62 @@ fn certificate_dns_names( names.into_iter().collect() } +fn include_generated_dns_names(entry: &TlsCertificateEntry) -> bool { + entry + .cert_manager + .include_generated_dns_names + .unwrap_or(entry.default || entry.legacy) +} + +fn validate_managed_certificate_san_config( + tenant: &Tenant, + namespace: &str, + config: &TlsConfig, + entry: &TlsCertificateEntry, +) -> Result<(), TlsValidationFailure> { + if !config.enable_internode_https || !entry.default || include_generated_dns_names(entry) { + return Ok(()); + } + + let generated_names = generated_dns_names(tenant, namespace); + let configured_names = + certificate_dns_names(tenant, namespace, &entry.cert_manager, &entry.hosts, false) + .into_iter() + .collect::>(); + let missing_names = generated_names + .iter() + .filter(|name| !configured_names.contains(*name)) + .cloned() + .collect::>(); + + if missing_names.is_empty() { + return Ok(()); + } + + Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate '{}' cannot set certManager.includeGeneratedDnsNames=false while enableInternodeHttps=true unless certManager.dnsNames or hosts explicitly cover generated peer DNS names such as '{}'", + entry.name, missing_names[0] + ), + }) +} + fn san_validation_dns_names( tenant: &Tenant, namespace: &str, config: &TlsConfig, entry: &TlsCertificateEntry, ) -> Vec { - let mut names = BTreeSet::new(); - names.extend(entry.hosts.iter().filter(|name| !name.is_empty()).cloned()); - names.extend( - entry - .cert_manager - .dns_names - .iter() - .filter(|name| !name.is_empty()) - .cloned(), - ); - if entry.default && config.enable_internode_https { - names.extend(generated_dns_names(tenant, namespace)); - } - names.into_iter().collect() + let include_generated_dns_names = + include_generated_dns_names(entry) || (entry.default && config.enable_internode_https); + certificate_dns_names( + tenant, + namespace, + &entry.cert_manager, + &entry.hosts, + include_generated_dns_names, + ) } fn generated_dns_names(tenant: &Tenant, namespace: &str) -> Vec { @@ -2172,6 +2316,7 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj "storage", cert_manager, &[], + true, "tenant-a-server-tls", "tenant-a-server", ); @@ -2254,7 +2399,7 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj let cert_manager = CertManagerTlsConfig { secret_name: Some("tenant-a-public-tls".to_string()), dns_names: vec!["custom.example.com".to_string()], - include_generated_dns_names: false, + include_generated_dns_names: Some(false), ..Default::default() }; @@ -2263,6 +2408,7 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj "storage", &cert_manager, &["s3.example.com".to_string()], + false, "tenant-a-public-tls", "tenant-a-public-tls", ); @@ -2287,6 +2433,238 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj ); } + #[test] + fn cert_manager_certificate_ownership_requires_current_tenant_owner_ref() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.uid = Some("uid-a".to_string()); + + let mut owned = DynamicObject::new("tenant-a-server", &certificate_api_resource()); + owned.metadata.owner_references = Some(vec![tenant.new_owner_ref()]); + assert!(cert_manager_certificate_owned_by_tenant(&owned, &tenant)); + + let mut wrong_uid = owned.clone(); + wrong_uid + .metadata + .owner_references + .as_mut() + .expect("owner ref should exist")[0] + .uid = "uid-b".to_string(); + assert!(!cert_manager_certificate_owned_by_tenant( + &wrong_uid, &tenant + )); + + let mut unowned = owned; + unowned.metadata.owner_references = None; + assert!(!cert_manager_certificate_owned_by_tenant(&unowned, &tenant)); + } + + #[test] + fn stale_cert_manager_certificate_cleanup_only_selects_owned_stale_certs() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.uid = Some("uid-a".to_string()); + + let mut desired_names = BTreeSet::new(); + desired_names.insert("tenant-a-internal-tls".to_string()); + + let desired = owned_certificate(&tenant, "tenant-a-internal-tls"); + let stale = owned_certificate(&tenant, "tenant-a-old-tls"); + let mut wrong_uid = owned_certificate(&tenant, "tenant-a-other-uid-tls"); + wrong_uid + .metadata + .owner_references + .as_mut() + .expect("owner ref should exist")[0] + .uid = "uid-b".to_string(); + let mut unowned = owned_certificate(&tenant, "tenant-a-unowned-tls"); + unowned.metadata.owner_references = None; + let mut nameless = owned_certificate(&tenant, "tenant-a-nameless-tls"); + nameless.metadata.name = None; + let certificates = vec![desired, stale, wrong_uid, unowned, nameless]; + + let stale_names = + stale_cert_manager_certificate_names(&tenant, &certificates, &desired_names) + .into_iter() + .map(ToString::to_string) + .collect::>(); + + assert_eq!(stale_names, vec!["tenant-a-old-tls".to_string()]); + } + + #[test] + fn non_default_certificate_omits_generated_dns_by_default() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.namespace = Some("storage".to_string()); + tenant.spec.pools[0].servers = 2; + let entry = TlsCertificateEntry { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig::default(), + legacy: false, + }; + + let certificate = build_cert_manager_certificate( + &tenant, + "storage", + &entry.cert_manager, + &entry.hosts, + include_generated_dns_names(&entry), + "tenant-a-public-tls", + "tenant-a-public-tls", + ); + let dns_names = certificate + .data + .pointer("/spec/dnsNames") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect::>() + }); + + assert_eq!(dns_names, Some(vec!["s3.example.com".to_string()])); + } + + #[test] + fn non_default_certificate_can_explicitly_include_generated_dns() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.namespace = Some("storage".to_string()); + tenant.spec.pools[0].servers = 1; + let entry = TlsCertificateEntry { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + include_generated_dns_names: Some(true), + ..Default::default() + }, + legacy: false, + }; + + let certificate = build_cert_manager_certificate( + &tenant, + "storage", + &entry.cert_manager, + &entry.hosts, + include_generated_dns_names(&entry), + "tenant-a-public-tls", + "tenant-a-public-tls", + ); + let dns_names = certificate + .data + .pointer("/spec/dnsNames") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect::>() + }) + .expect("dnsNames should render"); + + assert!(dns_names.contains(&"s3.example.com".to_string())); + assert!(dns_names.contains(&"tenant-a-io.storage.svc".to_string())); + assert!(dns_names.contains(&"tenant-a-hl.storage.svc.cluster.local".to_string())); + } + + #[test] + fn san_validation_dns_names_follow_certificate_generation_and_internode_requirements() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.namespace = Some("storage".to_string()); + tenant.spec.pools[0].servers = 1; + let config = TlsConfig { + enable_internode_https: true, + ..Default::default() + }; + let generated_pod_dns = generated_dns_names(&tenant, "storage") + .into_iter() + .find(|name| name.contains("-0.")) + .expect("generated names should include a pod DNS name"); + + let default_entry = TlsCertificateEntry { + name: "internal".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig { + include_generated_dns_names: Some(false), + ..Default::default() + }, + legacy: false, + }; + let names = san_validation_dns_names(&tenant, "storage", &config, &default_entry); + assert!(names.contains(&generated_pod_dns)); + + let non_default_entry = TlsCertificateEntry { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + include_generated_dns_names: Some(true), + ..Default::default() + }, + legacy: false, + }; + let names = san_validation_dns_names(&tenant, "storage", &config, &non_default_entry); + assert!(names.contains(&"s3.example.com".to_string())); + assert!(names.contains(&generated_pod_dns)); + + let public_default = TlsCertificateEntry { + cert_manager: CertManagerTlsConfig::default(), + ..non_default_entry + }; + let names = san_validation_dns_names(&tenant, "storage", &config, &public_default); + assert_eq!(names, vec!["s3.example.com".to_string()]); + } + + #[test] + fn managed_default_certificate_rejects_missing_internode_dns_when_generated_dns_disabled() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + tenant.metadata.namespace = Some("storage".to_string()); + tenant.spec.pools[0].servers = 1; + let config = TlsConfig { + enable_internode_https: true, + ..Default::default() + }; + let mut entry = TlsCertificateEntry { + name: "internal".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig { + manage_certificate: true, + include_generated_dns_names: Some(false), + ..Default::default() + }, + legacy: false, + }; + + let failure = validate_managed_certificate_san_config(&tenant, "storage", &config, &entry) + .expect_err("managed internode cert must cover generated peer DNS names"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!( + failure + .message + .contains("includeGeneratedDnsNames=false while enableInternodeHttps=true"), + "{}", + failure.message + ); + + entry.cert_manager.dns_names = generated_dns_names(&tenant, "storage"); + assert_eq!( + validate_managed_certificate_san_config(&tenant, "storage", &config, &entry), + Ok(()) + ); + } + #[test] fn multi_certificate_entries_validate_default_and_hosts() { let valid = TlsConfig { @@ -2366,6 +2744,32 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj assert_eq!(failure.reason, Reason::CertificateInvalid); assert!(failure.message.contains("must set hosts")); + + let non_default_ca_trust = TlsConfig { + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + ca_trust: Some(CaTrustConfig::default()), + ..Default::default() + }, + }, + ], + ..Default::default() + }; + let failure = certificate_entries(&non_default_ca_trust) + .expect_err("non-default cert must not accept process-wide CA trust"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!(failure.message.contains("must not set certManager.caTrust")); } #[test] @@ -2633,6 +3037,12 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj object } + fn owned_certificate(tenant: &Tenant, name: &str) -> DynamicObject { + let mut object = DynamicObject::new(name, &certificate_api_resource()); + object.metadata.owner_references = Some(vec![tenant.new_owner_ref()]); + object + } + fn tls_secret( name: &str, resource_version: &str, diff --git a/src/types/v1alpha1/tls.rs b/src/types/v1alpha1/tls.rs index ab368d1..965b14f 100644 --- a/src/types/v1alpha1/tls.rs +++ b/src/types/v1alpha1/tls.rs @@ -123,7 +123,7 @@ pub struct CertManagerPrivateKeyConfig { pub rotation_policy: Option, } -#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, PartialEq)] +#[derive(Deserialize, Serialize, Clone, Debug, Default, KubeSchema, PartialEq)] #[serde(rename_all = "camelCase")] pub struct CertManagerTlsConfig { #[serde(default)] @@ -147,8 +147,8 @@ pub struct CertManagerTlsConfig { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dns_names: Vec, - #[serde(default = "default_include_generated_dns_names")] - pub include_generated_dns_names: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub include_generated_dns_names: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub duration: Option, @@ -180,30 +180,6 @@ pub struct TlsCertificateConfig { pub cert_manager: CertManagerTlsConfig, } -fn default_include_generated_dns_names() -> bool { - true -} - -impl Default for CertManagerTlsConfig { - fn default() -> Self { - Self { - manage_certificate: false, - secret_name: None, - secret_type: None, - certificate_name: None, - issuer_ref: None, - common_name: None, - dns_names: Vec::new(), - include_generated_dns_names: default_include_generated_dns_names(), - duration: None, - renew_before: None, - private_key: None, - usages: Vec::new(), - ca_trust: None, - } - } -} - #[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, PartialEq)] #[serde(rename_all = "camelCase")] pub struct TlsConfig {