diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index 888c15c..e35e7eb 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -1400,6 +1400,45 @@ spec: tls: nullable: true properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object certManager: nullable: true properties: @@ -1506,6 +1545,128 @@ spec: type: string type: array type: object + certificates: + items: + properties: + certManager: + properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object + certificateName: + nullable: true + type: string + commonName: + nullable: true + type: string + dnsNames: + items: + type: string + type: array + duration: + nullable: true + type: string + includeGeneratedDnsNames: + default: true + type: boolean + issuerRef: + nullable: true + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + type: string + name: + type: string + required: + - name + type: object + manageCertificate: + default: false + type: boolean + privateKey: + nullable: true + properties: + algorithm: + nullable: true + type: string + encoding: + nullable: true + type: string + rotationPolicy: + nullable: true + type: string + size: + format: int32 + nullable: true + type: integer + type: object + renewBefore: + nullable: true + type: string + secretName: + nullable: true + type: string + secretType: + nullable: true + type: string + usages: + items: + type: string + type: array + type: object + default: + default: false + type: boolean + hosts: + items: + type: string + type: array + name: + type: string + required: + - certManager + - name + type: object + type: array enableInternodeHttps: default: false type: boolean @@ -1616,6 +1777,67 @@ spec: - kind - name type: object + certificates: + items: + properties: + certificateRef: + nullable: true + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + observedGeneration: + format: int64 + nullable: true + type: integer + ready: + nullable: true + type: boolean + reason: + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + default: + default: false + type: boolean + dnsNames: + items: + type: string + type: array + hosts: + items: + type: string + type: array + managedCertificate: + nullable: true + type: boolean + name: + type: string + serverSecretRef: + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + required: + - name + - serverSecretRef + type: object + type: array clientCaSecretRef: nullable: true properties: @@ -1725,7 +1947,7 @@ spec: description: Status of the condition (True, False, Unknown) type: string type: - description: Type of condition (Ready, Progressing, Degraded) + description: Type of condition (Ready, Reconciling, Degraded) type: string required: - message diff --git a/deploy/rustfs-operator/crds/tenant.yaml b/deploy/rustfs-operator/crds/tenant.yaml index 888c15c..e35e7eb 100755 --- a/deploy/rustfs-operator/crds/tenant.yaml +++ b/deploy/rustfs-operator/crds/tenant.yaml @@ -1400,6 +1400,45 @@ spec: tls: nullable: true properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object certManager: nullable: true properties: @@ -1506,6 +1545,128 @@ spec: type: string type: array type: object + certificates: + items: + properties: + certManager: + properties: + caTrust: + nullable: true + properties: + caSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + clientCaSecretRef: + nullable: true + properties: + key: + default: ca.crt + type: string + name: + type: string + required: + - name + type: object + source: + default: CertificateSecretCa + enum: + - CertificateSecretCa + - SecretRef + - SystemCa + type: string + trustLeafCertificateAsCa: + default: false + type: boolean + trustSystemCa: + default: false + type: boolean + type: object + certificateName: + nullable: true + type: string + commonName: + nullable: true + type: string + dnsNames: + items: + type: string + type: array + duration: + nullable: true + type: string + includeGeneratedDnsNames: + default: true + type: boolean + issuerRef: + nullable: true + properties: + group: + default: cert-manager.io + type: string + kind: + default: Issuer + type: string + name: + type: string + required: + - name + type: object + manageCertificate: + default: false + type: boolean + privateKey: + nullable: true + properties: + algorithm: + nullable: true + type: string + encoding: + nullable: true + type: string + rotationPolicy: + nullable: true + type: string + size: + format: int32 + nullable: true + type: integer + type: object + renewBefore: + nullable: true + type: string + secretName: + nullable: true + type: string + secretType: + nullable: true + type: string + usages: + items: + type: string + type: array + type: object + default: + default: false + type: boolean + hosts: + items: + type: string + type: array + name: + type: string + required: + - certManager + - name + type: object + type: array enableInternodeHttps: default: false type: boolean @@ -1616,6 +1777,67 @@ spec: - kind - name type: object + certificates: + items: + properties: + certificateRef: + nullable: true + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + observedGeneration: + format: int64 + nullable: true + type: integer + ready: + nullable: true + type: boolean + reason: + nullable: true + type: string + required: + - apiVersion + - kind + - name + type: object + default: + default: false + type: boolean + dnsNames: + items: + type: string + type: array + hosts: + items: + type: string + type: array + managedCertificate: + nullable: true + type: boolean + name: + type: string + serverSecretRef: + properties: + key: + nullable: true + type: string + name: + type: string + resourceVersion: + nullable: true + type: string + required: + - name + type: object + required: + - name + - serverSecretRef + type: object + type: array clientCaSecretRef: nullable: true properties: @@ -1725,7 +1947,7 @@ spec: description: Status of the condition (True, False, Unknown) type: string type: - description: Type of condition (Ready, Progressing, Degraded) + description: Type of condition (Ready, Reconciling, Degraded) type: string required: - message diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index f8252e9..ec04c6b 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -373,7 +373,9 @@ Important fields: | `rotationStrategy` | `Rollout` is supported. `HotReload` is accepted by the CRD but currently blocks reconciliation. | | `enableInternodeHttps` | Use HTTPS for RustFS peer communication. | | `requireSanMatch` | Require generated DNS names to match certificate SANs. Defaults to `true`. | -| `certManager` | Certificate settings when using cert-manager. `secretName` is required for `mode: certManager`. | +| `certManager` | Backward-compatible single certificate settings. `secretName` is required for `mode: certManager` when `certificates` is empty. | +| `certificates` | Multiple server certificate entries rendered into the RustFS TLS directory for SNI. Exactly one entry must set `default: true`; non-default entries must set `hosts`. | +| `caTrust` | Process-wide RustFS trust settings. This controls `ca.crt`, `client_ca.crt`, `RUSTFS_TRUST_SYSTEM_CA`, and server mTLS; it is not selected per SNI host. | For cert-manager managed certificates: @@ -395,6 +397,45 @@ spec: When `manageCertificate: true`, `issuerRef` is also required. The operator creates or reconciles the cert-manager `Certificate`, waits for the referenced Secret, validates `tls.crt` and `tls.key`, and uses `ca.crt` unless another CA trust source is configured. +For separate public and internal certificates: + +```yaml +spec: + tls: + mode: certManager + rotationStrategy: Rollout + enableInternodeHttps: true + caTrust: + source: CertificateSecretCa + certificates: + - name: internal + default: true + hosts: + - rustfs.internal.example.local + certManager: + manageCertificate: true + secretName: rustfs-internal-tls + issuerRef: + group: cert-manager.io + kind: Issuer + name: private-ca + includeGeneratedDnsNames: true + - name: public + hosts: + - s3.example.com + certManager: + manageCertificate: true + secretName: rustfs-public-tls + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: letsencrypt-prod + includeGeneratedDnsNames: false +``` + +The default certificate is projected to `rustfs_cert.pem` and `rustfs_key.pem` at `mountPath`, so RustFS can use it as the fallback certificate and for internode HTTPS. Each `hosts` value is projected as a RustFS SNI directory, for example `s3.example.com/rustfs_cert.pem` and `s3.example.com/rustfs_key.pem`. +When `certificates` is set, configure process-wide trust with top-level `caTrust` or the `caTrust` on the `default: true` certificate. The legacy `certManager.caTrust` field is only used by the single-certificate form. + ### 7.6 Logging Tenant logging is configured under `spec.logging`. diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index 8f7ee2b..d5d5e2a 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -375,7 +375,9 @@ Tenant TLS 通过 `spec.tls` 配置。 | `rotationStrategy` | 当前支持 `Rollout`。`HotReload` 会被 CRD 接受，但目前会阻塞 reconcile。 | | `enableInternodeHttps` | RustFS 节点间通信是否使用 HTTPS。 | | `requireSanMatch` | 是否要求证书 SAN 匹配生成的 DNS 名称。默认 `true`。 | -| `certManager` | 使用 cert-manager 时的证书配置。`mode: certManager` 必须设置 `secretName`。 | +| `certManager` | 兼容旧版本的单证书配置。`certificates` 为空且 `mode: certManager` 时必须设置 `secretName`。 | +| `certificates` | 多个服务端证书配置，会被渲染到 RustFS TLS 目录供 SNI 使用。必须且只能有一个条目设置 `default: true`，非默认条目必须设置 `hosts`。 | +| `caTrust` | RustFS 进程级信任配置。它控制 `ca.crt`、`client_ca.crt`、`RUSTFS_TRUST_SYSTEM_CA` 和服务端 mTLS，不会按 SNI host 分别生效。 | cert-manager 证书示例： @@ -397,6 +399,45 @@ spec: 当 `manageCertificate: true` 时，`issuerRef` 也是必填项。Operator 会创建或更新 cert-manager `Certificate`，等待引用的 Secret 就绪，校验 `tls.crt` 和 `tls.key`，并在未配置其它 CA trust source 时使用 `ca.crt`。 +公有域名和内部域名使用不同证书时： + +```yaml +spec: + tls: + mode: certManager + rotationStrategy: Rollout + enableInternodeHttps: true + caTrust: + source: CertificateSecretCa + certificates: + - name: internal + default: true + hosts: + - rustfs.internal.example.local + certManager: + manageCertificate: true + secretName: rustfs-internal-tls + issuerRef: + group: cert-manager.io + kind: Issuer + name: private-ca + includeGeneratedDnsNames: true + - name: public + hosts: + - s3.example.com + certManager: + manageCertificate: true + secretName: rustfs-public-tls + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: letsencrypt-prod + includeGeneratedDnsNames: false +``` + +默认条目会被投影到 `mountPath` 根目录下的 `rustfs_cert.pem` 和 `rustfs_key.pem`，供 RustFS 作为 fallback 证书和节点间 HTTPS 证书使用。每个 `hosts` 值会被投影为 RustFS SNI 子目录，例如 `s3.example.com/rustfs_cert.pem` 和 `s3.example.com/rustfs_key.pem`。 +配置了 `certificates` 时，进程级 trust 应放在顶层 `caTrust` 或 `default: true` 证书条目的 `caTrust` 中。旧的 `certManager.caTrust` 只对单证书写法生效。 + ### 7.6 日志配置 Tenant 日志通过 `spec.logging` 配置。 diff --git a/examples/README.md b/examples/README.md index 0215429..9d5aecc 100755 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,7 @@ This directory contains example Tenant configurations for the RustFS Kubernetes | [simple-tenant.yaml](./simple-tenant.yaml) | Documentation Reference | ⭐⭐ Moderate | Configurable | Learning all options | | [secret-credentials-tenant.yaml](./secret-credentials-tenant.yaml) | Secret-based Credentials | ⭐ Simple | Configurable | **Production credential security** | | [provisioning-tenant.yaml](./provisioning-tenant.yaml) | Policy/User/Bucket Provisioning | ⭐⭐ Moderate | 40Gi | Tenant bootstrap automation | +| [multi-cert-tls-tenant.yaml](./multi-cert-tls-tenant.yaml) | Public/Internal TLS | ⭐⭐ Moderate | 40Gi | Separate SNI certificates | | [multi-pool-tenant.yaml](./multi-pool-tenant.yaml) | Multiple Pools | ⭐⭐ Moderate | ~160Gi | Multi-pool setups | | [production-ha-tenant.yaml](./production-ha-tenant.yaml) | Production HA | ⭐⭐⭐ Advanced | 6.4TB | HA with zone distribution | | [cluster-expansion-tenant.yaml](./cluster-expansion-tenant.yaml) | Capacity Expansion | ⭐⭐⭐ Advanced | 384TB | Growing cluster capacity | diff --git a/examples/multi-cert-tls-tenant.yaml b/examples/multi-cert-tls-tenant.yaml new file mode 100644 index 0000000..ceb45ee --- /dev/null +++ b/examples/multi-cert-tls-tenant.yaml @@ -0,0 +1,65 @@ +# RustFS Tenant example for separate internal and public TLS certificates. +# +# This manifest does not create cert-manager Issuers. Before applying it, create +# the private Issuer and public ClusterIssuer referenced below, or adjust the +# issuerRef values to match your cluster. +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: multi-cert-tls + namespace: rustfs-system +spec: + image: rustfs/rustfs:latest + + tls: + mode: certManager + mountPath: /var/run/rustfs/tls + rotationStrategy: Rollout + enableInternodeHttps: true + requireSanMatch: true + caTrust: + source: CertificateSecretCa + trustSystemCa: false + trustLeafCertificateAsCa: false + certificates: + - name: internal + default: true + hosts: + - rustfs.internal.example.local + certManager: + manageCertificate: true + certificateName: multi-cert-tls-internal + secretName: multi-cert-tls-internal + issuerRef: + group: cert-manager.io + kind: Issuer + name: rustfs-private-ca + includeGeneratedDnsNames: true + usages: + - server auth + - name: public + hosts: + - s3.example.com + certManager: + manageCertificate: true + certificateName: multi-cert-tls-public + secretName: multi-cert-tls-public + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: letsencrypt-prod + includeGeneratedDnsNames: false + usages: + - server auth + + pools: + - name: primary + servers: 1 + persistence: + volumesPerServer: 4 + volumeClaimTemplate: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/src/reconcile/tls.rs b/src/reconcile/tls.rs index 47e984b..a19ec76 100644 --- a/src/reconcile/tls.rs +++ b/src/reconcile/tls.rs @@ -17,18 +17,19 @@ use crate::context::{self, Context}; use crate::status::{StatusBuilder, StatusError}; use crate::types::v1alpha1::status::Reason; use crate::types::v1alpha1::status::certificate::{ - CertificateObjectRef, SecretStatusRef, TlsCertificateStatus, + CertificateObjectRef, SecretStatusRef, TlsCertificateStatus, TlsServerCertificateStatus, }; use crate::types::v1alpha1::tenant::Tenant; use crate::types::v1alpha1::tls::{ - CaTrustSource, CertManagerIssuerRef, CertManagerTlsConfig, SecretKeyReference, TlsConfig, - TlsMode, TlsPlan, TlsRotationStrategy, + CaTrustSource, CertManagerIssuerRef, CertManagerTlsConfig, SecretKeyReference, + TlsCertificateConfig, TlsConfig, TlsMode, TlsPlan, TlsRotationStrategy, + TlsServerCertificateMount, }; use k8s_openapi::api::core::v1::Secret; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::api::{Api, Patch, PatchParams}; use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; -use rustls::pki_types::{CertificateDer, ServerName}; +use rustls::pki_types::{CertificateDer, DnsName, ServerName}; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; @@ -79,6 +80,24 @@ struct CertManagerCertificateObservation { message: Option, } +#[derive(Clone, Debug)] +struct TlsCertificateEntry { + name: String, + default: bool, + hosts: Vec, + cert_manager: CertManagerTlsConfig, + legacy: bool, +} + +#[derive(Clone, Debug)] +struct ObservedTlsCertificate { + entry: TlsCertificateEntry, + secret_name: String, + secret: Secret, + certificate_ref: Option, + san_dns_names: Vec, +} + impl CertManagerCertificateObservation { fn status_ref(&self) -> CertificateObjectRef { CertificateObjectRef { @@ -151,189 +170,218 @@ async fn reconcile_cert_manager_tls( namespace: &str, config: &TlsConfig, ) -> Result { - let Some(cert_manager) = config.cert_manager.as_ref() else { - return tls_blocked( - ctx, - tenant, - config, - Reason::CertificateSecretNotFound, - "spec.tls.certManager.secretName is required for certManager TLS mode".to_string(), - ) - .await; + let entries = match certificate_entries(config) { + Ok(entries) => entries, + Err(failure) => return tls_validation_blocked(ctx, tenant, config, failure).await, }; - let Some(secret_name) = cert_manager - .secret_name - .as_deref() - .filter(|name| !name.is_empty()) - else { - return tls_blocked( + if entries + .iter() + .any(|entry| entry.cert_manager.manage_certificate) + && let Err(error) = ensure_cert_manager_certificate_crd(ctx).await + { + return cert_manager_prerequisite_failed( ctx, tenant, config, - Reason::CertificateSecretNotFound, - "spec.tls.certManager.secretName is required for certManager TLS mode".to_string(), + CertManagerPrerequisite::CertificateCrd, + error, + format!( + "cert-manager Certificate CRD '{}' is not installed", + CERT_MANAGER_CERTIFICATE_CRD + ), ) .await; - }; + } - let mut certificate_ref = None; - if cert_manager.manage_certificate { - let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() else { + let mut observed = Vec::with_capacity(entries.len()); + for entry in entries { + let cert_manager = &entry.cert_manager; + let Some(secret_name) = cert_manager + .secret_name + .as_deref() + .filter(|name| !name.is_empty()) + .map(ToString::to_string) + else { return tls_blocked( ctx, tenant, config, - Reason::CertManagerIssuerNotFound, - "spec.tls.certManager.issuerRef is required when manageCertificate=true" - .to_string(), - ) - .await; - }; - let certificate_name = certificate_name(tenant, cert_manager); - - if let Err(error) = ensure_cert_manager_certificate_crd(ctx).await { - return cert_manager_prerequisite_failed( - ctx, - tenant, - config, - CertManagerPrerequisite::CertificateCrd, - error, - format!( - "cert-manager Certificate CRD '{}' is not installed", - CERT_MANAGER_CERTIFICATE_CRD - ), - ) - .await; - } - - if let Err(error) = ensure_cert_manager_issuer(ctx, namespace, issuer_ref).await { - return cert_manager_prerequisite_failed( - ctx, - tenant, - config, - issuer_prerequisite(issuer_ref), - error, + Reason::CertificateSecretNotFound, format!( - "cert-manager {} '{}' was not found", - issuer_ref.kind, issuer_ref.name + "spec.tls certificate '{}' requires certManager.secretName", + entry.name ), ) .await; - } + }; - let desired_certificate = build_cert_manager_certificate( - tenant, - namespace, - config, - cert_manager, - secret_name, - &certificate_name, - ); - let observed_certificate = match apply_cert_manager_certificate( - ctx, - namespace, - &certificate_name, - &desired_certificate, - ) - .await - { - Ok(certificate) => certificate, - Err(error) if context::is_kube_not_found(&error) => { + let mut certificate_ref = None; + if cert_manager.manage_certificate { + let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() else { return tls_blocked( ctx, tenant, config, - Reason::CertManagerCrdMissing, + Reason::CertManagerIssuerNotFound, format!( - "cert-manager Certificate API was not found while applying '{}'", - certificate_name + "spec.tls certificate '{}' requires certManager.issuerRef when manageCertificate=true", + entry.name ), ) .await; - } - Err(error) => { - return tls_blocked( + }; + let certificate_name = certificate_name(tenant, &entry); + + if let Err(error) = ensure_cert_manager_issuer(ctx, namespace, issuer_ref).await { + return cert_manager_prerequisite_failed( ctx, tenant, config, - Reason::CertManagerCertificateApplyFailed, + issuer_prerequisite(issuer_ref), + error, format!( - "failed to apply cert-manager Certificate '{}': {}", - certificate_name, - sanitize_status_message(&error.to_string()) + "cert-manager {} '{}' was not found", + issuer_ref.kind, issuer_ref.name ), ) .await; } - }; - let observation = observe_cert_manager_certificate(&observed_certificate); - certificate_ref = Some(observation.status_ref()); - if !observation.ready { - return tls_pending_with_certificate_ref( - ctx, + + let desired_certificate = build_cert_manager_certificate( tenant, - config, - Reason::CertManagerCertificateNotReady, - certificate_not_ready_message(&certificate_name, &observation), - certificate_ref.clone(), + namespace, + cert_manager, + &entry.hosts, + &secret_name, + &certificate_name, + ); + let observed_certificate = match apply_cert_manager_certificate( + ctx, + namespace, + &certificate_name, + &desired_certificate, ) - .await; + .await + { + Ok(certificate) => certificate, + Err(error) if context::is_kube_not_found(&error) => { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerCrdMissing, + format!( + "cert-manager Certificate API was not found while applying '{}'", + certificate_name + ), + ) + .await; + } + Err(error) => { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertManagerCertificateApplyFailed, + format!( + "failed to apply cert-manager Certificate '{}': {}", + certificate_name, + sanitize_status_message(&error.to_string()) + ), + ) + .await; + } + }; + let observation = observe_cert_manager_certificate(&observed_certificate); + certificate_ref = Some(observation.status_ref()); + if !observation.ready { + return tls_pending_with_certificate_ref( + ctx, + tenant, + config, + Reason::CertManagerCertificateNotReady, + certificate_not_ready_message(&certificate_name, &observation), + certificate_ref.clone(), + ) + .await; + } } - } - let secret = get_server_secret_or_tls_error( - ctx, - tenant, - config, - namespace, - secret_name, - cert_manager.manage_certificate, - certificate_ref.clone(), - ) - .await?; + let secret = get_server_secret_or_tls_error( + ctx, + tenant, + config, + namespace, + &secret_name, + cert_manager.manage_certificate, + certificate_ref.clone(), + ) + .await?; - if let Err(failure) = validate_tls_secret_type( - &secret, - secret_name, - cert_manager - .secret_type - .as_deref() - .filter(|secret_type| !secret_type.is_empty()), - ) { - return tls_validation_blocked(ctx, tenant, config, failure).await; - } + if let Err(failure) = validate_tls_secret_type( + &secret, + &secret_name, + cert_manager + .secret_type + .as_deref() + .filter(|secret_type| !secret_type.is_empty()), + ) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } - let cert_bytes = require_secret_key( - ctx, - tenant, - config, - &secret, - secret_name, - TLS_CERT_KEY, - Reason::CertificateSecretMissingKey, - ) - .await?; - require_secret_key( - ctx, - tenant, - config, - &secret, - secret_name, - TLS_KEY_KEY, - Reason::CertificateSecretMissingKey, - ) - .await?; + let cert_bytes = require_secret_key( + ctx, + tenant, + config, + &secret, + &secret_name, + TLS_CERT_KEY, + Reason::CertificateSecretMissingKey, + ) + .await?; + require_secret_key( + ctx, + tenant, + config, + &secret, + &secret_name, + TLS_KEY_KEY, + Reason::CertificateSecretMissingKey, + ) + .await?; - if config.require_san_match && config.enable_internode_https { - let expected_dns_names = certificate_dns_names(tenant, namespace, cert_manager); - if let Err(failure) = - validate_tls_secret_san_match(secret_name, &cert_bytes, &expected_dns_names) + let san_dns_names = san_validation_dns_names(tenant, namespace, config, &entry); + if config.require_san_match + && let Err(failure) = + validate_tls_secret_san_match(&secret_name, &cert_bytes, &san_dns_names) { return tls_validation_blocked(ctx, tenant, config, failure).await; } + + observed.push(ObservedTlsCertificate { + entry, + secret_name, + secret, + certificate_ref, + san_dns_names, + }); } + let Some(default_certificate) = observed + .iter() + .find(|certificate| certificate.entry.default) + else { + return tls_blocked( + ctx, + tenant, + config, + Reason::CertificateInvalid, + "spec.tls.certificates must contain exactly one default=true certificate".to_string(), + ) + .await; + }; + let ca_trust = config.ca_trust(); let trust_system_ca = ca_trust.trust_system_ca || ca_trust.source == CaTrustSource::SystemCa; let mut server_ca_key = None; @@ -343,8 +391,8 @@ async fn reconcile_cert_manager_tls( match ca_trust.source { CaTrustSource::CertificateSecretCa => match certificate_secret_ca_material( - &secret, - secret_name, + &default_certificate.secret, + &default_certificate.secret_name, config.enable_internode_https, trust_system_ca, ) { @@ -442,7 +490,7 @@ async fn reconcile_cert_manager_tls( let hash = tls_hash( config, - &secret, + &observed, explicit_ca.as_ref(), explicit_ca_bytes.as_deref(), client_ca.as_ref(), @@ -451,19 +499,28 @@ async fn reconcile_cert_manager_tls( ); let status = cert_manager_tls_status( config, - secret_name, - &secret, + &observed, explicit_ca.as_ref().zip(explicit_ca_secret.as_ref()), client_ca.as_ref().zip(client_ca_secret.as_ref()), &hash, - certificate_ref, ); + let server_certificates = observed + .iter() + .map(|certificate| TlsServerCertificateMount { + secret_name: certificate.secret_name.clone(), + domains: rustfs_certificate_domains(certificate), + ca_key: certificate + .entry + .default + .then(|| server_ca_key.clone()) + .flatten(), + }) + .collect(); - Ok(TlsPlan::rollout( + Ok(TlsPlan::rollout_certificates( config.mount_path.clone(), hash, - secret_name.to_string(), - server_ca_key, + server_certificates, explicit_ca, client_ca, config.enable_internode_https, @@ -516,6 +573,143 @@ async fn get_server_secret_or_tls_error( } } +fn certificate_entries( + config: &TlsConfig, +) -> Result, TlsValidationFailure> { + let entries = if config.certificates.is_empty() { + let Some(cert_manager) = config.cert_manager.clone() else { + return Err(TlsValidationFailure { + reason: Reason::CertificateSecretNotFound, + message: "spec.tls.certManager.secretName or spec.tls.certificates is required for certManager TLS mode".to_string(), + }); + }; + vec![TlsCertificateEntry { + name: "default".to_string(), + default: true, + hosts: Vec::new(), + cert_manager, + legacy: true, + }] + } else { + config.certificates.iter().map(certificate_entry).collect() + }; + + validate_certificate_entries(entries) +} + +fn certificate_entry(config: &TlsCertificateConfig) -> TlsCertificateEntry { + TlsCertificateEntry { + name: config.name.clone(), + default: config.default, + hosts: config.hosts.clone(), + cert_manager: config.cert_manager.clone(), + legacy: false, + } +} + +fn validate_certificate_entries( + entries: Vec, +) -> Result, TlsValidationFailure> { + if entries.is_empty() { + return Err(TlsValidationFailure { + reason: Reason::CertificateSecretNotFound, + message: "spec.tls.certificates must contain at least one certificate".to_string(), + }); + } + + let mut names = BTreeSet::new(); + let mut hosts = BTreeSet::new(); + let mut default_count = 0; + for entry in &entries { + validate_certificate_entry_name(&entry.name)?; + if !names.insert(entry.name.clone()) { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls.certificates contains duplicate name '{}'", + entry.name + ), + }); + } + if entry.default { + default_count += 1; + } else if entry.hosts.is_empty() { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate '{}' must set hosts unless default=true", + entry.name + ), + }); + } + for host in &entry.hosts { + validate_rustfs_sni_host(host)?; + if !hosts.insert(host.clone()) { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!("spec.tls.certificates contains duplicate host '{}'", host), + }); + } + } + } + + if default_count != 1 { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: "spec.tls.certificates must contain exactly one default=true certificate" + .to_string(), + }); + } + + Ok(entries) +} + +fn validate_certificate_entry_name(name: &str) -> Result<(), TlsValidationFailure> { + let valid = !name.is_empty() + && name.len() <= 63 + && name + .chars() + .all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '-') + && name + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && name + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric); + if valid { + return Ok(()); + } + Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate name '{}' must be a DNS label (lowercase alphanumeric or '-', 1-63 chars)", + name + ), + }) +} + +fn validate_rustfs_sni_host(host: &str) -> Result<(), TlsValidationFailure> { + let valid_path = !host.is_empty() + && host != "." + && host != ".." + && !host.starts_with('.') + && !host.contains('/') + && !host.contains('\\') + && !host.contains('*'); + if valid_path && DnsName::try_from(host).is_ok() { + return Ok(()); + } + Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "spec.tls certificate host '{}' must be a concrete DNS name usable as a RustFS SNI directory", + host + ), + }) +} + async fn ensure_cert_manager_certificate_crd(ctx: &Context) -> Result<(), context::Error> { let api: Api = Api::all(ctx.client.clone()); api.get(CERT_MANAGER_CERTIFICATE_CRD) @@ -590,8 +784,8 @@ async fn cert_manager_prerequisite_failed( fn build_cert_manager_certificate( tenant: &Tenant, namespace: &str, - _config: &TlsConfig, cert_manager: &CertManagerTlsConfig, + hosts: &[String], secret_name: &str, certificate_name: &str, ) -> DynamicObject { @@ -609,7 +803,12 @@ fn build_cert_manager_certificate( } spec.insert( "dnsNames".to_string(), - json!(certificate_dns_names(tenant, namespace, cert_manager)), + json!(certificate_dns_names( + tenant, + namespace, + cert_manager, + hosts + )), ); spec.insert( "usages".to_string(), @@ -768,13 +967,20 @@ fn missing_cert_manager_prerequisite_reason(prerequisite: CertManagerPrerequisit } } -fn certificate_name(tenant: &Tenant, cert_manager: &CertManagerTlsConfig) -> String { - cert_manager +fn certificate_name(tenant: &Tenant, entry: &TlsCertificateEntry) -> String { + entry + .cert_manager .certificate_name .as_deref() .filter(|name| !name.is_empty()) .map(ToString::to_string) - .unwrap_or_else(|| format!("{}-server-tls", tenant.name())) + .unwrap_or_else(|| { + if entry.legacy { + format!("{}-server-tls", tenant.name()) + } else { + format!("{}-{}-tls", tenant.name(), entry.name) + } + }) } fn issuer_ref_value(issuer_ref: &CertManagerIssuerRef) -> Value { @@ -789,8 +995,10 @@ fn certificate_dns_names( tenant: &Tenant, namespace: &str, cert_manager: &CertManagerTlsConfig, + hosts: &[String], ) -> Vec { let mut names = BTreeSet::new(); + names.extend(hosts.iter().filter(|name| !name.is_empty()).cloned()); names.extend( cert_manager .dns_names @@ -818,6 +1026,57 @@ fn certificate_dns_names( names.into_iter().collect() } +fn san_validation_dns_names( + tenant: &Tenant, + namespace: &str, + config: &TlsConfig, + entry: &TlsCertificateEntry, +) -> Vec { + let mut names = BTreeSet::new(); + names.extend(entry.hosts.iter().filter(|name| !name.is_empty()).cloned()); + names.extend( + entry + .cert_manager + .dns_names + .iter() + .filter(|name| !name.is_empty()) + .cloned(), + ); + if entry.default && config.enable_internode_https { + names.extend(generated_dns_names(tenant, namespace)); + } + names.into_iter().collect() +} + +fn generated_dns_names(tenant: &Tenant, namespace: &str) -> Vec { + let mut names = BTreeSet::new(); + let tenant_name = tenant.name(); + let io_service = format!("{tenant_name}-io"); + let headless_service = tenant.headless_service_name(); + names.insert(format!("{io_service}.{namespace}.svc")); + names.insert(format!("{io_service}.{namespace}.svc.cluster.local")); + names.insert(format!("{headless_service}.{namespace}.svc")); + names.insert(format!("{headless_service}.{namespace}.svc.cluster.local")); + for pool in &tenant.spec.pools { + for ordinal in 0..pool.servers.max(0) { + names.insert(format!( + "{tenant_name}-{}-{ordinal}.{headless_service}.{namespace}.svc.cluster.local", + pool.name + )); + } + } + names.into_iter().collect() +} + +fn rustfs_certificate_domains(certificate: &ObservedTlsCertificate) -> Vec> { + let mut domains = Vec::new(); + if certificate.entry.default { + domains.push(None); + } + domains.extend(certificate.entry.hosts.iter().cloned().map(Some)); + domains +} + fn certificate_usages(cert_manager: &CertManagerTlsConfig) -> Vec { if cert_manager.usages.is_empty() { vec!["server auth".to_string()] @@ -1182,41 +1441,57 @@ async fn patch_tls_error_with_certificate_ref( fn cert_manager_tls_status( config: &TlsConfig, - secret_name: &str, - secret: &Secret, + certificates: &[ObservedTlsCertificate], explicit_ca: Option<(&SecretKeyReference, &Secret)>, client_ca: Option<(&SecretKeyReference, &Secret)>, hash: &str, - certificate_ref: Option, ) -> TlsCertificateStatus { let ca_trust = config.ca_trust(); + let Some(default_certificate) = certificates + .iter() + .find(|certificate| certificate.entry.default) + .or_else(|| certificates.first()) + else { + return TlsCertificateStatus { + mode: tls_mode_name(config.mode).to_string(), + ready: false, + rotation_strategy: Some(rotation_strategy_name(config.rotation_strategy).to_string()), + mount_path: Some(config.mount_path.clone()), + trust_source: Some(ca_trust_source_name(ca_trust.source).to_string()), + last_error_reason: Some(Reason::CertificateInvalid.as_str().to_string()), + last_error_message: Some("no TLS server certificates were observed".to_string()), + ..Default::default() + }; + }; + let server_secret_ref = SecretStatusRef { + name: default_certificate.secret_name.clone(), + key: None, + resource_version: default_certificate.secret.metadata.resource_version.clone(), + }; TlsCertificateStatus { mode: tls_mode_name(config.mode).to_string(), ready: true, - managed_certificate: config - .cert_manager - .as_ref() - .map(|cert_manager| cert_manager.manage_certificate), + managed_certificate: Some(default_certificate.entry.cert_manager.manage_certificate), rotation_strategy: Some(rotation_strategy_name(config.rotation_strategy).to_string()), mount_path: Some(config.mount_path.clone()), - certificate_ref, - server_secret_ref: Some(SecretStatusRef { - name: secret_name.to_string(), - key: None, - resource_version: secret.metadata.resource_version.clone(), - }), - ca_secret_ref: ca_status_ref(secret_name, secret, explicit_ca), + certificate_ref: default_certificate.certificate_ref.clone(), + server_secret_ref: Some(server_secret_ref.clone()), + certificates: certificates + .iter() + .map(tls_server_certificate_status) + .collect(), + ca_secret_ref: ca_status_ref( + &default_certificate.secret_name, + &default_certificate.secret, + explicit_ca, + ), client_ca_secret_ref: client_ca.map(|(secret_ref, ca_secret)| SecretStatusRef { name: secret_ref.name.clone(), key: Some(secret_ref.key.clone()), resource_version: ca_secret.metadata.resource_version.clone(), }), observed_hash: Some(hash.to_string()), - dns_names: config - .cert_manager - .as_ref() - .map(|cert_manager| cert_manager.dns_names.clone()) - .unwrap_or_default(), + dns_names: default_certificate.san_dns_names.clone(), trust_source: Some(ca_trust_source_name(ca_trust.source).to_string()), last_validated_time: Some( chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), @@ -1225,6 +1500,24 @@ fn cert_manager_tls_status( } } +fn tls_server_certificate_status( + certificate: &ObservedTlsCertificate, +) -> TlsServerCertificateStatus { + TlsServerCertificateStatus { + name: certificate.entry.name.clone(), + default: certificate.entry.default, + hosts: certificate.entry.hosts.clone(), + managed_certificate: Some(certificate.entry.cert_manager.manage_certificate), + certificate_ref: certificate.certificate_ref.clone(), + server_secret_ref: SecretStatusRef { + name: certificate.secret_name.clone(), + key: None, + resource_version: certificate.secret.metadata.resource_version.clone(), + }, + dns_names: certificate.san_dns_names.clone(), + } +} + fn ca_status_ref( secret_name: &str, secret: &Secret, @@ -1274,7 +1567,7 @@ fn error_tls_status_with_certificate_ref( fn tls_hash( config: &TlsConfig, - secret: &Secret, + certificates: &[ObservedTlsCertificate], explicit_ca: Option<&SecretKeyReference>, explicit_ca_bytes: Option<&[u8]>, client_ca: Option<&SecretKeyReference>, @@ -1302,17 +1595,44 @@ fn tls_hash( "trustSystemCa", if trust_system_ca { "true" } else { "false" }, ); - hash_str( - &mut hasher, - "serverSecret.resourceVersion", - secret.metadata.resource_version.as_deref().unwrap_or(""), - ); - hash_bytes(&mut hasher, "tls.crt", secret_bytes(secret, TLS_CERT_KEY)); - hash_bytes( - &mut hasher, - "secret.ca.crt", - secret_bytes(secret, CA_CERT_KEY), - ); + for certificate in certificates { + hash_str(&mut hasher, "certificate.name", &certificate.entry.name); + hash_str( + &mut hasher, + "certificate.default", + if certificate.entry.default { + "true" + } else { + "false" + }, + ); + for host in &certificate.entry.hosts { + hash_str(&mut hasher, "certificate.host", host); + } + hash_str(&mut hasher, "serverSecret.name", &certificate.secret_name); + hash_str( + &mut hasher, + "serverSecret.resourceVersion", + certificate + .secret + .metadata + .resource_version + .as_deref() + .unwrap_or(""), + ); + hash_bytes( + &mut hasher, + "tls.crt", + secret_bytes(&certificate.secret, TLS_CERT_KEY), + ); + if certificate.entry.default { + hash_bytes( + &mut hasher, + "secret.ca.crt", + secret_bytes(&certificate.secret, CA_CERT_KEY), + ); + } + } if let Some(secret_ref) = explicit_ca { hash_str(&mut hasher, "explicitCa.name", &secret_ref.name); hash_str(&mut hasher, "explicitCa.key", &secret_ref.key); @@ -1382,7 +1702,7 @@ const fn ca_trust_source_name(source: CaTrustSource) -> &'static str { mod tests { use super::*; use crate::types::v1alpha1::tls::{ - CaTrustConfig, CertManagerPrivateKeyConfig, CertManagerTlsConfig, + CaTrustConfig, CertManagerPrivateKeyConfig, CertManagerTlsConfig, TlsCertificateConfig, }; use k8s_openapi::ByteString; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; @@ -1635,19 +1955,56 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj Some(b"-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----\n"), ); - let baseline = tls_hash(&config, &first, None, None, None, None, false); + let baseline_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + first, + CertManagerTlsConfig::default(), + vec![], + )]; + let private_key_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + changed_private_key, + CertManagerTlsConfig::default(), + vec![], + )]; + let resource_version_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + changed_resource_version, + CertManagerTlsConfig::default(), + vec![], + )]; + let ca_certs = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + changed_ca, + CertManagerTlsConfig::default(), + vec![], + )]; + + let baseline = tls_hash(&config, &baseline_certs, None, None, None, None, false); let private_key_changed = - tls_hash(&config, &changed_private_key, None, None, None, None, false); + tls_hash(&config, &private_key_certs, None, None, None, None, false); let resource_version_changed = tls_hash( &config, - &changed_resource_version, + &resource_version_certs, None, None, None, None, false, ); - let ca_changed = tls_hash(&config, &changed_ca, None, None, None, None, false); + let ca_changed = tls_hash(&config, &ca_certs, None, None, None, None, false); assert_eq!(baseline, private_key_changed); assert_ne!(baseline, resource_version_changed); @@ -1742,14 +2099,21 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj Some(PUBLIC_CERT_PEM), ); + let observed = vec![observed_tls_certificate( + "default", + true, + vec![], + "server-tls", + server, + config.cert_manager.as_ref().unwrap().clone(), + vec![], + )]; let status = cert_manager_tls_status( &config, - "server-tls", - &server, + &observed, Some((&secret_ref("server-ca", "ca.crt"), &ca)), Some((&secret_ref("client-ca", "client_ca.crt"), &client_ca)), "sha256:test", - None, ); assert_eq!( @@ -1806,8 +2170,8 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj let certificate = build_cert_manager_certificate( &tenant, "storage", - &config, cert_manager, + &[], "tenant-a-server-tls", "tenant-a-server", ); @@ -1883,6 +2247,209 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj ); } + #[test] + fn managed_certificate_manifest_includes_sni_hosts() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + let cert_manager = CertManagerTlsConfig { + secret_name: Some("tenant-a-public-tls".to_string()), + dns_names: vec!["custom.example.com".to_string()], + include_generated_dns_names: false, + ..Default::default() + }; + + let certificate = build_cert_manager_certificate( + &tenant, + "storage", + &cert_manager, + &["s3.example.com".to_string()], + "tenant-a-public-tls", + "tenant-a-public-tls", + ); + let dns_names = certificate + .data + .pointer("/spec/dnsNames") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(ToString::to_string) + .collect::>() + }); + + assert_eq!( + dns_names, + Some(vec![ + "custom.example.com".to_string(), + "s3.example.com".to_string() + ]) + ); + } + + #[test] + fn multi_certificate_entries_validate_default_and_hosts() { + let valid = TlsConfig { + mode: TlsMode::CertManager, + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig { + secret_name: Some("internal-tls".to_string()), + ..Default::default() + }, + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + secret_name: Some("public-tls".to_string()), + ..Default::default() + }, + }, + ], + ..Default::default() + }; + + let entries = certificate_entries(&valid).expect("valid multi-cert config should pass"); + + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].name, "internal"); + assert!(entries[0].default); + assert_eq!(entries[1].hosts, vec!["s3.example.com".to_string()]); + + let ip_host = TlsConfig { + mode: TlsMode::CertManager, + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["127.0.0.1".to_string()], + cert_manager: CertManagerTlsConfig::default(), + }, + ], + ..Default::default() + }; + let failure = certificate_entries(&ip_host).expect_err("SNI host should reject IP values"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!(failure.message.contains("concrete DNS name")); + + let missing_host = TlsConfig { + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec![], + cert_manager: CertManagerTlsConfig::default(), + }, + ], + ..valid + }; + let failure = + certificate_entries(&missing_host).expect_err("non-default cert requires hosts"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!(failure.message.contains("must set hosts")); + } + + #[test] + fn multi_certificate_status_keeps_default_fields_and_lists_all_certs() { + let config = TlsConfig { + mode: TlsMode::CertManager, + certificates: vec![ + TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: vec![], + cert_manager: CertManagerTlsConfig { + secret_name: Some("internal-tls".to_string()), + ..Default::default() + }, + }, + TlsCertificateConfig { + name: "public".to_string(), + default: false, + hosts: vec!["s3.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + secret_name: Some("public-tls".to_string()), + ..Default::default() + }, + }, + ], + ..Default::default() + }; + let observed = vec![ + observed_tls_certificate( + "internal", + true, + vec![], + "internal-tls", + tls_secret( + "internal-tls", + "7", + Some("kubernetes.io/tls"), + true, + true, + Some(PUBLIC_CERT_PEM), + ), + config.certificates[0].cert_manager.clone(), + vec!["tenant-a-io.storage.svc"], + ), + observed_tls_certificate( + "public", + false, + vec!["s3.example.com"], + "public-tls", + tls_secret( + "public-tls", + "9", + Some("kubernetes.io/tls"), + true, + true, + None, + ), + config.certificates[1].cert_manager.clone(), + vec!["s3.example.com"], + ), + ]; + + let status = cert_manager_tls_status(&config, &observed, None, None, "sha256:multi"); + + assert_eq!( + status + .server_secret_ref + .as_ref() + .map(|secret| secret.name.as_str()), + Some("internal-tls") + ); + assert_eq!(status.certificates.len(), 2); + assert!(status.certificates[0].default); + assert_eq!(status.certificates[1].hosts, vec!["s3.example.com"]); + assert_eq!( + status.certificates[1] + .server_secret_ref + .resource_version + .as_deref(), + Some("9") + ); + } + #[test] fn certificate_observation_requires_ready_condition_for_current_generation() { let ready = certificate_object( @@ -2036,6 +2603,30 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj } } + fn observed_tls_certificate( + name: &str, + default: bool, + hosts: Vec<&str>, + secret_name: &str, + secret: Secret, + cert_manager: CertManagerTlsConfig, + san_dns_names: Vec<&str>, + ) -> ObservedTlsCertificate { + ObservedTlsCertificate { + entry: TlsCertificateEntry { + name: name.to_string(), + default, + hosts: hosts.into_iter().map(ToString::to_string).collect(), + cert_manager, + legacy: false, + }, + secret_name: secret_name.to_string(), + secret, + certificate_ref: None, + san_dns_names: san_dns_names.into_iter().map(ToString::to_string).collect(), + } + } + fn certificate_object(name: &str, generation: Option, data: Value) -> DynamicObject { let mut object = DynamicObject::new(name, &certificate_api_resource()).data(data); object.metadata.generation = generation; diff --git a/src/types/v1alpha1/status/certificate.rs b/src/types/v1alpha1/status/certificate.rs index eb62f18..836fac0 100755 --- a/src/types/v1alpha1/status/certificate.rs +++ b/src/types/v1alpha1/status/certificate.rs @@ -50,6 +50,9 @@ pub struct TlsCertificateStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub server_secret_ref: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub certificates: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] pub ca_secret_ref: Option, @@ -93,6 +96,29 @@ pub struct TlsCertificateStatus { pub last_error_message: Option, } +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TlsServerCertificateStatus { + pub name: String, + + #[serde(default)] + pub default: bool, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hosts: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub managed_certificate: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub certificate_ref: Option, + + pub server_secret_ref: SecretStatusRef, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dns_names: Vec, +} + #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub struct CertificateObjectRef { diff --git a/src/types/v1alpha1/tenant/workloads.rs b/src/types/v1alpha1/tenant/workloads.rs index 89f9a85..43ad03e 100755 --- a/src/types/v1alpha1/tenant/workloads.rs +++ b/src/types/v1alpha1/tenant/workloads.rs @@ -1052,6 +1052,29 @@ mod tests { .iter() .any(|volume| volume.name == "rustfs-tls-server") ); + let server_volume = volumes + .iter() + .find(|volume| volume.name == "rustfs-tls-server") + .expect("TLS server volume should exist"); + let projected_items = server_volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .expect("TLS server volume should be projected") + .iter() + .flat_map(|source| { + source + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .into_iter() + .flatten() + }) + .map(|item| (item.key.as_str(), item.path.as_str())) + .collect::>(); + assert!(projected_items.contains(&("tls.crt", "rustfs_cert.pem"))); + assert!(projected_items.contains(&("tls.key", "rustfs_key.pem"))); + assert!(projected_items.contains(&("ca.crt", "ca.crt"))); let container = &pod_spec.containers[0]; let env = container.env.as_ref().expect("TLS env should be present"); @@ -1072,18 +1095,8 @@ mod tests { .expect("TLS volume mounts should be present"); assert!(mounts.iter().any(|mount| { mount.name == "rustfs-tls-server" - && mount.mount_path == "/var/run/rustfs/tls/rustfs_cert.pem" - && mount.sub_path.as_deref() == Some("rustfs_cert.pem") - })); - assert!(mounts.iter().any(|mount| { - mount.name == "rustfs-tls-server" - && mount.mount_path == "/var/run/rustfs/tls/rustfs_key.pem" - && mount.sub_path.as_deref() == Some("rustfs_key.pem") - })); - assert!(mounts.iter().any(|mount| { - mount.name == "rustfs-tls-server" - && mount.mount_path == "/var/run/rustfs/tls/ca.crt" - && mount.sub_path.as_deref() == Some("ca.crt") + && mount.mount_path == "/var/run/rustfs/tls" + && mount.sub_path.is_none() })); assert_eq!( diff --git a/src/types/v1alpha1/tls.rs b/src/types/v1alpha1/tls.rs index d8c603d..ab368d1 100644 --- a/src/types/v1alpha1/tls.rs +++ b/src/types/v1alpha1/tls.rs @@ -166,6 +166,20 @@ pub struct CertManagerTlsConfig { pub ca_trust: Option, } +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TlsCertificateConfig { + pub name: String, + + #[serde(default)] + pub default: bool, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hosts: Vec, + + pub cert_manager: CertManagerTlsConfig, +} + fn default_include_generated_dns_names() -> bool { true } @@ -210,6 +224,12 @@ pub struct TlsConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub cert_manager: Option, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub certificates: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_trust: Option, } fn default_tls_mount_path() -> String { @@ -229,6 +249,8 @@ impl Default for TlsConfig { enable_internode_https: false, require_san_match: default_require_san_match(), cert_manager: None, + certificates: Vec::new(), + ca_trust: None, } } } @@ -239,9 +261,22 @@ impl TlsConfig { } pub fn ca_trust(&self) -> CaTrustConfig { - self.cert_manager - .as_ref() - .and_then(|cert_manager| cert_manager.ca_trust.clone()) + if let Some(ca_trust) = self.ca_trust.clone() { + return ca_trust; + } + + if self.certificates.is_empty() { + return self + .cert_manager + .as_ref() + .and_then(|cert_manager| cert_manager.ca_trust.clone()) + .unwrap_or_default(); + } + + self.certificates + .iter() + .find(|certificate| certificate.default) + .and_then(|certificate| certificate.cert_manager.ca_trust.clone()) .unwrap_or_default() } } @@ -260,7 +295,120 @@ mod tests { assert!(!config.enable_internode_https); assert!(config.require_san_match); assert!(config.cert_manager.is_none()); + assert!(config.certificates.is_empty()); + assert!(config.ca_trust.is_none()); } + + #[test] + fn multi_certificate_ca_trust_ignores_legacy_cert_manager() { + let config = TlsConfig { + cert_manager: Some(CertManagerTlsConfig { + ca_trust: Some(CaTrustConfig { + source: CaTrustSource::SystemCa, + ..Default::default() + }), + ..Default::default() + }), + certificates: vec![TlsCertificateConfig { + name: "internal".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig { + ca_trust: Some(CaTrustConfig { + source: CaTrustSource::SecretRef, + ca_secret_ref: Some(SecretKeyReference { + name: "internal-ca".to_string(), + key: "bundle.pem".to_string(), + }), + ..Default::default() + }), + ..Default::default() + }, + }], + ..Default::default() + }; + + let ca_trust = config.ca_trust(); + + assert_eq!(ca_trust.source, CaTrustSource::SecretRef); + assert_eq!( + ca_trust + .ca_secret_ref + .as_ref() + .map(|secret| { (secret.name.as_str(), secret.key.as_str()) }), + Some(("internal-ca", "bundle.pem")) + ); + } + + #[test] + fn tls_plan_projects_multiple_certificates_into_rustfs_sni_layout() { + let plan = TlsPlan::rollout_certificates( + DEFAULT_TLS_MOUNT_PATH.to_string(), + "sha256:multi".to_string(), + vec![ + TlsServerCertificateMount { + secret_name: "internal-tls".to_string(), + domains: vec![None, Some("rustfs.internal.example.local".to_string())], + ca_key: Some("ca.crt".to_string()), + }, + TlsServerCertificateMount { + secret_name: "public-tls".to_string(), + domains: vec![Some("s3.example.com".to_string())], + ca_key: None, + }, + ], + None, + None, + true, + false, + false, + None, + ); + + let volume = plan + .volumes + .iter() + .find(|volume| volume.name == "rustfs-tls-server") + .expect("TLS volume should exist"); + let paths = volume + .projected + .as_ref() + .and_then(|projected| projected.sources.as_ref()) + .expect("TLS volume should use projected sources") + .iter() + .flat_map(|source| { + source + .secret + .as_ref() + .and_then(|secret| secret.items.as_ref()) + .into_iter() + .flatten() + }) + .map(|item| item.path.as_str()) + .collect::>(); + + assert!(paths.contains(&"rustfs_cert.pem")); + assert!(paths.contains(&"rustfs_key.pem")); + assert!(paths.contains(&"ca.crt")); + assert!(paths.contains(&"rustfs.internal.example.local/rustfs_cert.pem")); + assert!(paths.contains(&"rustfs.internal.example.local/rustfs_key.pem")); + assert!(paths.contains(&"s3.example.com/rustfs_cert.pem")); + assert!(paths.contains(&"s3.example.com/rustfs_key.pem")); + assert_eq!( + plan.volume_mounts + .iter() + .find(|mount| mount.name == "rustfs-tls-server") + .map(|mount| (mount.mount_path.as_str(), mount.sub_path.as_deref())), + Some((DEFAULT_TLS_MOUNT_PATH, None)) + ); + } +} + +#[derive(Clone, Debug, Default)] +pub struct TlsServerCertificateMount { + pub secret_name: String, + pub domains: Vec>, + pub ca_key: Option, } #[derive(Clone, Debug, Default)] @@ -303,6 +451,35 @@ impl TlsPlan { trust_system_ca: bool, trust_leaf_certificate_as_ca: bool, status: Option, + ) -> Self { + Self::rollout_certificates( + mount_path, + hash, + vec![TlsServerCertificateMount { + secret_name: server_secret_name, + domains: vec![None], + ca_key: server_ca_key, + }], + explicit_ca, + client_ca, + enable_internode_https, + trust_system_ca, + trust_leaf_certificate_as_ca, + status, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn rollout_certificates( + mount_path: String, + hash: String, + server_certificates: Vec, + explicit_ca: Option, + client_ca: Option, + enable_internode_https: bool, + trust_system_ca: bool, + trust_leaf_certificate_as_ca: bool, + status: Option, ) -> Self { let mut annotations = BTreeMap::new(); annotations.insert(TLS_HASH_ANNOTATION.to_string(), hash); @@ -334,63 +511,32 @@ impl TlsPlan { }); } - let has_server_ca = server_ca_key.is_some(); - let mut server_items = vec![ - key_to_path("tls.crt", RUSTFS_TLS_CERT_FILE), - key_to_path("tls.key", RUSTFS_TLS_KEY_FILE), - ]; - if let Some(ca_key) = server_ca_key.as_deref() { - server_items.push(key_to_path(ca_key, RUSTFS_CA_FILE)); + let mut sources = server_certificates + .iter() + .map(|certificate| { + secret_projection( + &certificate.secret_name, + server_certificate_items(certificate), + ) + }) + .collect::>(); + + if let Some(explicit_ca) = &explicit_ca { + sources.push(secret_projection( + &explicit_ca.name, + vec![key_to_path(&explicit_ca.key, RUSTFS_CA_FILE)], + )); } - - let (mut volumes, mut volume_mounts) = if let Some(explicit_ca) = &explicit_ca { - ( - vec![projected_volume( - "rustfs-tls-server", - vec![ - secret_projection(&server_secret_name, server_items), - secret_projection( - &explicit_ca.name, - vec![key_to_path(&explicit_ca.key, RUSTFS_CA_FILE)], - ), - ], - )], - vec![directory_mount("rustfs-tls-server", &mount_path)], - ) - } else { - let mut volume_mounts = vec![ - file_mount("rustfs-tls-server", &mount_path, RUSTFS_TLS_CERT_FILE), - file_mount("rustfs-tls-server", &mount_path, RUSTFS_TLS_KEY_FILE), - ]; - if has_server_ca { - volume_mounts.push(file_mount("rustfs-tls-server", &mount_path, RUSTFS_CA_FILE)); - } - ( - vec![secret_volume( - "rustfs-tls-server", - &server_secret_name, - server_items, - )], - volume_mounts, - ) - }; - if let Some(client_ca) = &client_ca { - volumes.push(secret_volume( - "rustfs-tls-client-ca", + sources.push(secret_projection( &client_ca.name, vec![key_to_path(&client_ca.key, RUSTFS_CLIENT_CA_FILE)], )); - volume_mounts.push(file_mount( - "rustfs-tls-client-ca", - &mount_path, - RUSTFS_CLIENT_CA_FILE, - )); } Self { enabled: true, - mount_path, + mount_path: mount_path.clone(), internode_scheme: if enable_internode_https { "https" } else { @@ -399,8 +545,8 @@ impl TlsPlan { probe_scheme: "HTTPS", pod_template_annotations: annotations, env, - volumes, - volume_mounts, + volumes: vec![projected_volume("rustfs-tls-server", sources)], + volume_mounts: vec![directory_mount("rustfs-tls-server", &mount_path)], status, } } @@ -422,17 +568,31 @@ impl TlsPlan { } } -fn secret_volume(name: &str, secret_name: &str, items: Vec) -> corev1::Volume { - corev1::Volume { - name: name.to_string(), - secret: Some(corev1::SecretVolumeSource { - secret_name: Some(secret_name.to_string()), - items: Some(items), - optional: Some(false), - ..Default::default() - }), - ..Default::default() +fn server_certificate_items(certificate: &TlsServerCertificateMount) -> Vec { + let mut items = Vec::new(); + for domain in &certificate.domains { + items.push(key_to_path("tls.crt", &server_cert_path(domain.as_deref()))); + items.push(key_to_path("tls.key", &server_key_path(domain.as_deref()))); + } + if let Some(ca_key) = certificate.ca_key.as_deref() { + items.push(key_to_path(ca_key, RUSTFS_CA_FILE)); } + items +} + +fn server_cert_path(domain: Option<&str>) -> String { + rustfs_tls_path(domain, RUSTFS_TLS_CERT_FILE) +} + +fn server_key_path(domain: Option<&str>) -> String { + rustfs_tls_path(domain, RUSTFS_TLS_KEY_FILE) +} + +fn rustfs_tls_path(domain: Option<&str>, file: &str) -> String { + domain + .filter(|domain| !domain.is_empty()) + .map(|domain| format!("{domain}/{file}")) + .unwrap_or_else(|| file.to_string()) } fn projected_volume(name: &str, sources: Vec) -> corev1::Volume { @@ -465,16 +625,6 @@ fn key_to_path(key: &str, path: &str) -> corev1::KeyToPath { } } -fn file_mount(volume_name: &str, mount_path: &str, file_name: &str) -> corev1::VolumeMount { - corev1::VolumeMount { - name: volume_name.to_string(), - mount_path: format!("{}/{}", mount_path.trim_end_matches('/'), file_name), - sub_path: Some(file_name.to_string()), - read_only: Some(true), - ..Default::default() - } -} - fn directory_mount(volume_name: &str, mount_path: &str) -> corev1::VolumeMount { corev1::VolumeMount { name: volume_name.to_string(),