-
Notifications
You must be signed in to change notification settings - Fork 72
fix: Use merged DWOC config for HTTP client certificate initialization #1645
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
9c13dbf
de8aa1d
b1b3bac
d1fa4a5
8e5c4d5
2669f17
6fb6b83
1348318
271ee31
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| // Copyright (c) 2019-2025 Red Hat, Inc. | ||
| // Copyright (c) 2019-2026 Red Hat, Inc. | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
|
|
@@ -17,12 +17,15 @@ import ( | |
| "context" | ||
| "crypto/tls" | ||
| "crypto/x509" | ||
| "fmt" | ||
| "net/http" | ||
| "net/url" | ||
| "reflect" | ||
| "sync" | ||
| "time" | ||
|
|
||
| controller "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1" | ||
| "github.com/devfile/devworkspace-operator/pkg/config" | ||
|
|
||
| "k8s.io/apimachinery/pkg/types" | ||
|
|
||
| "github.com/go-logr/logr" | ||
|
|
@@ -32,87 +35,253 @@ import ( | |
| "golang.org/x/net/http/httpproxy" | ||
| ) | ||
|
|
||
| var ( | ||
| httpClient *http.Client | ||
| var httpClientsHolder HttpClientsHolder | ||
|
|
||
| type HttpClientsHolder interface { | ||
| GetHttpClient() *http.Client | ||
|
|
||
| // GetHealthCheckHttpClient returns an HTTP client that skips TLS verification. | ||
| // This client MUST only be used for workspace health/readiness checks, not for | ||
| // fetching external content or making security-sensitive requests. | ||
| GetHealthCheckHttpClient() *http.Client | ||
|
|
||
| ConfigureHttpClients(context.Context, *controller.RoutingConfig) | ||
| } | ||
|
|
||
| type DefaultHttpClientsHolder struct { | ||
| k8s client.Client | ||
| logger logr.Logger | ||
|
|
||
| client *http.Client | ||
| healthCheckHttpClient *http.Client | ||
| ) | ||
|
|
||
| func setupHttpClients(k8s client.Client, logger logr.Logger) { | ||
| transport := http.DefaultTransport.(*http.Transport).Clone() | ||
| healthCheckTransport := http.DefaultTransport.(*http.Transport).Clone() | ||
| healthCheckTransport.TLSClientConfig = &tls.Config{ | ||
| InsecureSkipVerify: true, | ||
| mu sync.RWMutex | ||
|
|
||
| lastProxyConfig *controller.Proxy | ||
| lastCertsCMVersion string | ||
|
|
||
| systemCertPool *x509.CertPool | ||
| } | ||
|
|
||
| func SetupHttpClients(k8s client.Client, logger logr.Logger) error { | ||
| systemCertPool, err := x509.SystemCertPool() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to load system cert pool: %w", err) | ||
| } | ||
|
|
||
| globalConfig := config.GetGlobalConfig() | ||
| httpClientsHolder = &DefaultHttpClientsHolder{ | ||
| k8s: k8s, | ||
| logger: logger, | ||
| systemCertPool: systemCertPool, | ||
| } | ||
| httpClientsHolder.ConfigureHttpClients(context.Background(), config.GetGlobalConfig().Routing) | ||
|
|
||
| if globalConfig.Routing != nil && globalConfig.Routing.ProxyConfig != nil { | ||
| proxyConf := httpproxy.Config{} | ||
| if globalConfig.Routing.ProxyConfig.HttpProxy != nil { | ||
| proxyConf.HTTPProxy = *globalConfig.Routing.ProxyConfig.HttpProxy | ||
| return nil | ||
| } | ||
|
|
||
| func (h *DefaultHttpClientsHolder) GetHttpClient() *http.Client { | ||
| h.mu.RLock() | ||
| defer h.mu.RUnlock() | ||
|
|
||
| return h.client | ||
| } | ||
|
|
||
| func (h *DefaultHttpClientsHolder) GetHealthCheckHttpClient() *http.Client { | ||
| h.mu.RLock() | ||
| defer h.mu.RUnlock() | ||
|
|
||
| return h.healthCheckHttpClient | ||
| } | ||
|
|
||
| func (h *DefaultHttpClientsHolder) ConfigureHttpClients(ctx context.Context, routingConfig *controller.RoutingConfig) { | ||
| var newProxyConfig *controller.Proxy | ||
| var newCertsCM *corev1.ConfigMap | ||
|
|
||
| if routingConfig != nil { | ||
| if routingConfig.ProxyConfig != nil { | ||
| newProxyConfig = routingConfig.ProxyConfig | ||
| } | ||
| if globalConfig.Routing.ProxyConfig.HttpsProxy != nil { | ||
| proxyConf.HTTPSProxy = *globalConfig.Routing.ProxyConfig.HttpsProxy | ||
|
|
||
| if routingConfig.TLSCertificateConfigmapRef != nil { | ||
| certsCM, err := h.readCertCM(ctx, routingConfig.TLSCertificateConfigmapRef) | ||
| if err != nil { | ||
| h.logger.Error(err, "Failed to read TLS certificate ConfigMap") | ||
|
|
||
| // certsCM == nil, | ||
| // http clients will be rebuilt with a system cert pool, not an issue at all | ||
| } | ||
|
|
||
| newCertsCM = certsCM | ||
| } | ||
| if globalConfig.Routing.ProxyConfig.NoProxy != nil { | ||
| proxyConf.NoProxy = *globalConfig.Routing.ProxyConfig.NoProxy | ||
| } | ||
|
|
||
| buildNewHttpClient, buildNewHealthCheckHttpClient := h.shouldRebuildClients(newProxyConfig, newCertsCM) | ||
|
|
||
| if buildNewHttpClient || buildNewHealthCheckHttpClient { | ||
| newClient, newHealthCheckClient := h.buildNewClients( | ||
| buildNewHttpClient, | ||
| buildNewHealthCheckHttpClient, | ||
| newProxyConfig, | ||
| newCertsCM, | ||
| ) | ||
|
|
||
| h.setNewClients( | ||
| newClient, | ||
| newHealthCheckClient, | ||
| newProxyConfig, | ||
| newCertsCM, | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| func (h *DefaultHttpClientsHolder) shouldRebuildClients(newProxyConfig *controller.Proxy, newCertsCM *corev1.ConfigMap) (bool, bool) { | ||
| h.mu.RLock() | ||
| defer h.mu.RUnlock() | ||
|
|
||
| // Always rebuild if clients haven't been initialized yet | ||
| if h.client == nil || h.healthCheckHttpClient == nil { | ||
| return true, true | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Race condition: TOCTOU gap between shouldRebuildClients and setNewClients
Suggested fix: Use a single Lock for the entire |
||
| } | ||
|
|
||
| if !reflect.DeepEqual(newProxyConfig, h.lastProxyConfig) { | ||
| return true, true | ||
| } | ||
|
|
||
| certsCMVersion := "" | ||
| if newCertsCM != nil { | ||
| certsCMVersion = newCertsCM.ResourceVersion | ||
| } | ||
|
|
||
| if certsCMVersion != h.lastCertsCMVersion { | ||
|
tolusha marked this conversation as resolved.
|
||
| return true, false | ||
| } | ||
|
tolusha marked this conversation as resolved.
|
||
|
|
||
| return false, false | ||
|
tolusha marked this conversation as resolved.
|
||
| } | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Comment contradicts actual behavior The comment says "not an issue at all" but this IS an issue - the operator was configured to trust custom certificates. Rebuilding without them on transient failure breaks the trust chain. Consider updating this comment after fixing the transient failure handling. |
||
|
|
||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Critical: Transient ConfigMap failure silently strips custom certificates When A single transient Kubernetes API error will break TLS trust cluster-wide for all subsequent workspace operations. Suggested fix: On if routingConfig.TLSCertificateConfigmapRef != nil {
certsCM, err := h.readCertCM(ctx, routingConfig.TLSCertificateConfigmapRef)
if err != nil {
h.logger.Error(err, "Failed to read TLS certificate ConfigMap")
// Preserve existing HTTP clients on transient failure
return
}
newCertsCM = certsCM
} |
||
| func (h *DefaultHttpClientsHolder) buildNewClients( | ||
|
tolusha marked this conversation as resolved.
|
||
| buildNewHttpClient bool, | ||
| buildNewHealthCheckHttpClient bool, | ||
| newProxyConfig *controller.Proxy, | ||
| newCertsCM *corev1.ConfigMap, | ||
| ) (*http.Client, *http.Client) { | ||
|
|
||
| var newClient *http.Client | ||
| var newHealthCheckClient *http.Client | ||
|
|
||
| proxyFunc := h.getProxyFunc(newProxyConfig) | ||
| caCertPool := h.getCaCertPool(newCertsCM) | ||
|
|
||
| if buildNewHttpClient { | ||
| transport := http.DefaultTransport.(*http.Transport).Clone() | ||
| transport.Proxy = proxyFunc | ||
| transport.TLSClientConfig = &tls.Config{ | ||
| RootCAs: caCertPool, | ||
| } | ||
|
|
||
| proxyFunc := func(req *http.Request) (*url.URL, error) { | ||
| return proxyConf.ProxyFunc()(req.URL) | ||
| newClient = &http.Client{ | ||
| Transport: transport, | ||
| Timeout: 5 * time.Second, | ||
| } | ||
| transport.Proxy = proxyFunc | ||
| } | ||
|
|
||
|
tolusha marked this conversation as resolved.
|
||
| if buildNewHealthCheckHttpClient { | ||
| healthCheckTransport := http.DefaultTransport.(*http.Transport).Clone() | ||
| healthCheckTransport.Proxy = proxyFunc | ||
| healthCheckTransport.TLSClientConfig = &tls.Config{ | ||
| InsecureSkipVerify: true, | ||
| } | ||
|
|
||
| newHealthCheckClient = &http.Client{ | ||
| Transport: healthCheckTransport, | ||
| Timeout: 500 * time.Millisecond, | ||
| } | ||
| } | ||
|
|
||
| return newClient, newHealthCheckClient | ||
| } | ||
|
|
||
| func (h *DefaultHttpClientsHolder) setNewClients( | ||
| newClient *http.Client, | ||
| newHealthCheckClient *http.Client, | ||
| newProxyConfig *controller.Proxy, | ||
| newCertsCM *corev1.ConfigMap, | ||
| ) { | ||
| h.mu.Lock() | ||
| defer h.mu.Unlock() | ||
|
|
||
| if newClient != nil { | ||
| h.client = newClient | ||
| } | ||
|
|
||
| if newHealthCheckClient != nil { | ||
| h.healthCheckHttpClient = newHealthCheckClient | ||
| } | ||
|
|
||
| httpClient = &http.Client{ | ||
| Transport: transport, | ||
| if newProxyConfig != nil { | ||
| h.lastProxyConfig = newProxyConfig.DeepCopy() | ||
| } else { | ||
| h.lastProxyConfig = nil | ||
| } | ||
| healthCheckHttpClient = &http.Client{ | ||
| Transport: healthCheckTransport, | ||
| Timeout: 500 * time.Millisecond, | ||
|
|
||
| if newCertsCM != nil { | ||
| h.lastCertsCMVersion = newCertsCM.ResourceVersion | ||
| } else { | ||
| h.lastCertsCMVersion = "" | ||
| } | ||
| InjectCertificates(k8s, logger) | ||
| } | ||
|
|
||
| func InjectCertificates(k8s client.Client, logger logr.Logger) { | ||
| if certs, ok := readCertificates(k8s, logger); ok { | ||
| for _, certsPem := range certs { | ||
| injectCertificates([]byte(certsPem), httpClient.Transport.(*http.Transport), logger) | ||
| func (h *DefaultHttpClientsHolder) getProxyFunc(proxyConfig *controller.Proxy) func(*http.Request) (*url.URL, error) { | ||
|
tolusha marked this conversation as resolved.
|
||
| if proxyConfig != nil { | ||
| proxyConf := httpproxy.Config{} | ||
| if proxyConfig.HttpProxy != nil { | ||
| proxyConf.HTTPProxy = *proxyConfig.HttpProxy | ||
| } | ||
| if proxyConfig.HttpsProxy != nil { | ||
| proxyConf.HTTPSProxy = *proxyConfig.HttpsProxy | ||
| } | ||
| if proxyConfig.NoProxy != nil { | ||
|
tolusha marked this conversation as resolved.
|
||
| proxyConf.NoProxy = *proxyConfig.NoProxy | ||
| } | ||
|
|
||
| return func(req *http.Request) (*url.URL, error) { | ||
| return proxyConf.ProxyFunc()(req.URL) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func readCertificates(k8s client.Client, logger logr.Logger) (map[string]string, bool) { | ||
| configmapRef := config.GetGlobalConfig().Routing.TLSCertificateConfigmapRef | ||
| if configmapRef == nil { | ||
| return nil, false | ||
| func (h *DefaultHttpClientsHolder) getCaCertPool(cm *corev1.ConfigMap) *x509.CertPool { | ||
| if cm == nil { | ||
| return nil | ||
| } | ||
| configMap := &corev1.ConfigMap{} | ||
| namespacedName := &types.NamespacedName{ | ||
| Name: configmapRef.Name, | ||
| Namespace: configmapRef.Namespace, | ||
| } | ||
| err := k8s.Get(context.Background(), *namespacedName, configMap) | ||
| if err != nil { | ||
| logger.Error(err, "Failed to read configmap with certificates") | ||
| return nil, false | ||
|
|
||
| caCertPool := h.systemCertPool.Clone() | ||
|
|
||
| for _, certsPem := range cm.Data { | ||
| if !caCertPool.AppendCertsFromPEM([]byte(certsPem)) { | ||
| h.logger.V(1).Info("Warning: failed to parse one or more certificates from ConfigMap") | ||
| } | ||
| } | ||
| return configMap.Data, true | ||
|
|
||
| return caCertPool | ||
| } | ||
|
|
||
| func injectCertificates(certsPem []byte, transport *http.Transport, logger logr.Logger) { | ||
| caCertPool := transport.TLSClientConfig.RootCAs | ||
| if caCertPool == nil { | ||
| systemCertPool, err := x509.SystemCertPool() | ||
| if err != nil { | ||
| logger.Error(err, "Failed to load system cert pool") | ||
| caCertPool = x509.NewCertPool() | ||
| } else { | ||
| caCertPool = systemCertPool | ||
| } | ||
| func (h *DefaultHttpClientsHolder) readCertCM(ctx context.Context, cmReference *controller.ConfigmapReference) (*corev1.ConfigMap, error) { | ||
| if cmReference == nil { | ||
| return nil, nil | ||
| } | ||
| if ok := caCertPool.AppendCertsFromPEM(certsPem); ok { | ||
| transport.TLSClientConfig = &tls.Config{RootCAs: caCertPool} | ||
|
|
||
| namespacedName := types.NamespacedName{ | ||
| Name: cmReference.Name, | ||
| Namespace: cmReference.Namespace, | ||
| } | ||
|
|
||
| configMap := &corev1.ConfigMap{} | ||
| if err := h.k8s.Get(ctx, namespacedName, configMap); err != nil { | ||
| return nil, fmt.Errorf("failed to read ConfigMap %s/%s containing certificates: %w", cmReference.Namespace, cmReference.Name, err) | ||
| } | ||
|
|
||
| return configMap, nil | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
System-level concern: Shared HTTP client with per-workspace config
The
httpClientsHolderis shared across ALL concurrent reconciliations. If different workspaces have different mergedTLSCertificateConfigmapRefvalues, the "last reconciled workspace wins" for HTTP client configuration.Needs verification: Does DWOC merging always produce identical routing config regardless of workspace? Or can per-namespace overrides create different cert references?
If configs can differ, consider scoping the HTTP client holder per-namespace or per-config-hash.