Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions controllers/workspace/devworkspace_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,12 +141,11 @@ func (r *DevWorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request
workspace.DevWorkspace = rawWorkspace
workspace.Config = config

httpClientsHolder.ConfigureHttpClients(ctx, config.Routing)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

System-level concern: Shared HTTP client with per-workspace config

The httpClientsHolder is shared across ALL concurrent reconciliations. If different workspaces have different merged TLSCertificateConfigmapRef values, the "last reconciled workspace wins" for HTTP client configuration.

Needs verification: Does DWOC merging always produce identical routing config regardless of workspace? Or can per-namespace overrides create different cert references?

If configs can differ, consider scoping the HTTP client holder per-namespace or per-config-hash.


reqLogger = reqLogger.WithValues(constants.DevWorkspaceIDLoggerKey, workspace.Status.DevWorkspaceId)
reqLogger.Info("Reconciling Workspace", "resolvedConfig", configString)

// Inject ca certificates to the http client, if the certificates configmap is created and defined in the config.
InjectCertificates(r.Client, r.Log)

// Check if the DevWorkspaceRouting instance is marked to be deleted, which is
// indicated by the deletion timestamp being set.
if workspace.GetDeletionTimestamp() != nil {
Expand Down Expand Up @@ -264,7 +263,7 @@ func (r *DevWorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request
WorkspaceNamespace: workspace.Namespace,
Context: ctx,
K8sClient: r.Client,
HttpClient: httpClient,
HttpClient: httpClientsHolder.GetHttpClient(),
DefaultResourceRequirements: workspace.Config.Workspace.DefaultContainerResources,
}

Expand Down Expand Up @@ -788,7 +787,10 @@ func (r *DevWorkspaceReconciler) getWorkspaceId(ctx context.Context, workspace *
}

func (r *DevWorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
setupHttpClients(mgr.GetClient(), mgr.GetLogger())
err := SetupHttpClients(mgr.GetClient(), mgr.GetLogger())
if err != nil {
return err
}

maxConcurrentReconciles, err := wkspConfig.GetMaxConcurrentReconciles()
if err != nil {
Expand Down
285 changes: 227 additions & 58 deletions controllers/workspace/http.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2019-2025 Red Hat, Inc.
// Copyright (c) 2019-2026 Red Hat, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
Expand All @@ -17,12 +17,15 @@ import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"net/url"
"reflect"
"sync"
"time"

controller "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
"github.com/devfile/devworkspace-operator/pkg/config"

"k8s.io/apimachinery/pkg/types"

"github.com/go-logr/logr"
Expand All @@ -32,87 +35,253 @@ import (
"golang.org/x/net/http/httpproxy"
)

var (
httpClient *http.Client
var httpClientsHolder HttpClientsHolder

type HttpClientsHolder interface {
GetHttpClient() *http.Client

// GetHealthCheckHttpClient returns an HTTP client that skips TLS verification.
// This client MUST only be used for workspace health/readiness checks, not for
// fetching external content or making security-sensitive requests.
GetHealthCheckHttpClient() *http.Client

ConfigureHttpClients(context.Context, *controller.RoutingConfig)
}

type DefaultHttpClientsHolder struct {
k8s client.Client
logger logr.Logger

client *http.Client
healthCheckHttpClient *http.Client
)

func setupHttpClients(k8s client.Client, logger logr.Logger) {
transport := http.DefaultTransport.(*http.Transport).Clone()
healthCheckTransport := http.DefaultTransport.(*http.Transport).Clone()
healthCheckTransport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
mu sync.RWMutex

lastProxyConfig *controller.Proxy
lastCertsCMVersion string

systemCertPool *x509.CertPool
}

func SetupHttpClients(k8s client.Client, logger logr.Logger) error {
systemCertPool, err := x509.SystemCertPool()
if err != nil {
return fmt.Errorf("failed to load system cert pool: %w", err)
}

globalConfig := config.GetGlobalConfig()
httpClientsHolder = &DefaultHttpClientsHolder{
k8s: k8s,
logger: logger,
systemCertPool: systemCertPool,
}
httpClientsHolder.ConfigureHttpClients(context.Background(), config.GetGlobalConfig().Routing)

if globalConfig.Routing != nil && globalConfig.Routing.ProxyConfig != nil {
proxyConf := httpproxy.Config{}
if globalConfig.Routing.ProxyConfig.HttpProxy != nil {
proxyConf.HTTPProxy = *globalConfig.Routing.ProxyConfig.HttpProxy
return nil
}

func (h *DefaultHttpClientsHolder) GetHttpClient() *http.Client {
h.mu.RLock()
defer h.mu.RUnlock()

return h.client
}

func (h *DefaultHttpClientsHolder) GetHealthCheckHttpClient() *http.Client {
h.mu.RLock()
defer h.mu.RUnlock()

return h.healthCheckHttpClient
}

func (h *DefaultHttpClientsHolder) ConfigureHttpClients(ctx context.Context, routingConfig *controller.RoutingConfig) {
var newProxyConfig *controller.Proxy
var newCertsCM *corev1.ConfigMap

if routingConfig != nil {
if routingConfig.ProxyConfig != nil {
newProxyConfig = routingConfig.ProxyConfig
}
if globalConfig.Routing.ProxyConfig.HttpsProxy != nil {
proxyConf.HTTPSProxy = *globalConfig.Routing.ProxyConfig.HttpsProxy

if routingConfig.TLSCertificateConfigmapRef != nil {
certsCM, err := h.readCertCM(ctx, routingConfig.TLSCertificateConfigmapRef)
if err != nil {
h.logger.Error(err, "Failed to read TLS certificate ConfigMap")

// certsCM == nil,
// http clients will be rebuilt with a system cert pool, not an issue at all
}

newCertsCM = certsCM
}
if globalConfig.Routing.ProxyConfig.NoProxy != nil {
proxyConf.NoProxy = *globalConfig.Routing.ProxyConfig.NoProxy
}

buildNewHttpClient, buildNewHealthCheckHttpClient := h.shouldRebuildClients(newProxyConfig, newCertsCM)

if buildNewHttpClient || buildNewHealthCheckHttpClient {
newClient, newHealthCheckClient := h.buildNewClients(
buildNewHttpClient,
buildNewHealthCheckHttpClient,
newProxyConfig,
newCertsCM,
)

h.setNewClients(
newClient,
newHealthCheckClient,
newProxyConfig,
newCertsCM,
)
}
}

func (h *DefaultHttpClientsHolder) shouldRebuildClients(newProxyConfig *controller.Proxy, newCertsCM *corev1.ConfigMap) (bool, bool) {
h.mu.RLock()
defer h.mu.RUnlock()

// Always rebuild if clients haven't been initialized yet
if h.client == nil || h.healthCheckHttpClient == nil {
return true, true

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Race condition: TOCTOU gap between shouldRebuildClients and setNewClients

ConfigureHttpClients calls shouldRebuildClients (RLock), then buildNewClients (no lock), then setNewClients (Lock). Between releasing the RLock and acquiring the Lock, another concurrent reconciliation could also pass the check and both would rebuild clients.

Suggested fix: Use a single Lock for the entire ConfigureHttpClients method to make check-and-set atomic.

}

if !reflect.DeepEqual(newProxyConfig, h.lastProxyConfig) {
return true, true
}

certsCMVersion := ""
if newCertsCM != nil {
certsCMVersion = newCertsCM.ResourceVersion
}

if certsCMVersion != h.lastCertsCMVersion {
Comment thread
tolusha marked this conversation as resolved.
return true, false
}
Comment thread
tolusha marked this conversation as resolved.

return false, false
Comment thread
tolusha marked this conversation as resolved.
}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Comment contradicts actual behavior

The comment says "not an issue at all" but this IS an issue - the operator was configured to trust custom certificates. Rebuilding without them on transient failure breaks the trust chain.

Consider updating this comment after fixing the transient failure handling.


Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Critical: Transient ConfigMap failure silently strips custom certificates

When readCertCM fails (transient API error, RBAC issue), newCertsCM becomes nil. The shouldRebuildClients method then detects a "change" and rebuilds the HTTP client with getCaCertPool(nil), stripping all custom CA certificates.

A single transient Kubernetes API error will break TLS trust cluster-wide for all subsequent workspace operations.

Suggested fix: On readCertCM failure, return early without rebuilding clients. Only rebuild when we successfully read a ConfigMap with a different ResourceVersion.

if routingConfig.TLSCertificateConfigmapRef != nil {
    certsCM, err := h.readCertCM(ctx, routingConfig.TLSCertificateConfigmapRef)
    if err != nil {
        h.logger.Error(err, "Failed to read TLS certificate ConfigMap")
        // Preserve existing HTTP clients on transient failure
        return
    }
    newCertsCM = certsCM
}

func (h *DefaultHttpClientsHolder) buildNewClients(
Comment thread
tolusha marked this conversation as resolved.
buildNewHttpClient bool,
buildNewHealthCheckHttpClient bool,
newProxyConfig *controller.Proxy,
newCertsCM *corev1.ConfigMap,
) (*http.Client, *http.Client) {

var newClient *http.Client
var newHealthCheckClient *http.Client

proxyFunc := h.getProxyFunc(newProxyConfig)
caCertPool := h.getCaCertPool(newCertsCM)

if buildNewHttpClient {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.Proxy = proxyFunc
transport.TLSClientConfig = &tls.Config{
RootCAs: caCertPool,
}

proxyFunc := func(req *http.Request) (*url.URL, error) {
return proxyConf.ProxyFunc()(req.URL)
newClient = &http.Client{
Transport: transport,
Timeout: 5 * time.Second,
}
transport.Proxy = proxyFunc
}

Comment thread
tolusha marked this conversation as resolved.
if buildNewHealthCheckHttpClient {
healthCheckTransport := http.DefaultTransport.(*http.Transport).Clone()
healthCheckTransport.Proxy = proxyFunc
healthCheckTransport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
}

newHealthCheckClient = &http.Client{
Transport: healthCheckTransport,
Timeout: 500 * time.Millisecond,
}
}

return newClient, newHealthCheckClient
}

func (h *DefaultHttpClientsHolder) setNewClients(
newClient *http.Client,
newHealthCheckClient *http.Client,
newProxyConfig *controller.Proxy,
newCertsCM *corev1.ConfigMap,
) {
h.mu.Lock()
defer h.mu.Unlock()

if newClient != nil {
h.client = newClient
}

if newHealthCheckClient != nil {
h.healthCheckHttpClient = newHealthCheckClient
}

httpClient = &http.Client{
Transport: transport,
if newProxyConfig != nil {
h.lastProxyConfig = newProxyConfig.DeepCopy()
} else {
h.lastProxyConfig = nil
}
healthCheckHttpClient = &http.Client{
Transport: healthCheckTransport,
Timeout: 500 * time.Millisecond,

if newCertsCM != nil {
h.lastCertsCMVersion = newCertsCM.ResourceVersion
} else {
h.lastCertsCMVersion = ""
}
InjectCertificates(k8s, logger)
}

func InjectCertificates(k8s client.Client, logger logr.Logger) {
if certs, ok := readCertificates(k8s, logger); ok {
for _, certsPem := range certs {
injectCertificates([]byte(certsPem), httpClient.Transport.(*http.Transport), logger)
func (h *DefaultHttpClientsHolder) getProxyFunc(proxyConfig *controller.Proxy) func(*http.Request) (*url.URL, error) {
Comment thread
tolusha marked this conversation as resolved.
if proxyConfig != nil {
proxyConf := httpproxy.Config{}
if proxyConfig.HttpProxy != nil {
proxyConf.HTTPProxy = *proxyConfig.HttpProxy
}
if proxyConfig.HttpsProxy != nil {
proxyConf.HTTPSProxy = *proxyConfig.HttpsProxy
}
if proxyConfig.NoProxy != nil {
Comment thread
tolusha marked this conversation as resolved.
proxyConf.NoProxy = *proxyConfig.NoProxy
}

return func(req *http.Request) (*url.URL, error) {
return proxyConf.ProxyFunc()(req.URL)
}
}

return nil
}

func readCertificates(k8s client.Client, logger logr.Logger) (map[string]string, bool) {
configmapRef := config.GetGlobalConfig().Routing.TLSCertificateConfigmapRef
if configmapRef == nil {
return nil, false
func (h *DefaultHttpClientsHolder) getCaCertPool(cm *corev1.ConfigMap) *x509.CertPool {
if cm == nil {
return nil
}
configMap := &corev1.ConfigMap{}
namespacedName := &types.NamespacedName{
Name: configmapRef.Name,
Namespace: configmapRef.Namespace,
}
err := k8s.Get(context.Background(), *namespacedName, configMap)
if err != nil {
logger.Error(err, "Failed to read configmap with certificates")
return nil, false

caCertPool := h.systemCertPool.Clone()

for _, certsPem := range cm.Data {
if !caCertPool.AppendCertsFromPEM([]byte(certsPem)) {
h.logger.V(1).Info("Warning: failed to parse one or more certificates from ConfigMap")
}
}
return configMap.Data, true

return caCertPool
}

func injectCertificates(certsPem []byte, transport *http.Transport, logger logr.Logger) {
caCertPool := transport.TLSClientConfig.RootCAs
if caCertPool == nil {
systemCertPool, err := x509.SystemCertPool()
if err != nil {
logger.Error(err, "Failed to load system cert pool")
caCertPool = x509.NewCertPool()
} else {
caCertPool = systemCertPool
}
func (h *DefaultHttpClientsHolder) readCertCM(ctx context.Context, cmReference *controller.ConfigmapReference) (*corev1.ConfigMap, error) {
if cmReference == nil {
return nil, nil
}
if ok := caCertPool.AppendCertsFromPEM(certsPem); ok {
transport.TLSClientConfig = &tls.Config{RootCAs: caCertPool}

namespacedName := types.NamespacedName{
Name: cmReference.Name,
Namespace: cmReference.Namespace,
}

configMap := &corev1.ConfigMap{}
if err := h.k8s.Get(ctx, namespacedName, configMap); err != nil {
return nil, fmt.Errorf("failed to read ConfigMap %s/%s containing certificates: %w", cmReference.Namespace, cmReference.Name, err)
}

return configMap, nil
}
Loading
Loading