diff --git a/Makefile b/Makefile index fddd3db..436c5ea 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ generate-configuration: # Format: example_path::observed_resources_path (observed_resources_path optional). EXAMPLES := \ examples/securitystacks/minimal.yaml:: \ + examples/securitystacks/nodepool.yaml:: \ examples/securitystacks/standard.yaml:: render\:all: diff --git a/apis/securitystacks/definition.yaml b/apis/securitystacks/definition.yaml index ad8ecef..1d69ffc 100644 --- a/apis/securitystacks/definition.yaml +++ b/apis/securitystacks/definition.yaml @@ -59,6 +59,34 @@ spec: enum: - ProviderConfig - ClusterProviderConfig + nodePool: + description: Optional dedicated Karpenter NodePool for security workloads. When enabled, non-DaemonSet pods are scheduled with workload-type=security and tolerate security=true:NoSchedule; DaemonSets only receive tolerations. + type: object + properties: + enabled: + description: Whether to create a dedicated NodePool. Defaults to false. + type: boolean + default: false + name: + description: NodePool name on the target cluster. Defaults to "hops-security". + type: string + nodeClassName: + description: EKS Auto Mode NodeClass to reference. Defaults to "hops-default". + type: string + limits: + description: Karpenter NodePool resource limits. Defaults to nodes=10. + type: object + x-kubernetes-preserve-unknown-fields: true + requirements: + description: Karpenter scheduling requirements. Defaults to amd64/linux spot or on-demand nodes. + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + disruption: + description: Karpenter disruption settings. Defaults to WhenEmptyOrUnderutilized after 60s. + type: object + x-kubernetes-preserve-unknown-fields: true trivy: description: Trivy Operator configuration. type: object diff --git a/examples/securitystacks/nodepool.yaml b/examples/securitystacks/nodepool.yaml new file mode 100644 index 0000000..deda3b4 --- /dev/null +++ b/examples/securitystacks/nodepool.yaml @@ -0,0 +1,11 @@ +apiVersion: hops.ops.com.ai/v1alpha1 +kind: SecurityStack +metadata: + name: security + namespace: default +spec: + clusterName: default + labels: + team: platform + nodePool: + enabled: true diff --git a/functions/render/000-state-init.yaml.gotmpl b/functions/render/000-state-init.yaml.gotmpl index d4fedf8..4f2ef1a 100644 --- a/functions/render/000-state-init.yaml.gotmpl +++ b/functions/render/000-state-init.yaml.gotmpl @@ -38,6 +38,36 @@ "kind" ($k8sPCR.kind | default "ProviderConfig") }} +# ============================================================================== +# NodePool configuration +# ============================================================================== +{{- $nodePoolSpec := $spec.nodePool | default dict }} +{{- $nodePoolEnabled := false }} +{{- if hasKey $nodePoolSpec "enabled" }} + {{- $nodePoolEnabled = $nodePoolSpec.enabled }} +{{- end }} +{{- $nodePoolName := $nodePoolSpec.name | default "hops-security" }} +{{- $nodePoolNodeClassName := $nodePoolSpec.nodeClassName | default "hops-default" }} +{{- $nodePoolLimits := $nodePoolSpec.limits | default (dict "nodes" 10) }} +{{- $nodePoolRequirements := $nodePoolSpec.requirements | default (list + (dict "key" "karpenter.sh/capacity-type" "operator" "In" "values" (list "spot" "on-demand")) + (dict "key" "eks.amazonaws.com/instance-category" "operator" "In" "values" (list "c" "m" "r")) + (dict "key" "eks.amazonaws.com/instance-generation" "operator" "Gt" "values" (list "4")) + (dict "key" "eks.amazonaws.com/instance-memory" "operator" "Gt" "values" (list "7999")) + (dict "key" "eks.amazonaws.com/instance-cpu" "operator" "Gt" "values" (list "1")) + (dict "key" "kubernetes.io/arch" "operator" "In" "values" (list "amd64")) + (dict "key" "kubernetes.io/os" "operator" "In" "values" (list "linux")) +) }} +{{- $nodePoolDisruption := $nodePoolSpec.disruption | default (dict "consolidationPolicy" "WhenEmptyOrUnderutilized" "consolidateAfter" "60s") }} +{{- $nodePoolTaintKey := "security" }} +{{- $nodePoolTaintValue := "true" }} +{{- $nodePoolNodeSelector := dict }} +{{- $nodePoolTolerations := list }} +{{- if $nodePoolEnabled }} + {{- $nodePoolNodeSelector = dict "workload-type" "security" }} + {{- $nodePoolTolerations = list (dict "key" $nodePoolTaintKey "operator" "Equal" "value" $nodePoolTaintValue "effect" "NoSchedule") }} +{{- end }} + # ============================================================================== # Trivy Operator component # ============================================================================== @@ -115,6 +145,18 @@ "labels" $labels "helmProviderConfigRef" $helmPCR "kubernetesProviderConfigRef" $k8sPCR + "nodePool" (dict + "enabled" $nodePoolEnabled + "name" $nodePoolName + "nodeClassName" $nodePoolNodeClassName + "limits" $nodePoolLimits + "requirements" $nodePoolRequirements + "disruption" $nodePoolDisruption + "taintKey" $nodePoolTaintKey + "taintValue" $nodePoolTaintValue + "nodeSelector" $nodePoolNodeSelector + "tolerations" $nodePoolTolerations + ) "trivy" (dict "enabled" $tvEnabled "namespace" $tvNamespace diff --git a/functions/render/155-nodepool.yaml.gotmpl b/functions/render/155-nodepool.yaml.gotmpl new file mode 100644 index 0000000..b1b7502 --- /dev/null +++ b/functions/render/155-nodepool.yaml.gotmpl @@ -0,0 +1,76 @@ +# code: language=yaml +# +# Optional Karpenter NodePool for security workloads. +# DaemonSets get tolerations from their Helm values; Deployments and Jobs also +# get node selectors so they land on this pool. +# + +{{- if $state.nodePool.enabled }} +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: {{ $state.name }}-nodepool-security + annotations: + {{ setResourceNameAnnotation "nodepool-security" }} + labels: {{ $state.labels | toJson }} +spec: + managementPolicies: {{ $state.managementPolicies | toJson }} + forProvider: + manifest: + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: {{ $state.nodePool.name }} + spec: + template: + metadata: + labels: + workload-type: security + spec: + nodeClassRef: + group: eks.amazonaws.com + kind: NodeClass + name: {{ $state.nodePool.nodeClassName }} + taints: + - key: {{ $state.nodePool.taintKey }} + value: {{ $state.nodePool.taintValue | quote }} + effect: NoSchedule + requirements: {{ $state.nodePool.requirements | toJson }} + limits: {{ $state.nodePool.limits | toJson }} + disruption: {{ $state.nodePool.disruption | toJson }} + providerConfigRef: + name: {{ $state.kubernetesProviderConfigRef.name }} + kind: {{ $state.kubernetesProviderConfigRef.kind }} + +{{- $nodePoolObjectName := printf "%s-nodepool-security" $state.name }} +{{- $securityReleases := list + (dict "enabled" $state.trivy.enabled "name" $state.trivy.releaseName "usageName" "trivy") + (dict "enabled" $state.falco.enabled "name" $state.falco.releaseName "usageName" "falco") + (dict "enabled" $state.kubescape.enabled "name" $state.kubescape.releaseName "usageName" "kubescape") +}} +{{- range $release := $securityReleases }} + {{- if $release.enabled }} +--- +apiVersion: protection.crossplane.io/v1beta1 +kind: Usage +metadata: + name: {{ $state.name }}-delete-{{ $release.usageName }}-before-nodepool + annotations: + {{ setResourceNameAnnotation (printf "usage-%s-before-nodepool" $release.usageName) }} + labels: {{ $state.labels | toJson }} +spec: + replayDeletion: true + of: + apiVersion: kubernetes.m.crossplane.io/v1alpha1 + kind: Object + resourceRef: + name: {{ $nodePoolObjectName }} + by: + apiVersion: helm.m.crossplane.io/v1beta1 + kind: Release + resourceRef: + name: {{ $release.name }} + {{- end }} +{{- end }} +{{- end }} diff --git a/functions/render/200-trivy.yaml.gotmpl b/functions/render/200-trivy.yaml.gotmpl index 540c2c1..fca3bba 100644 --- a/functions/render/200-trivy.yaml.gotmpl +++ b/functions/render/200-trivy.yaml.gotmpl @@ -45,10 +45,16 @@ spec: # operator Deployment, NOT `operator.resources` (which is ignored). resources: requests: - cpu: 100m - memory: 1Gi + cpu: 50m + memory: 826Mi limits: memory: 1Gi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $state.nodePool.nodeSelector | nindent 8 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 8 }} + {{- end }} trivy: # Scan targets — image vulns + workload config + RBAC posture + secrets. # Cluster-compliance (CIS / NSA) is left off here; Kubescape covers that @@ -71,6 +77,22 @@ spec: # signal cadence once ObserveStack alerts are wired. vulnerabilityScannerScanOnlyCurrentRevisions: true configAuditScannerScanOnlyCurrentRevisions: true + {{- if $state.nodePool.enabled }} + scanJobNodeSelector: + {{- toYaml $state.nodePool.nodeSelector | nindent 10 }} + scanJobTolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} + {{- if $state.nodePool.enabled }} + nodeCollector: + # Node collector is per-node; tolerate all taints so node scans are not + # silently skipped on dedicated pools. + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + {{- end }} serviceMonitor: # Gated on spec.observe.enabled (default true) — flips on when # ObserveStack integration is wired. Prometheus on the platform uses diff --git a/functions/render/210-falco.yaml.gotmpl b/functions/render/210-falco.yaml.gotmpl index e32cbf3..6d369a0 100644 --- a/functions/render/210-falco.yaml.gotmpl +++ b/functions/render/210-falco.yaml.gotmpl @@ -60,11 +60,29 @@ spec: tty: false resources: requests: - cpu: 100m - memory: 512Mi + cpu: 50m + memory: 826Mi limits: cpu: 1000m memory: 2Gi + falcoctl: + artifact: + install: + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + cpu: 100m + memory: 128Mi + follow: + resources: + requests: + cpu: 11m + memory: 50Mi + limits: + cpu: 100m + memory: 128Mi # Run on every node, mirroring the ambient DaemonSet (ztunnel / istio-cni) # tolerations pattern. Don't miss events on tainted node pools. tolerations: @@ -98,11 +116,17 @@ spec: # Burstable: stateless event forwarder, restart-cheap. resources: requests: - cpu: 50m - memory: 64Mi + cpu: 15m + memory: 100Mi limits: cpu: 100m - memory: 128Mi + memory: 256Mi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $state.nodePool.nodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} {{- if $state.observe.enabled }} config: # Direct Loki output — Falcosidekick 2.31 does NOT expose an OTLP @@ -133,11 +157,17 @@ spec: k8s-metacollector: resources: requests: - cpu: 50m - memory: 128Mi + cpu: 15m + memory: 100Mi limits: cpu: 100m memory: 256Mi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $state.nodePool.nodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} {{- if gt (len $fl.values) 0 }} {{- toYaml $fl.values | nindent 6 }} {{- end }} diff --git a/functions/render/220-kubescape.yaml.gotmpl b/functions/render/220-kubescape.yaml.gotmpl index ab9bc12..983e61e 100644 --- a/functions/render/220-kubescape.yaml.gotmpl +++ b/functions/render/220-kubescape.yaml.gotmpl @@ -30,6 +30,7 @@ {{- $ks := $state.kubescape }} {{- if $ks.enabled }} +{{- $securityNodeSelector := mergeOverwrite (dict "kubernetes.io/os" "linux") $state.nodePool.nodeSelector }} --- apiVersion: helm.m.crossplane.io/v1beta1 kind: Release @@ -97,13 +98,106 @@ spec: # gated on BOTH this capability AND kubescape.serviceMonitor.enabled # (set below). prometheusExporter: {{ if $state.observe.enabled }}enable{{ else }}disable{{ end }} - {{- if $state.observe.enabled }} + {{- if $state.nodePool.enabled }} + customScheduling: + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} kubescape: + resources: + requests: + cpu: 716m + memory: 454Mi + limits: + cpu: 1000m + memory: 1Gi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} + {{- if $state.observe.enabled }} # Gates the prometheus-exporter ServiceMonitor template (chart logic # requires BOTH this AND capabilities.prometheusExporter=enable). serviceMonitor: enabled: true {{- end }} + operator: + resources: + requests: + cpu: 15m + memory: 100Mi + limits: + cpu: 300m + memory: 512Mi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + admissionWebhooks: + patch: + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 14 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 14 }} + {{- end }} + storage: + resources: + requests: + cpu: 296m + memory: 156Mi + limits: + cpu: 1000m + memory: 4735Mi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + mtls: + certgen: + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 14 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 14 }} + {{- end }} + synchronizer: + resources: + requests: + cpu: 15m + memory: 100Mi + limits: + cpu: 200m + memory: 500Mi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} + prometheusExporter: + resources: + requests: + cpu: 15m + memory: 100Mi + limits: + cpu: 100m + memory: 256Mi + {{- if $state.nodePool.enabled }} + nodeSelector: + {{- toYaml $securityNodeSelector | nindent 10 }} + tolerations: + {{- toYaml $state.nodePool.tolerations | nindent 10 }} + {{- end }} + nodeAgent: + # DaemonSet: no workload-type nodeSelector, tolerate all taints. + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists {{- if gt (len $ks.values) 0 }} {{- toYaml $ks.values | nindent 6 }} {{- end }} diff --git a/tests/test-render/main.k b/tests/test-render/main.k index 6668f27..5387fbc 100644 --- a/tests/test-render/main.k +++ b/tests/test-render/main.k @@ -727,6 +727,106 @@ _items = [ ] } } + + # ========================================================================== + # Test 16: nodePool adds placement while DaemonSets keep broad tolerations. + # ========================================================================== + metav1alpha1.CompositionTest { + metadata.name = "nodepool-adds-security-placement-and-resources" + spec = { + compositionPath = "apis/securitystacks/composition.yaml" + xrdPath = "apis/securitystacks/definition.yaml" + timeoutSeconds = 60 + validate = False + xr = stacksv1alpha1.SecurityStack { + metadata.name = "security-nodepool" + spec = { + clusterName = "my-cluster" + nodePool.enabled = True + } + } + assertResources = [ + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata.name = "trivy-operator" + spec.forProvider.values = { + nodeSelector = {"workload-type" = "security"} + tolerations = [ + { + key = "security" + operator = "Equal" + value = "true" + effect = "NoSchedule" + } + ] + resources.requests = { + cpu = "50m" + memory = "826Mi" + } + nodeCollector.tolerations = [ + {operator = "Exists", effect = "NoSchedule"} + {operator = "Exists", effect = "NoExecute"} + ] + } + } + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata.name = "falco" + spec.forProvider.values = { + tolerations = [ + {operator = "Exists", effect = "NoSchedule"} + {operator = "Exists", effect = "NoExecute"} + ] + resources.requests = { + cpu = "50m" + memory = "826Mi" + } + falcoctl.artifact.follow.resources.requests = { + cpu = "11m" + memory = "50Mi" + } + falcosidekick.nodeSelector = {"workload-type" = "security"} + falcosidekick.tolerations = [ + { + key = "security" + operator = "Equal" + value = "true" + effect = "NoSchedule" + } + ] + } + } + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata.name = "kubescape" + spec.forProvider.values.kubescape = { + nodeSelector = { + "kubernetes.io/os" = "linux" + "workload-type" = "security" + } + resources.requests = { + cpu = "716m" + memory = "454Mi" + } + } + } + { + apiVersion = "kubernetes.m.crossplane.io/v1alpha1" + kind = "Object" + metadata.name = "security-nodepool-nodepool-security" + spec.forProvider.manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata.name = "hops-security" + spec.template.metadata.labels = {"workload-type" = "security"} + } + } + ] + } + } ] items = _items