Surface volatile-excluded rules as time-bounded exceptions in report output

cmd/validate/input.go

@@ -244,6 +244,7 @@ func validateInputCmd(validate InputValidationFunc) *cobra.Command {
 
 					if err == nil {
 						res.input.Violations = out.Violations()
+						res.input.Exceptions = out.Exceptions()
 
 						warnings := out.Warnings()
 						if showWarnings {

internal/evaluator/DESIGN.md

@@ -26,6 +26,30 @@ The key insight: terms (+100pts) and specific rules (+100pts) always override co
 (10pts) or wildcard (1pt) patterns. This lets operators exclude a broad category while
 including specific exceptions, or vice versa.
 
+## Config-Based Exclusions Surface as Exceptions
+
+`Config.Exclude` and `VolatileConfig.Exclude` entries cause matching rules to appear in
+`Outcome.Exceptions` rather than being silently dropped. This lets callers distinguish
+"rule ran and failed" from "rule was excluded by policy".
+
+`VolatileConfig.Exclude` entries may carry `EffectiveOn` and `EffectiveUntil` CRD fields.
+When present, these are stamped onto the exception result as `effective_on` / `effective_until`
+metadata. Presence of `effective_until` distinguishes a time-bounded exception from a
+permanent one — permanent (Config.Exclude) exceptions carry neither field.
+
+**Rego-native exceptions are intentionally excluded from `Outcome.Exceptions`.** The conftest
+evaluation path supports a `data.<namespace>.exception` query that lets Rego policy authors
+suppress deny results. These are implementation details of specific policies — structural
+"not applicable" decisions baked into the policy itself, not operator-level waivers. No
+production Conforma policies use this mechanism; it exists for conftest compatibility.
+Only config-sourced exclusions (from `Config.Exclude` / `VolatileConfig.Exclude`) surface
+in `Outcome.Exceptions`.
+
+**Implementation note**: criteria metadata is keyed by the raw criteria value (e.g. `"pkg"`,
+`"pkg.*"`, `"@collection"`), not by the resolved rule ID (`"pkg.rule"`). When a rule is
+excluded, `PolicyResolutionResult.MatchingExcludeCriteria` records which raw pattern matched,
+so `FilterResults` can retrieve the correct metadata via `getMeta(criteriaValue)`.
+
 ## Adding a New Filter
 
 Follow the pattern in `IncludeExcludePolicyResolver`: embed `basePolicyResolver`, implement

internal/evaluator/base_evaluator.go

@@ -280,7 +280,10 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults
 	effectiveTime := b.policy.EffectiveTime()
 	ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime)
 
+	// Exceptions are not counted toward totalRules to preserve the fail-closed invariant:
+	// an all-excluded policy must not report success with zero real checks performed.
 	totalRules := 0
+	allExceptions := 0
 
 	missingIncludes := map[string]bool{}
 	for _, defaultItem := range b.include.defaultItems {
@@ -306,21 +309,22 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults
 			addRuleMetadata(ctx, &allResults[j], b.rules)
 		}
 
-		filteredResults, updatedMissingIncludes := unifiedFilter.FilterResults(
+		filteredResults, updatedMissingIncludes, configExceptions := unifiedFilter.FilterResults(
 			allResults, b.allRules, target.Target, target.ComponentName, missingIncludes, effectiveTime)
 		missingIncludes = updatedMissingIncludes
 
-		warnings, failures, exceptions, skipped := unifiedFilter.CategorizeResults(
+		warnings, failures, _, skipped := unifiedFilter.CategorizeResults(
 			filteredResults, result, effectiveTime)
 
 		result.Warnings = warnings
 		result.Failures = failures
-		result.Exceptions = exceptions
+		result.Exceptions = ensureNonNilSlice(configExceptions)
 		result.Skipped = skipped
 
 		result.Successes = b.computeSuccesses(result, b.rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime)
 
 		totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes)
+		allExceptions += len(configExceptions)
 		results = append(results, result)
 	}
 
@@ -337,6 +341,12 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults
 	trim(&results)
 
 	if totalRules == 0 {
+		if allExceptions > 0 {
+			// All rules were excluded by policy config: warn but return the exceptions so
+			// downstream consumers (e.g. SVR generators) can represent the waivers.
+			log.Warn("all rules were excluded by policy config; no policy checks were performed")
+			return results, nil
+		}
 		log.Error("no successes, warnings, or failures, check input")
 		return nil, fmt.Errorf("no successes, warnings, or failures, check input")
 	}
@@ -395,7 +405,7 @@ func (b *basePolicyEvaluator) computeSuccesses(
 		}
 
 		if unifiedFilter != nil {
-			filteredResults, _ := unifiedFilter.FilterResults(
+			filteredResults, _, _ := unifiedFilter.FilterResults(
 				[]Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime)
 			if len(filteredResults) == 0 {
 				log.Debugf("Skipping result success: %#v", success)

internal/evaluator/conftest_evaluator.go

@@ -168,7 +168,7 @@
 }
 
 const (
 	effectiveOnFormat   = "2006-01-02T15:04:05Z"
 	effectiveOnTimeout  = -90 * 24 * time.Hour // keep effective_on metadata up to 90 days
 	metadataQuery       = "query"
 	metadataCode        = "code"
@@ -176,8 +176,9 @@
 	metadataDependsOn   = "depends_on"
 	metadataDescription = "description"
 	metadataSeverity    = "severity"
-	metadataEffectiveOn = "effective_on"
-	metadataSolution    = "solution"
+	metadataEffectiveOn    = "effective_on"
+	metadataEffectiveUntil = "effective_until"
+	metadataSolution       = "solution"
 	metadataTerm        = "term"
 	metadataTitle       = "title"
 )
@@ -613,8 +614,10 @@
 	ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime)
 
 	// Track how many rules have been processed. This is used later on to determine if anything
-	// at all was processed.
+	// at all was processed. Exceptions are not counted because an all-excluded policy would
+	// otherwise falsely report success with no real checks performed.
 	totalRules := 0
+	allExceptions := 0
 
 	// Populate a list with all the include directives specified in the
 	// policy config.
@@ -650,25 +653,26 @@
 		}
 
 		// Filter results using the unified filter
-		filteredResults, updatedMissingIncludes := unifiedFilter.FilterResults(
+		filteredResults, updatedMissingIncludes, configExceptions := unifiedFilter.FilterResults(
 			allResults, allRules, target.Target, target.ComponentName, missingIncludes, effectiveTime)
 
 		// Update missing includes
 		missingIncludes = updatedMissingIncludes
 
 		// Categorize results using the unified filter
-		warnings, failures, exceptions, skipped := unifiedFilter.CategorizeResults(
+		warnings, failures, _, skipped := unifiedFilter.CategorizeResults(
 			filteredResults, result, effectiveTime)
 
 		result.Warnings = warnings
 		result.Failures = failures
-		result.Exceptions = exceptions
+		result.Exceptions = ensureNonNilSlice(configExceptions)
 		result.Skipped = skipped
 
 		// Replace the placeholder successes slice with the actual successes.
 		result.Successes = c.computeSuccesses(result, rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime)
 
 		totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes)
+		allExceptions += len(configExceptions)
 
 		results = append(results, result)
 	}
@@ -688,6 +692,12 @@
 	// If no rules were checked, then we have effectively failed, because no tests were actually
 	// ran due to input error, etc.
 	if totalRules == 0 {
+		if allExceptions > 0 {
+			// All rules were excluded by policy config: warn but return the exceptions so
+			// downstream consumers (e.g. SVR generators) can represent the waivers.
+			log.Warn("all rules were excluded by policy config; no policy checks were performed")
+			return results, nil
+		}
 		log.Error("no successes, warnings, or failures, check input")
 		return nil, fmt.Errorf("no successes, warnings, or failures, check input")
 	}
@@ -863,7 +873,7 @@
 		// Use unified filtering approach for consistency
 		if unifiedFilter != nil {
 			// Use the unified filter to check if this success should be included
-			filteredResults, _ := unifiedFilter.FilterResults(
+			filteredResults, _, _ := unifiedFilter.FilterResults(
 				[]Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime)
 
 			if len(filteredResults) == 0 {

internal/evaluator/conftest_evaluator_unit_filtering_test.go

@@ -67,8 +67,11 @@ func TestConftestEvaluatorIncludeExclude(t *testing.T) {
 					Warnings: []Result{
 						{Metadata: map[string]any{"code": "lunch.ham"}},
 					},
-					Skipped:    []Result{},
-					Exceptions: []Result{},
+					Skipped: []Result{},
+					Exceptions: []Result{
+						{Metadata: map[string]any{"code": "breakfast.ham"}},
+						{Metadata: map[string]any{"code": "breakfast.spam"}},
+					},
 				},
 			},
 		},

internal/evaluator/criteria.go

@@ -26,16 +26,24 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// criteriaItemMeta carries optional time metadata from volatile config CRD fields.
+type criteriaItemMeta struct {
+	effectiveOn    string
+	effectiveUntil string
+}
+
 // contains include/exclude items
 // digestItems stores include/exclude items that are specific with an imageRef
 // - the imageRef is the key, value is the policy to include/exclude.
 // componentItems stores include/exclude items that are specific to a component name
 // - the component name is the key, value is the policy to include/exclude.
 // defaultItems are include/exclude items without an imageRef
+// meta maps criteria value strings to their time metadata (volatile config only).
 type Criteria struct {
 	digestItems    map[string][]string
 	componentItems map[string][]string
 	defaultItems   []string
+	meta           map[string]criteriaItemMeta
 }
 
 func (c *Criteria) len() int {
@@ -71,6 +79,30 @@ func (c *Criteria) addArray(key string, values []string) {
 	}
 }
 
+func (c *Criteria) addItemWithMeta(key, value string, m criteriaItemMeta) {
+	c.addItem(key, value)
+	c.storeMeta(value, m)
+}
+
+// storeMeta records time metadata for value without adding a new default item.
+// Use this for component-scoped criteria where the value is already registered
+// via addComponentItem but the meta map still needs to be populated.
+func (c *Criteria) storeMeta(value string, m criteriaItemMeta) {
+	if m.effectiveOn != "" || m.effectiveUntil != "" {
+		if c.meta == nil {
+			c.meta = make(map[string]criteriaItemMeta)
+		}
+		c.meta[value] = m
+	}
+}
+
+func (c *Criteria) getMeta(value string) criteriaItemMeta {
+	if c.meta == nil {
+		return criteriaItemMeta{}
+	}
+	return c.meta[value]
+}
+
 func (c *Criteria) addComponentItem(componentName, value string) {
 	if c.componentItems == nil {
 		c.componentItems = make(map[string][]string)
@@ -194,19 +226,21 @@ func collectVolatileConfigItems(items *Criteria, volatileCriteria []ecc.Volatile
 			until = at
 		}
 		if until.Compare(at) >= 0 && from.Compare(at) <= 0 {
+			meta := criteriaItemMeta{effectiveOn: c.EffectiveOn, effectiveUntil: c.EffectiveUntil}
 			// DEPRECATED: use c.ImageDigest instead
 			if c.ImageRef != "" {
-				items.addItem(c.ImageRef, c.Value)
+				items.addItemWithMeta(c.ImageRef, c.Value, meta)
 			} else if c.ImageUrl != "" {
-				items.addItem(c.ImageUrl, c.Value)
+				items.addItemWithMeta(c.ImageUrl, c.Value, meta)
 			} else if c.ImageDigest != "" {
-				items.addItem(c.ImageDigest, c.Value)
+				items.addItemWithMeta(c.ImageDigest, c.Value, meta)
 			} else if len(c.ComponentNames) > 0 {
 				for _, componentName := range c.ComponentNames {
 					items.addComponentItem(string(componentName), c.Value)
 				}
+				items.storeMeta(c.Value, meta)
 			} else {
-				items.addItem("", c.Value)
+				items.addItemWithMeta("", c.Value, meta)
 			}
 		}
 	}

internal/evaluator/criteria_test.go

@@ -1075,3 +1075,63 @@ func TestOriginalComponentName(t *testing.T) {
 		})
 	}
 }
+
+func TestAddItemWithMeta(t *testing.T) {
+	tests := []struct {
+		name       string
+		key        string
+		value      string
+		meta       criteriaItemMeta
+		expectMeta bool
+	}{
+		{
+			name:       "stores meta when effectiveOn is set",
+			key:        "",
+			value:      "pkg.rule",
+			meta:       criteriaItemMeta{effectiveOn: "2025-01-01T00:00:00Z"},
+			expectMeta: true,
+		},
+		{
+			name:       "stores meta when effectiveUntil is set",
+			key:        "",
+			value:      "pkg.rule",
+			meta:       criteriaItemMeta{effectiveUntil: "2025-12-31T23:59:59Z"},
+			expectMeta: true,
+		},
+		{
+			name:       "stores meta when both fields are set",
+			key:        "",
+			value:      "pkg.rule",
+			meta:       criteriaItemMeta{effectiveOn: "2025-01-01T00:00:00Z", effectiveUntil: "2025-12-31T23:59:59Z"},
+			expectMeta: true,
+		},
+		{
+			name:       "no meta stored when both fields are empty",
+			key:        "",
+			value:      "pkg.rule",
+			meta:       criteriaItemMeta{},
+			expectMeta: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Criteria{}
+			c.addItemWithMeta(tt.key, tt.value, tt.meta)
+			require.Contains(t, c.defaultItems, tt.value)
+			got := c.getMeta(tt.value)
+			if tt.expectMeta {
+				require.Equal(t, tt.meta, got)
+			} else {
+				require.Equal(t, criteriaItemMeta{}, got)
+			}
+		})
+	}
+}
+
+func TestGetMetaZeroForNonVolatile(t *testing.T) {
+	c := &Criteria{}
+	c.addItem("", "pkg.rule")
+	got := c.getMeta("pkg.rule")
+	require.Equal(t, criteriaItemMeta{}, got)
+}