From 446d9ffeca7094f451d6b68a186fd4ca1153951c Mon Sep 17 00:00:00 2001 From: arewm Date: Thu, 16 Jul 2026 12:27:13 -0400 Subject: [PATCH 1/2] output: Expose exceptions with expiry metadata in reports Exception results were computed but not exposed to the report due to missing exceptions field in Input struct. Preserve effective_until metadata so downstream tooling can consume time-bounded expiry. Assisted-by: Claude Code (Haiku 4.5) --- cmd/validate/input.go | 1 + internal/input/report.go | 1 + internal/input/report_test.go | 73 ++++++++++++++++++++++++++++ internal/output/output.go | 13 ++++- internal/output/output_test.go | 88 ++++++++++++++++++++++++++++++++++ internal/server/handler.go | 1 + 6 files changed, 176 insertions(+), 1 deletion(-) diff --git a/cmd/validate/input.go b/cmd/validate/input.go index 5d056bc66..11d208f5a 100644 --- a/cmd/validate/input.go +++ b/cmd/validate/input.go @@ -244,6 +244,7 @@ func validateInputCmd(validate InputValidationFunc) *cobra.Command { if err == nil { res.input.Violations = out.Violations() + res.input.Exceptions = out.Exceptions() warnings := out.Warnings() if showWarnings { diff --git a/internal/input/report.go b/internal/input/report.go index e8d74eb6a..6e01b3b8f 100644 --- a/internal/input/report.go +++ b/internal/input/report.go @@ -39,6 +39,7 @@ type Input struct { Violations []evaluator.Result `json:"violations"` Warnings []evaluator.Result `json:"warnings"` Successes []evaluator.Result `json:"successes"` + Exceptions []evaluator.Result `json:"exceptions,omitempty"` Success bool `json:"success"` SuccessCount int `json:"success-count"` } diff --git a/internal/input/report_test.go b/internal/input/report_test.go index 1ac2ab454..d7cc6b28b 100644 --- a/internal/input/report_test.go +++ b/internal/input/report_test.go @@ -284,6 +284,79 @@ func testInputsFor(filePaths []string) []Input { return inputs } +func Test_ReportJson_WithExceptions(t *testing.T) { + ctx := context.Background() + testPolicy := createTestPolicy(t, ctx) + testEffectiveTime := testPolicy.EffectiveTime().UTC().Format(time.RFC3339Nano) + + inputs := []Input{ + { + FilePath: "/path/to/file1.yaml", + Violations: []evaluator.Result{ + {Message: "violation1"}, + }, + Exceptions: []evaluator.Result{ + { + Message: "excepted rule", + Metadata: map[string]interface{}{ + "code": "some.rule", + "effective_on": "2023-01-01T00:00:00Z", + "effective_until": "2025-12-31T23:59:59Z", + }, + }, + }, + Success: false, + }, + } + + report, err := NewReport(inputs, testPolicy, nil, false, true, true) + assert.NoError(t, err) + + expected := fmt.Sprintf(` + { + "success": false, + "filepaths": [ + { + "filepath": "/path/to/file1.yaml", + "violations": [{"msg": "violation1"}], + "warnings": null, + "successes": null, + "exceptions": [ + { + "msg": "excepted rule", + "metadata": { + "code": "some.rule", + "effective_on": "2023-01-01T00:00:00Z", + "effective_until": "2025-12-31T23:59:59Z" + } + } + ], + "success": false, + "success-count": 0 + } + ], + "policy": { + "name": "Default", + "description": "Stuff and things", + "sources": [ + { + "name": "Default", + "policy": ["github.com/org/repo//policy"], + "data": ["github.com/org/repo//data"], + "config": {"include": ["basic"]} + } + ] + }, + "ec-version": "development", + "effective-time": %q + } + `, testEffectiveTime) + + reportJson, err := report.toFormat(JSON) + assert.NoError(t, err) + assert.JSONEq(t, expected, string(reportJson)) +} + func Test_ReportText(t *testing.T) { inputs := []Input{ { diff --git a/internal/output/output.go b/internal/output/output.go index a85a9b8ce..26caeb5d3 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -218,13 +218,24 @@ func keepSomeMetadata(results []evaluator.Result) { func keepSomeMetadataSingle(result evaluator.Result) { for key := range result.Metadata { - if key == "code" || key == "effective_on" || key == "term" { + if key == "code" || key == "effective_on" || key == "effective_until" || key == "term" { continue } delete(result.Metadata, key) } } +// Exceptions aggregates and returns all exceptions. +func (o Output) Exceptions() []evaluator.Result { + exceptions := make([]evaluator.Result, 0, 10) + for _, result := range o.PolicyCheck { + exceptions = append(exceptions, result.Exceptions...) + } + + exceptions = sortResults(exceptions) + return exceptions +} + // addCheckResultsToViolations appends the Failures from CheckResult to the violations slice. func (o Output) addCheckResultsToViolations(violations []evaluator.Result) []evaluator.Result { for _, check := range o.PolicyCheck { diff --git a/internal/output/output_test.go b/internal/output/output_test.go index 613d26f9b..160e97686 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -1085,6 +1085,24 @@ func TestKeepSomeMetadataSingle(t *testing.T) { "effective_on": "2023-01-01", }, }, + { + name: "preserves effective_until on exception results", + input: evaluator.Result{ + Message: "Test exception", + Metadata: map[string]interface{}{ + "code": "test.rule", + "effective_on": "2023-01-01T00:00:00Z", + "effective_until": "2025-12-31T23:59:59Z", + "title": "Test Rule Title", + "description": "should be removed", + }, + }, + expectedMetadata: map[string]interface{}{ + "code": "test.rule", + "effective_on": "2023-01-01T00:00:00Z", + "effective_until": "2025-12-31T23:59:59Z", + }, + }, } for _, tc := range cases { @@ -1106,3 +1124,73 @@ func TestKeepSomeMetadataSingle(t *testing.T) { }) } } + +func Test_Exceptions(t *testing.T) { + cases := []struct { + name string + output Output + expected []evaluator.Result + }{ + { + name: "no-exceptions", + output: Output{}, + expected: []evaluator.Result{}, + }, + { + name: "single exception", + output: Output{ + PolicyCheck: []evaluator.Outcome{ + { + Exceptions: []evaluator.Result{ + { + Message: "excepted rule", + Metadata: map[string]interface{}{ + "code": "some.rule", + "effective_on": "2023-01-01T00:00:00Z", + "effective_until": "2025-12-31T23:59:59Z", + }, + }, + }, + }, + }, + }, + expected: []evaluator.Result{ + { + Message: "excepted rule", + Metadata: map[string]interface{}{ + "code": "some.rule", + "effective_on": "2023-01-01T00:00:00Z", + "effective_until": "2025-12-31T23:59:59Z", + }, + }, + }, + }, + { + name: "multiple exceptions across outcomes", + output: Output{ + PolicyCheck: []evaluator.Outcome{ + { + Exceptions: []evaluator.Result{ + {Message: "exception b", Metadata: map[string]interface{}{"code": "pkg.rule_b"}}, + }, + }, + { + Exceptions: []evaluator.Result{ + {Message: "exception a", Metadata: map[string]interface{}{"code": "pkg.rule_a"}}, + }, + }, + }, + }, + expected: []evaluator.Result{ + {Message: "exception a", Metadata: map[string]interface{}{"code": "pkg.rule_a"}}, + {Message: "exception b", Metadata: map[string]interface{}{"code": "pkg.rule_b"}}, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.expected, c.output.Exceptions()) + }) + } +} diff --git a/internal/server/handler.go b/internal/server/handler.go index 729334f03..6ace83566 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -132,6 +132,7 @@ func (s *Server) evaluateAndBuildReport(ctx context.Context, inputPath string) ( } inp.Violations = out.Violations() + inp.Exceptions = out.Exceptions() warnings := out.Warnings() if s.cfg.ShowWarnings { From 745d0f60c50862ac3f428974cb870459f3881564 Mon Sep 17 00:00:00 2001 From: arewm Date: Thu, 16 Jul 2026 14:19:43 -0400 Subject: [PATCH 2/2] evaluator: Surface config-excluded rules as time-bounded policy exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VolatileConfig and Config exclusions now appear in Outcome.Exceptions with effective_on/effective_until timestamps so downstream tooling can represent time-bounded policy waivers rather than silently dropping them. Rego-native exceptions (conftest data.namespace.exception) are intentionally excluded — they are implementation details of specific policies, not operator-level waivers, and no production policies use them. The totalRules invariant is preserved: exceptions are not counted as evaluated rules, and a warning fires when config exclusions account for all rules with no actual policy checks performed. Assisted-by: Claude Code (Sonnet 4.6) --- internal/evaluator/DESIGN.md | 24 +++ internal/evaluator/base_evaluator.go | 18 +- internal/evaluator/conftest_evaluator.go | 24 ++- .../conftest_evaluator_unit_filtering_test.go | 7 +- internal/evaluator/criteria.go | 42 ++++- internal/evaluator/criteria_test.go | 60 +++++++ internal/evaluator/filters.go | 91 ++++++++-- internal/evaluator/filters_test.go | 132 ++++++++++++++- .../opa_evaluator_integration_test.go | 160 ++++++++++++++++++ internal/input/report.go | 10 +- 10 files changed, 528 insertions(+), 40 deletions(-) diff --git a/internal/evaluator/DESIGN.md b/internal/evaluator/DESIGN.md index 868679e85..73f5656c0 100644 --- a/internal/evaluator/DESIGN.md +++ b/internal/evaluator/DESIGN.md @@ -26,6 +26,30 @@ The key insight: terms (+100pts) and specific rules (+100pts) always override co (10pts) or wildcard (1pt) patterns. This lets operators exclude a broad category while including specific exceptions, or vice versa. +## Config-Based Exclusions Surface as Exceptions + +`Config.Exclude` and `VolatileConfig.Exclude` entries cause matching rules to appear in +`Outcome.Exceptions` rather than being silently dropped. This lets callers distinguish +"rule ran and failed" from "rule was excluded by policy". + +`VolatileConfig.Exclude` entries may carry `EffectiveOn` and `EffectiveUntil` CRD fields. +When present, these are stamped onto the exception result as `effective_on` / `effective_until` +metadata. Presence of `effective_until` distinguishes a time-bounded exception from a +permanent one — permanent (Config.Exclude) exceptions carry neither field. + +**Rego-native exceptions are intentionally excluded from `Outcome.Exceptions`.** The conftest +evaluation path supports a `data..exception` query that lets Rego policy authors +suppress deny results. These are implementation details of specific policies — structural +"not applicable" decisions baked into the policy itself, not operator-level waivers. No +production Conforma policies use this mechanism; it exists for conftest compatibility. +Only config-sourced exclusions (from `Config.Exclude` / `VolatileConfig.Exclude`) surface +in `Outcome.Exceptions`. + +**Implementation note**: criteria metadata is keyed by the raw criteria value (e.g. `"pkg"`, +`"pkg.*"`, `"@collection"`), not by the resolved rule ID (`"pkg.rule"`). When a rule is +excluded, `PolicyResolutionResult.MatchingExcludeCriteria` records which raw pattern matched, +so `FilterResults` can retrieve the correct metadata via `getMeta(criteriaValue)`. + ## Adding a New Filter Follow the pattern in `IncludeExcludePolicyResolver`: embed `basePolicyResolver`, implement diff --git a/internal/evaluator/base_evaluator.go b/internal/evaluator/base_evaluator.go index 5b2495b43..289ae1292 100644 --- a/internal/evaluator/base_evaluator.go +++ b/internal/evaluator/base_evaluator.go @@ -280,7 +280,10 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults effectiveTime := b.policy.EffectiveTime() ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime) + // Exceptions are not counted toward totalRules to preserve the fail-closed invariant: + // an all-excluded policy must not report success with zero real checks performed. totalRules := 0 + allExceptions := 0 missingIncludes := map[string]bool{} for _, defaultItem := range b.include.defaultItems { @@ -306,21 +309,22 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults addRuleMetadata(ctx, &allResults[j], b.rules) } - filteredResults, updatedMissingIncludes := unifiedFilter.FilterResults( + filteredResults, updatedMissingIncludes, configExceptions := unifiedFilter.FilterResults( allResults, b.allRules, target.Target, target.ComponentName, missingIncludes, effectiveTime) missingIncludes = updatedMissingIncludes - warnings, failures, exceptions, skipped := unifiedFilter.CategorizeResults( + warnings, failures, _, skipped := unifiedFilter.CategorizeResults( filteredResults, result, effectiveTime) result.Warnings = warnings result.Failures = failures - result.Exceptions = exceptions + result.Exceptions = ensureNonNilSlice(configExceptions) result.Skipped = skipped result.Successes = b.computeSuccesses(result, b.rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime) totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) + allExceptions += len(configExceptions) results = append(results, result) } @@ -337,6 +341,12 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults trim(&results) if totalRules == 0 { + if allExceptions > 0 { + // All rules were excluded by policy config: warn but return the exceptions so + // downstream consumers (e.g. SVR generators) can represent the waivers. + log.Warn("all rules were excluded by policy config; no policy checks were performed") + return results, nil + } log.Error("no successes, warnings, or failures, check input") return nil, fmt.Errorf("no successes, warnings, or failures, check input") } @@ -395,7 +405,7 @@ func (b *basePolicyEvaluator) computeSuccesses( } if unifiedFilter != nil { - filteredResults, _ := unifiedFilter.FilterResults( + filteredResults, _, _ := unifiedFilter.FilterResults( []Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime) if len(filteredResults) == 0 { log.Debugf("Skipping result success: %#v", success) diff --git a/internal/evaluator/conftest_evaluator.go b/internal/evaluator/conftest_evaluator.go index 26ad323be..b5668bfe3 100644 --- a/internal/evaluator/conftest_evaluator.go +++ b/internal/evaluator/conftest_evaluator.go @@ -176,8 +176,9 @@ const ( metadataDependsOn = "depends_on" metadataDescription = "description" metadataSeverity = "severity" - metadataEffectiveOn = "effective_on" - metadataSolution = "solution" + metadataEffectiveOn = "effective_on" + metadataEffectiveUntil = "effective_until" + metadataSolution = "solution" metadataTerm = "term" metadataTitle = "title" ) @@ -613,8 +614,10 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime) // Track how many rules have been processed. This is used later on to determine if anything - // at all was processed. + // at all was processed. Exceptions are not counted because an all-excluded policy would + // otherwise falsely report success with no real checks performed. totalRules := 0 + allExceptions := 0 // Populate a list with all the include directives specified in the // policy config. @@ -650,25 +653,26 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget } // Filter results using the unified filter - filteredResults, updatedMissingIncludes := unifiedFilter.FilterResults( + filteredResults, updatedMissingIncludes, configExceptions := unifiedFilter.FilterResults( allResults, allRules, target.Target, target.ComponentName, missingIncludes, effectiveTime) // Update missing includes missingIncludes = updatedMissingIncludes // Categorize results using the unified filter - warnings, failures, exceptions, skipped := unifiedFilter.CategorizeResults( + warnings, failures, _, skipped := unifiedFilter.CategorizeResults( filteredResults, result, effectiveTime) result.Warnings = warnings result.Failures = failures - result.Exceptions = exceptions + result.Exceptions = ensureNonNilSlice(configExceptions) result.Skipped = skipped // Replace the placeholder successes slice with the actual successes. result.Successes = c.computeSuccesses(result, rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime) totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) + allExceptions += len(configExceptions) results = append(results, result) } @@ -688,6 +692,12 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget // If no rules were checked, then we have effectively failed, because no tests were actually // ran due to input error, etc. if totalRules == 0 { + if allExceptions > 0 { + // All rules were excluded by policy config: warn but return the exceptions so + // downstream consumers (e.g. SVR generators) can represent the waivers. + log.Warn("all rules were excluded by policy config; no policy checks were performed") + return results, nil + } log.Error("no successes, warnings, or failures, check input") return nil, fmt.Errorf("no successes, warnings, or failures, check input") } @@ -863,7 +873,7 @@ func (c conftestEvaluator) computeSuccesses( // Use unified filtering approach for consistency if unifiedFilter != nil { // Use the unified filter to check if this success should be included - filteredResults, _ := unifiedFilter.FilterResults( + filteredResults, _, _ := unifiedFilter.FilterResults( []Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime) if len(filteredResults) == 0 { diff --git a/internal/evaluator/conftest_evaluator_unit_filtering_test.go b/internal/evaluator/conftest_evaluator_unit_filtering_test.go index 471efcb21..6f890cea2 100644 --- a/internal/evaluator/conftest_evaluator_unit_filtering_test.go +++ b/internal/evaluator/conftest_evaluator_unit_filtering_test.go @@ -67,8 +67,11 @@ func TestConftestEvaluatorIncludeExclude(t *testing.T) { Warnings: []Result{ {Metadata: map[string]any{"code": "lunch.ham"}}, }, - Skipped: []Result{}, - Exceptions: []Result{}, + Skipped: []Result{}, + Exceptions: []Result{ + {Metadata: map[string]any{"code": "breakfast.ham"}}, + {Metadata: map[string]any{"code": "breakfast.spam"}}, + }, }, }, }, diff --git a/internal/evaluator/criteria.go b/internal/evaluator/criteria.go index 2180965a0..6fc1cf83b 100644 --- a/internal/evaluator/criteria.go +++ b/internal/evaluator/criteria.go @@ -26,16 +26,24 @@ import ( log "github.com/sirupsen/logrus" ) +// criteriaItemMeta carries optional time metadata from volatile config CRD fields. +type criteriaItemMeta struct { + effectiveOn string + effectiveUntil string +} + // contains include/exclude items // digestItems stores include/exclude items that are specific with an imageRef // - the imageRef is the key, value is the policy to include/exclude. // componentItems stores include/exclude items that are specific to a component name // - the component name is the key, value is the policy to include/exclude. // defaultItems are include/exclude items without an imageRef +// meta maps criteria value strings to their time metadata (volatile config only). type Criteria struct { digestItems map[string][]string componentItems map[string][]string defaultItems []string + meta map[string]criteriaItemMeta } func (c *Criteria) len() int { @@ -71,6 +79,30 @@ func (c *Criteria) addArray(key string, values []string) { } } +func (c *Criteria) addItemWithMeta(key, value string, m criteriaItemMeta) { + c.addItem(key, value) + c.storeMeta(value, m) +} + +// storeMeta records time metadata for value without adding a new default item. +// Use this for component-scoped criteria where the value is already registered +// via addComponentItem but the meta map still needs to be populated. +func (c *Criteria) storeMeta(value string, m criteriaItemMeta) { + if m.effectiveOn != "" || m.effectiveUntil != "" { + if c.meta == nil { + c.meta = make(map[string]criteriaItemMeta) + } + c.meta[value] = m + } +} + +func (c *Criteria) getMeta(value string) criteriaItemMeta { + if c.meta == nil { + return criteriaItemMeta{} + } + return c.meta[value] +} + func (c *Criteria) addComponentItem(componentName, value string) { if c.componentItems == nil { c.componentItems = make(map[string][]string) @@ -194,19 +226,21 @@ func collectVolatileConfigItems(items *Criteria, volatileCriteria []ecc.Volatile until = at } if until.Compare(at) >= 0 && from.Compare(at) <= 0 { + meta := criteriaItemMeta{effectiveOn: c.EffectiveOn, effectiveUntil: c.EffectiveUntil} // DEPRECATED: use c.ImageDigest instead if c.ImageRef != "" { - items.addItem(c.ImageRef, c.Value) + items.addItemWithMeta(c.ImageRef, c.Value, meta) } else if c.ImageUrl != "" { - items.addItem(c.ImageUrl, c.Value) + items.addItemWithMeta(c.ImageUrl, c.Value, meta) } else if c.ImageDigest != "" { - items.addItem(c.ImageDigest, c.Value) + items.addItemWithMeta(c.ImageDigest, c.Value, meta) } else if len(c.ComponentNames) > 0 { for _, componentName := range c.ComponentNames { items.addComponentItem(string(componentName), c.Value) } + items.storeMeta(c.Value, meta) } else { - items.addItem("", c.Value) + items.addItemWithMeta("", c.Value, meta) } } } diff --git a/internal/evaluator/criteria_test.go b/internal/evaluator/criteria_test.go index a119e7b68..4015932c7 100644 --- a/internal/evaluator/criteria_test.go +++ b/internal/evaluator/criteria_test.go @@ -1075,3 +1075,63 @@ func TestOriginalComponentName(t *testing.T) { }) } } + +func TestAddItemWithMeta(t *testing.T) { + tests := []struct { + name string + key string + value string + meta criteriaItemMeta + expectMeta bool + }{ + { + name: "stores meta when effectiveOn is set", + key: "", + value: "pkg.rule", + meta: criteriaItemMeta{effectiveOn: "2025-01-01T00:00:00Z"}, + expectMeta: true, + }, + { + name: "stores meta when effectiveUntil is set", + key: "", + value: "pkg.rule", + meta: criteriaItemMeta{effectiveUntil: "2025-12-31T23:59:59Z"}, + expectMeta: true, + }, + { + name: "stores meta when both fields are set", + key: "", + value: "pkg.rule", + meta: criteriaItemMeta{effectiveOn: "2025-01-01T00:00:00Z", effectiveUntil: "2025-12-31T23:59:59Z"}, + expectMeta: true, + }, + { + name: "no meta stored when both fields are empty", + key: "", + value: "pkg.rule", + meta: criteriaItemMeta{}, + expectMeta: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Criteria{} + c.addItemWithMeta(tt.key, tt.value, tt.meta) + require.Contains(t, c.defaultItems, tt.value) + got := c.getMeta(tt.value) + if tt.expectMeta { + require.Equal(t, tt.meta, got) + } else { + require.Equal(t, criteriaItemMeta{}, got) + } + }) + } +} + +func TestGetMetaZeroForNonVolatile(t *testing.T) { + c := &Criteria{} + c.addItem("", "pkg.rule") + got := c.getMeta("pkg.rule") + require.Equal(t, criteriaItemMeta{}, got) +} diff --git a/internal/evaluator/filters.go b/internal/evaluator/filters.go index d6d2d9448..cfddae35d 100644 --- a/internal/evaluator/filters.go +++ b/internal/evaluator/filters.go @@ -66,8 +66,8 @@ type FilterFactory interface { // It handles include/exclude criteria, severity promotion/demotion, // effective time filtering, and success computation. type PostEvaluationFilter interface { - // FilterResults processes all result types and returns the filtered results - // along with updated missing includes tracking. + // FilterResults processes all result types and returns the filtered results, + // updated missing includes tracking, and config-excluded exception results. FilterResults( results []Result, rules policyRules, @@ -75,7 +75,7 @@ type PostEvaluationFilter interface { componentName string, missingIncludes map[string]bool, effectiveTime time.Time, - ) ([]Result, map[string]bool) + ) ([]Result, map[string]bool, []Result) // CategorizeResults takes filtered results and categorizes them by type // (warnings, failures, exceptions, skipped) with appropriate severity logic. @@ -424,16 +424,21 @@ type PolicyResolutionResult struct { MissingIncludes map[string]bool // Explanations provides reasons for why rules/packages were included/excluded Explanations map[string]string + // MatchingExcludeCriteria maps each excluded rule ID to the criteria pattern that excluded it. + // This is used to look up time metadata (effectiveOn/effectiveUntil) for the matching criteria, + // since meta is keyed by the raw criteria value (e.g. "pkg", "pkg.*") not by the resolved rule ID. + MatchingExcludeCriteria map[string]string } // NewPolicyResolutionResult creates a new PolicyResolutionResult with initialized maps func NewPolicyResolutionResult() PolicyResolutionResult { return PolicyResolutionResult{ - IncludedRules: make(map[string]bool), - ExcludedRules: make(map[string]bool), - IncludedPackages: make(map[string]bool), - MissingIncludes: make(map[string]bool), - Explanations: make(map[string]string), + IncludedRules: make(map[string]bool), + ExcludedRules: make(map[string]bool), + IncludedPackages: make(map[string]bool), + MissingIncludes: make(map[string]bool), + Explanations: make(map[string]string), + MatchingExcludeCriteria: make(map[string]string), } } @@ -605,6 +610,7 @@ func (r *basePolicyResolver) baseEvaluateRuleInclusion(ruleID string, ruleInfo r log.Debugf("[evaluateRuleInclusion] Rule: %s INCLUDED", ruleID) } else if excludeScore > 0 { result.ExcludedRules[ruleID] = true + result.MatchingExcludeCriteria[ruleID] = bestMatchingPattern(matchers, r.exclude.get(target, "")) result.Explanations[ruleID] = fmt.Sprintf("excluded (include score: %d, exclude score: %d)", includeScore, excludeScore) log.Debugf("[evaluateRuleInclusion] Rule: %s EXCLUDED", ruleID) } else { @@ -793,6 +799,25 @@ func LegacyIsResultIncluded(result Result, imageRef string, componentName string return includeScore > excludeScore } +// bestMatchingPattern returns the pattern from patterns that has the highest individual +// LegacyScore and appears in matchers. Used to identify which exclude criteria pattern +// triggered a rule exclusion, so its time metadata can be retrieved via getMeta. +func bestMatchingPattern(matchers []string, patterns []string) string { + best := "" + bestScore := 0 + for _, needle := range matchers { + for _, hay := range patterns { + if hay == needle { + if s := LegacyScore(hay); s > bestScore { + bestScore = s + best = hay + } + } + } + } + return best +} + // LegacyScoreMatches returns the combined score for every match between needles and haystack. // 'toBePruned' contains items that will be removed (pruned) from this map if a match is found. // This is the legacy scoring function. @@ -946,7 +971,7 @@ func (f *LegacyPostEvaluationFilter) FilterResults( componentName string, missingIncludes map[string]bool, effectiveTime time.Time, -) ([]Result, map[string]bool) { +) ([]Result, map[string]bool, []Result) { // Filter results based on include/exclude criteria only (no pipeline intention) var filteredResults []Result for _, result := range results { @@ -957,7 +982,7 @@ func (f *LegacyPostEvaluationFilter) FilterResults( } } - return filteredResults, missingIncludes + return filteredResults, missingIncludes, nil } // CategorizeResults implements the PostEvaluationFilter interface for legacy compatibility. @@ -1027,7 +1052,7 @@ func (f *UnifiedPostEvaluationFilter) FilterResults( componentName string, missingIncludes map[string]bool, effectiveTime time.Time, -) ([]Result, map[string]bool) { +) ([]Result, map[string]bool, []Result) { // Check if we're using an ECPolicyResolver (which handles pipeline intentions) // vs IncludeExcludePolicyResolver (which doesn't) if ecResolver, ok := f.policyResolver.(*ECPolicyResolver); ok { @@ -1035,6 +1060,7 @@ func (f *UnifiedPostEvaluationFilter) FilterResults( policyResolution := ecResolver.ResolvePolicy(rules, imageRef) var filteredResults []Result + var configExceptions []Result for _, result := range results { code := ExtractStringFromMetadata(result, metadataCode) // For results without codes, always include them (matches legacy behavior) @@ -1046,6 +1072,21 @@ func (f *UnifiedPostEvaluationFilter) FilterResults( // Check if the result's rule is included based on policy resolution if policyResolution.IncludedRules[code] { filteredResults = append(filteredResults, result) + } else if policyResolution.ExcludedRules[code] { + criteriaValue := policyResolution.MatchingExcludeCriteria[code] + meta := f.policyResolver.Excludes().getMeta(criteriaValue) + newMeta := make(map[string]interface{}, len(result.Metadata)) + for k, v := range result.Metadata { + newMeta[k] = v + } + if meta.effectiveOn != "" { + newMeta[metadataEffectiveOn] = meta.effectiveOn + } + if meta.effectiveUntil != "" { + newMeta[metadataEffectiveUntil] = meta.effectiveUntil + } + result.Metadata = newMeta + configExceptions = append(configExceptions, result) } } @@ -1056,11 +1097,12 @@ func (f *UnifiedPostEvaluationFilter) FilterResults( } } - return filteredResults, missingIncludes + return filteredResults, missingIncludes, configExceptions } // Fall back to legacy filtering for other policy resolvers var filteredResults []Result + var configExceptions []Result for _, result := range results { code := ExtractStringFromMetadata(result, metadataCode) // For results without codes, always include them (matches legacy behavior) @@ -1070,9 +1112,28 @@ func (f *UnifiedPostEvaluationFilter) FilterResults( continue } - // Use legacy filtering logic for all results - if LegacyIsResultIncluded(result, imageRef, componentName, missingIncludes, f.policyResolver.Includes(), f.policyResolver.Excludes()) { + ruleMatchers := LegacyMakeMatchers(result) + excludePatterns := f.policyResolver.Excludes().get(imageRef, componentName) + excludeScore := LegacyScoreMatches(ruleMatchers, excludePatterns, map[string]bool{}) + includeScore := LegacyScoreMatches(ruleMatchers, f.policyResolver.Includes().get(imageRef, componentName), missingIncludes) + + if includeScore > excludeScore { filteredResults = append(filteredResults, result) + } else if excludeScore > 0 { + pattern := bestMatchingPattern(ruleMatchers, excludePatterns) + meta := f.policyResolver.Excludes().getMeta(pattern) + newMeta := make(map[string]interface{}, len(result.Metadata)) + for k, v := range result.Metadata { + newMeta[k] = v + } + if meta.effectiveOn != "" { + newMeta[metadataEffectiveOn] = meta.effectiveOn + } + if meta.effectiveUntil != "" { + newMeta[metadataEffectiveUntil] = meta.effectiveUntil + } + result.Metadata = newMeta + configExceptions = append(configExceptions, result) } } @@ -1096,7 +1157,7 @@ func (f *UnifiedPostEvaluationFilter) FilterResults( } } - return filteredResults, missingIncludes + return filteredResults, missingIncludes, configExceptions } // CategorizeResults takes filtered results and categorizes them by type diff --git a/internal/evaluator/filters_test.go b/internal/evaluator/filters_test.go index 293b2eb59..5402c3de7 100644 --- a/internal/evaluator/filters_test.go +++ b/internal/evaluator/filters_test.go @@ -625,7 +625,7 @@ func TestUnifiedPostEvaluationFilter(t *testing.T) { "@redhat": true, } - filteredResults, updatedMissingIncludes := filter.FilterResults( + filteredResults, updatedMissingIncludes, _ := filter.FilterResults( results, rules, "test-target", "", missingIncludes, time.Now()) // Should include cve.high_severity and tasks.build_task, exclude test.test_data_found @@ -694,7 +694,7 @@ func TestUnifiedPostEvaluationFilter(t *testing.T) { "*": true, } - filteredResults, updatedMissingIncludes := filter.FilterResults( + filteredResults, updatedMissingIncludes, _ := filter.FilterResults( results, rules, "test-target", "", missingIncludes, time.Now()) // Should only include release.security_check (matches pipeline intention) @@ -745,7 +745,7 @@ func TestUnifiedPostEvaluationFilter(t *testing.T) { "cve": true, } - filteredResults, updatedMissingIncludes := filter.FilterResults( + filteredResults, updatedMissingIncludes, _ := filter.FilterResults( results, rules, "test-target", "", missingIncludes, time.Now()) // Should include the CVE result @@ -872,7 +872,7 @@ func TestUnifiedPostEvaluationFilterVsLegacy(t *testing.T) { "@redhat": true, "security.*": true, } - newFilteredResults, newUpdatedMissingIncludes := newFilter.FilterResults( + newFilteredResults, newUpdatedMissingIncludes, _ := newFilter.FilterResults( results, rules, "test-target", "", newMissingIncludes, time.Now()) // Test the legacy approach using the standalone functions @@ -1796,3 +1796,127 @@ func TestPipelineIntentionWithMultipleValues(t *testing.T) { "security package should be included (has included rules)") }) } + +func TestConfigExceptionsInFilterResults(t *testing.T) { + now := time.Now() + effectiveOn := now.Add(-time.Hour).Format(time.RFC3339) + effectiveUntil := now.Add(time.Hour).Format(time.RFC3339) + + makeProvider := func() ConfigProvider { + return &simpleConfigProvider{effectiveTime: now} + } + + makeRules := func() policyRules { + return policyRules{ + "pkg.rule": rule.Info{Code: "pkg.rule", Package: "pkg", ShortName: "rule"}, + } + } + + t.Run("volatile-excluded result surfaces as exception", func(t *testing.T) { + src := ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + {Value: "pkg.rule", EffectiveOn: effectiveOn, EffectiveUntil: effectiveUntil}, + }, + }, + } + filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(src, makeProvider())) + + results := []Result{{ + Message: "test failure", + Metadata: map[string]interface{}{metadataCode: "pkg.rule"}, + }} + + filtered, _, exceptions := filter.FilterResults(results, makeRules(), "", "", map[string]bool{}, now) + + assert.Empty(t, filtered) + require.Len(t, exceptions, 1) + }) + + t.Run("volatile-excluded result with EffectiveUntil gets effective_until stamped", func(t *testing.T) { + src := ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + {Value: "pkg.rule", EffectiveOn: effectiveOn, EffectiveUntil: effectiveUntil}, + }, + }, + } + filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(src, makeProvider())) + + results := []Result{{ + Message: "test failure", + Metadata: map[string]interface{}{metadataCode: "pkg.rule"}, + }} + + _, _, exceptions := filter.FilterResults(results, makeRules(), "", "", map[string]bool{}, now) + + require.Len(t, exceptions, 1) + assert.Equal(t, effectiveUntil, exceptions[0].Metadata[metadataEffectiveUntil]) + assert.Equal(t, effectiveOn, exceptions[0].Metadata[metadataEffectiveOn]) + }) + + t.Run("volatile-excluded result without EffectiveUntil does not get effective_until stamped", func(t *testing.T) { + src := ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + {Value: "pkg.rule", EffectiveOn: effectiveOn}, + }, + }, + } + filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(src, makeProvider())) + + results := []Result{{ + Message: "test failure", + Metadata: map[string]interface{}{metadataCode: "pkg.rule"}, + }} + + _, _, exceptions := filter.FilterResults(results, makeRules(), "", "", map[string]bool{}, now) + + require.Len(t, exceptions, 1) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveUntil) + assert.Equal(t, effectiveOn, exceptions[0].Metadata[metadataEffectiveOn]) + }) + + t.Run("volatile-excluded result with EffectiveUntil only has effective_until stamped but not effective_on", func(t *testing.T) { + src := ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + {Value: "pkg.rule", EffectiveUntil: effectiveUntil}, + }, + }, + } + filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(src, makeProvider())) + + results := []Result{{ + Message: "test failure", + Metadata: map[string]interface{}{metadataCode: "pkg.rule"}, + }} + + _, _, exceptions := filter.FilterResults(results, makeRules(), "", "", map[string]bool{}, now) + + require.Len(t, exceptions, 1) + assert.Equal(t, effectiveUntil, exceptions[0].Metadata[metadataEffectiveUntil]) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveOn) + }) + + t.Run("permanent-excluded result has no time fields", func(t *testing.T) { + src := ecc.Source{ + Config: &ecc.SourceConfig{ + Exclude: []string{"pkg.rule"}, + }, + } + filter := NewUnifiedPostEvaluationFilter(NewECPolicyResolver(src, makeProvider())) + + results := []Result{{ + Message: "test failure", + Metadata: map[string]interface{}{metadataCode: "pkg.rule"}, + }} + + _, _, exceptions := filter.FilterResults(results, makeRules(), "", "", map[string]bool{}, now) + + require.Len(t, exceptions, 1) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveOn) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveUntil) + }) +} + diff --git a/internal/evaluator/opa_evaluator_integration_test.go b/internal/evaluator/opa_evaluator_integration_test.go index b9d575f0e..5137b7f69 100644 --- a/internal/evaluator/opa_evaluator_integration_test.go +++ b/internal/evaluator/opa_evaluator_integration_test.go @@ -447,6 +447,20 @@ deny contains result if { } assert.False(t, hasCheckA, "Expected check_a to be excluded for comp1") assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp1") + + // check_a should appear as an exception with time metadata, not silently disappear. + var checkAException *Result + for _, outcome := range results { + for i, exc := range outcome.Exceptions { + if code, ok := exc.Metadata["code"].(string); ok && code == "test.check_a" { + checkAException = &outcome.Exceptions[i] + break + } + } + } + require.NotNil(t, checkAException, "Expected check_a to surface as an exception for comp1") + assert.Equal(t, "2024-01-01T00:00:00Z", checkAException.Metadata["effective_on"], "Expected effective_on to be stamped") + assert.Equal(t, "2025-01-01T00:00:00Z", checkAException.Metadata["effective_until"], "Expected effective_until to be stamped") }) t.Run("comp2 evaluates both checks", func(t *testing.T) { @@ -474,3 +488,149 @@ deny contains result if { assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp2") }) } + +func TestOPAEvaluatorIntegrationVolatileExclusionSurfacesAsException(t *testing.T) { + ctx := context.Background() + + effectiveOn := "2025-01-01T00:00:00Z" + effectiveUntil := "2030-01-01T00:00:00Z" + now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + + policyContent := `package mypkg + +import rego.v1 + +# METADATA +# title: Always failing rule +# custom: +# short_name: always_fail +deny contains result if { + result := { + "code": "mypkg.always_fail", + "msg": "This rule always fails", + } +} +` + // Each scenario gets its own temp dir so download cache entries don't collide. + makeEvaluator := func(t *testing.T, src ecc.Source) (Evaluator, string) { + t.Helper() + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + ps := &source.PolicyUrl{Url: "file://" + policyDir, Kind: source.PolicyKind} + + cp := &mockConfigProvider{} + cp.On("EffectiveTime").Return(now) + cp.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + cp.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{Policy: []string{"file://" + policyDir}}}, + }) + ev, err := NewOPAEvaluator(ctx, []source.PolicySource{ps}, cp, src, nil) + require.NoError(t, err) + t.Cleanup(ev.Destroy) + return ev, tmpDir + } + + t.Run("exact-code volatile exclusion surfaces as exception with metadata", func(t *testing.T) { + ev, tmpDir := makeEvaluator(t, ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{{ + Value: "mypkg.always_fail", + EffectiveOn: effectiveOn, + EffectiveUntil: effectiveUntil, + }}, + }, + }) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, []byte(`{}`), 0o600)) + + results, err := ev.Evaluate(ctx, EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"}) + require.NoError(t, err) + + var failures, exceptions []Result + for _, outcome := range results { + failures = append(failures, outcome.Failures...) + exceptions = append(exceptions, outcome.Exceptions...) + } + assert.Empty(t, failures, "excluded rule must not appear as failure") + require.Len(t, exceptions, 1) + assert.Equal(t, effectiveOn, exceptions[0].Metadata[metadataEffectiveOn]) + assert.Equal(t, effectiveUntil, exceptions[0].Metadata[metadataEffectiveUntil]) + }) + + t.Run("package-level volatile exclusion surfaces as exception with metadata", func(t *testing.T) { + ev, tmpDir := makeEvaluator(t, ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{{ + Value: "mypkg", + EffectiveOn: effectiveOn, + EffectiveUntil: effectiveUntil, + }}, + }, + }) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, []byte(`{}`), 0o600)) + + results, err := ev.Evaluate(ctx, EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"}) + require.NoError(t, err) + + var failures, exceptions []Result + for _, outcome := range results { + failures = append(failures, outcome.Failures...) + exceptions = append(exceptions, outcome.Exceptions...) + } + assert.Empty(t, failures, "excluded rule must not appear as failure") + require.Len(t, exceptions, 1) + assert.Equal(t, effectiveOn, exceptions[0].Metadata[metadataEffectiveOn]) + assert.Equal(t, effectiveUntil, exceptions[0].Metadata[metadataEffectiveUntil]) + }) + + t.Run("volatile exclusion without effective_until has no expiry stamped", func(t *testing.T) { + ev, tmpDir := makeEvaluator(t, ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{{ + Value: "mypkg.always_fail", + EffectiveOn: effectiveOn, + }}, + }, + }) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, []byte(`{}`), 0o600)) + + results, err := ev.Evaluate(ctx, EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"}) + require.NoError(t, err) + + var exceptions []Result + for _, outcome := range results { + exceptions = append(exceptions, outcome.Exceptions...) + } + require.Len(t, exceptions, 1) + assert.Equal(t, effectiveOn, exceptions[0].Metadata[metadataEffectiveOn]) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveUntil) + }) + + t.Run("permanent config exclusion surfaces as exception with no time fields", func(t *testing.T) { + ev, tmpDir := makeEvaluator(t, ecc.Source{ + Config: &ecc.SourceConfig{ + Exclude: []string{"mypkg.always_fail"}, + }, + }) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, []byte(`{}`), 0o600)) + + results, err := ev.Evaluate(ctx, EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"}) + require.NoError(t, err) + + var failures, exceptions []Result + for _, outcome := range results { + failures = append(failures, outcome.Failures...) + exceptions = append(exceptions, outcome.Exceptions...) + } + assert.Empty(t, failures, "excluded rule must not appear as failure") + require.Len(t, exceptions, 1) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveOn) + assert.NotContains(t, exceptions[0].Metadata, metadataEffectiveUntil) + }) +} diff --git a/internal/input/report.go b/internal/input/report.go index 6e01b3b8f..df43505a0 100644 --- a/internal/input/report.go +++ b/internal/input/report.go @@ -35,10 +35,12 @@ import ( ) type Input struct { - FilePath string `json:"filepath"` - Violations []evaluator.Result `json:"violations"` - Warnings []evaluator.Result `json:"warnings"` - Successes []evaluator.Result `json:"successes"` + FilePath string `json:"filepath"` + Violations []evaluator.Result `json:"violations"` + Warnings []evaluator.Result `json:"warnings"` + Successes []evaluator.Result `json:"successes"` + // Exceptions holds policy rules excluded from evaluation by the policy configuration, + // carrying time-bounded waiver metadata such as effective_on and effective_until. Exceptions []evaluator.Result `json:"exceptions,omitempty"` Success bool `json:"success"` SuccessCount int `json:"success-count"`