feat: improve reporting (#317)

Signed-off-by: Charles-Edouard Brétéché <[email protected]>

pkg/commands/scan/options.go

@@ -88,7 +88,8 @@ func (c *options) run(cmd *cobra.Command, _ []string) error {
 				if rule.Error != nil {
 					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "ERROR:", rule.Error.Error())
 				} else if len(rule.Violations) != 0 {
-					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "FAILED:", strings.Join(rule.Violations, "; "))
+					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "FAILED")
+					out.println(rule.Violations.Error())
 				} else {
 					// TODO: handle skip, warn
 					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "PASSED")

pkg/commands/scan/output.go

@@ -32,7 +32,7 @@ func (t *jsonOutput) println(args ...any) {
 }
 
 func (t *jsonOutput) responses(responses ...jsonengine.Response) {
-	payload, err := json.MarshalIndent(responses, "", "  ")
+	payload, err := json.MarshalIndent(ToReports(responses...), "", "  ")
 	if err != nil {
 		fmt.Fprintln(t.out, err)
 	} else {

pkg/commands/scan/report.go

@@ -0,0 +1,81 @@
+package scan
+
+import (
+	"github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1"
+	jsonengine "github.com/kyverno/kyverno-json/pkg/json-engine"
+)
+
+type Report struct {
+	Resource any            `json:"resource"`
+	Results  []PolicyReport `json:"results,omitempty"`
+}
+
+type PolicyReport struct {
+	Policy *v1alpha1.ValidatingPolicy `json:"policy"`
+	Rules  []RuleReport               `json:"rules,omitempty"`
+}
+
+type RuleReport struct {
+	Rule       v1alpha1.ValidatingRule `json:"rule"`
+	Identifier string                  `json:"identifier,omitempty"`
+	Error      string                  `json:"error,omitempty"`
+	Violations []ViolationReport       `json:"violations,omitempty"`
+}
+
+type ViolationReport struct {
+	Message string        `json:"message,omitempty"`
+	Errors  []ErrorReport `json:"errors,omitempty"`
+}
+
+type ErrorReport struct {
+	Type     string `json:"type,omitempty"`
+	Field    string `json:"field,omitempty"`
+	BadValue any    `json:"badValue,omitempty"`
+	Detail   string `json:"detail,omitempty"`
+}
+
+func ToReport(response jsonengine.Response) Report {
+	report := Report{
+		Resource: response.Resource,
+	}
+	for _, policy := range response.Policies {
+		policyReport := PolicyReport{
+			Policy: policy.Policy,
+		}
+		for _, rule := range policy.Rules {
+			ruleReport := RuleReport{
+				Rule:       rule.Rule,
+				Identifier: rule.Identifier,
+			}
+			if rule.Error != nil {
+				ruleReport.Error = rule.Error.Error()
+			}
+			for _, violation := range rule.Violations {
+				violationReport := ViolationReport{
+					Message: violation.Message,
+				}
+				for _, err := range violation.ErrorList {
+					if err != nil {
+						violationReport.Errors = append(violationReport.Errors, ErrorReport{
+							Type:     string(err.Type),
+							Field:    err.Field,
+							BadValue: err.BadValue,
+							Detail:   err.Detail,
+						})
+					}
+				}
+				ruleReport.Violations = append(ruleReport.Violations, violationReport)
+			}
+			policyReport.Rules = append(policyReport.Rules, ruleReport)
+		}
+		report.Results = append(report.Results, policyReport)
+	}
+	return report
+}
+
+func ToReports(responses ...jsonengine.Response) (reports []Report) {
+	for _, response := range responses {
+		reports = append(reports, ToReport(response))
+	}
+	return reports
+}

pkg/json-engine/engine.go

@@ -32,7 +32,7 @@ type RuleResponse struct {
 	Rule       v1alpha1.ValidatingRule
 	Identifier string
 	Error      error
-	Violations []string
+	Violations matching.Results
 }
 
 // PolicyResult specifies state of a policy result

pkg/matching/match.go

@@ -2,6 +2,7 @@ package matching
 
 import (
 	"context"
+	"strings"
 
 	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1"
@@ -10,12 +11,39 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
-func MatchAssert(ctx context.Context, path *field.Path, match *v1alpha1.Assert, actual any, bindings binding.Bindings, opts ...template.Option) ([]string, error) {
+type Result struct {
+	field.ErrorList
+	Message string
+}
+
+func (r Result) Error() string {
+	var lines []string
+	if r.Message != "" {
+		lines = append(lines, "-> "+r.Message)
+	}
+	for _, err := range r.ErrorList {
+		lines = append(lines, " -> "+err.Error())
+	}
+	return strings.Join(lines, "\n")
+}
+
+type Results []Result
+
+func (r Results) Error() string {
+	var lines []string
+	for _, err := range r {
+		lines = append(lines, err.Error())
+	}
+	return strings.Join(lines, "\n")
+}
+
+// func MatchAssert(ctx context.Context, path *field.Path, match *v1alpha1.Assert, actual any, bindings binding.Bindings, opts ...template.Option) ([]error, error) {
+func MatchAssert(ctx context.Context, path *field.Path, match *v1alpha1.Assert, actual any, bindings binding.Bindings, opts ...template.Option) ([]Result, error) {
 	if match == nil || (len(match.Any) == 0 && len(match.All) == 0) {
 		return nil, field.Invalid(path, match, "an empty assert is not valid")
 	} else {
 		if len(match.Any) != 0 {
-			var fails []string
+			var fails []Result
 			path := path.Child("any")
 			for i, assertion := range match.Any {
 				checkFails, err := assert.Assert(ctx, path.Index(i).Child("check"), assert.Parse(ctx, assertion.Check.Value), actual, bindings, opts...)
@@ -26,34 +54,34 @@ func MatchAssert(ctx context.Context, path *field.Path, match *v1alpha1.Assert,
 					fails = nil
 					break
 				}
+				fail := Result{
+					ErrorList: checkFails,
+				}
 				if assertion.Message != "" {
-					msg := template.String(ctx, assertion.Message, actual, bindings, opts...)
-					msg += ": " + checkFails.ToAggregate().Error()
-					fails = append(fails, msg)
-				} else {
-					fails = append(fails, checkFails.ToAggregate().Error())
+					fail.Message = template.String(ctx, assertion.Message, actual, bindings, opts...)
 				}
+				fails = append(fails, fail)
 			}
 			if fails != nil {
 				return fails, nil
 			}
 		}
 		if len(match.All) != 0 {
-			var fails []string
+			var fails []Result
 			path := path.Child("all")
 			for i, assertion := range match.All {
 				checkFails, err := assert.Assert(ctx, path.Index(i).Child("check"), assert.Parse(ctx, assertion.Check.Value), actual, bindings, opts...)
 				if err != nil {
 					return fails, err
 				}
 				if len(checkFails) > 0 {
+					fail := Result{
+						ErrorList: checkFails,
+					}
 					if assertion.Message != "" {
-						msg := template.String(ctx, assertion.Message, actual, bindings, opts...)
-						msg += ": " + checkFails.ToAggregate().Error()
-						fails = append(fails, msg)
-					} else {
-						fails = append(fails, checkFails.ToAggregate().Error())
+						fail.Message = template.String(ctx, assertion.Message, actual, bindings, opts...)
 					}
+					fails = append(fails, fail)
 				}
 			}
 			return fails, nil

pkg/server/model/response.go

@@ -1,8 +1,6 @@
 package model
 
 import (
-	"strings"
-
 	jsonengine "github.com/kyverno/kyverno-json/pkg/json-engine"
 )
 
@@ -52,7 +50,7 @@ func makeMessage(rule jsonengine.RuleResponse) string {
 		return rule.Error.Error()
 	}
 	if len(rule.Violations) != 0 {
-		return strings.Join(rule.Violations, "; ")
+		return rule.Violations.Error()
 	}
 	return ""
 }

test/api/go/main/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"log"
-	"strings"
 
 	jsonengine "github.com/kyverno/kyverno-json/pkg/json-engine"
 	"github.com/kyverno/kyverno-json/pkg/policy"
@@ -70,7 +69,7 @@ func main() {
 			if rule.Error != nil {
 				logger.Printf("error: %s/%s -> %s: %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Error)
 			} else if len(rule.Violations) != 0 {
-				logger.Printf("fail: %s/%s -> %s: %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, strings.Join(rule.Violations, "; "))
+				logger.Printf("fail: %s/%s -> %s\n%s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Violations.Error())
 			} else {
 				logger.Printf("pass: %s/%s -> %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier)
 			}

test/commands/scan/dockerfile/out.txt

@@ -2,5 +2,9 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- check-dockerfile / deny-external-calls /  FAILED: HTTP calls are not allowed: all[0].check.~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://')): Invalid value: true: Expected value: false; wget is not allowed: all[3].check.~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget')): Invalid value: true: Expected value: false
+- check-dockerfile / deny-external-calls /  FAILED
+-> HTTP calls are not allowed
+ -> all[0].check.~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://')): Invalid value: true: Expected value: false
+-> wget is not allowed
+ -> all[3].check.~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget')): Invalid value: true: Expected value: false
 Done

test/commands/scan/payload-yaml/out.txt

@@ -2,5 +2,7 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED: Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}: all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
+- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED
+-> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}
+ -> all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
 Done

test/commands/scan/pod-no-latest/out.txt

@@ -2,5 +2,14 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / pod-no-latest / webserver FAILED: [all[0].check.spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false, all[0].check.spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false, all[0].check.spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false]; [all[1].check.spec.~.containers->foo[0].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[1].check.spec.~.containers->foo[1].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[1].check.spec.~.containers->foo[2].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false]; [all[2].check.~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[2].check.~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[2].check.~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest')): Invalid value: true: Expected value: false]
+- test / pod-no-latest / webserver FAILED
+ -> all[0].check.spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false
+ -> all[0].check.spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false
+ -> all[0].check.spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false
+ -> all[1].check.spec.~.containers->foo[0].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false
+ -> all[1].check.spec.~.containers->foo[1].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false
+ -> all[1].check.spec.~.containers->foo[2].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false
+ -> all[2].check.~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest')): Invalid value: true: Expected value: false
+ -> all[2].check.~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest')): Invalid value: true: Expected value: false
+ -> all[2].check.~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest')): Invalid value: true: Expected value: false
 Done

test/commands/scan/tf-plan/out.txt

@@ -2,5 +2,7 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED: Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}: all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
+- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED
+-> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}
+ -> all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
 Done

test/commands/scan/tf-s3/out.txt

@@ -2,5 +2,6 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- s3 / check-tags /  FAILED: all[0].check.planned_values.root_module.~.resources[0].values.(keys(tags_all)).(contains(@, 'Team')): Invalid value: false: Expected value: true
+- s3 / check-tags /  FAILED
+ -> all[0].check.planned_values.root_module.~.resources[0].values.(keys(tags_all)).(contains(@, 'Team')): Invalid value: false: Expected value: true
 Done

test/commands/scan/wildcard/out.txt

@@ -2,5 +2,6 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / bucket1 FAILED: all[0].check.tags.(wildcard('?*', Team)): Invalid value: true: Expected value: false
+- required-s3-tags / require-team-tag / bucket1 FAILED
+ -> all[0].check.tags.(wildcard('?*', Team)): Invalid value: true: Expected value: false
 Done