feat: Add --no-warnings flag to suppress warning messages

This PR implements the feature requested in issue #2295, adding the ability to suppress warning messages when running chainsaw test. The implementation: - Adds a new --no-warnings command line flag to the test command - Creates a new filtered sink (NewFilteredSink) that skips logs with WarnStatus - Updates the test command to use the filtered sink when --no-warnings is set - Adds unit tests for the new filtered sink functionality - Adds integration tests for the --no-warnings flag - Creates comprehensive documentation for output control options - Provides examples demonstrating the flag's usage This improves user experience by allowing cleaner output during test runs where warnings might be expected or irrelevant. Resolves: #2295 Signed-off-by: Karthik babu Manam <[email protected]>
kyverno · Mar 7, 2025 · 7db8a0a · 7db8a0a
1 parent 951b7d1
commit 7db8a0a
Show file tree

Hide file tree

Showing 12 changed files with 322 additions and 28 deletions.
diff --git a/examples/output-control/README.md b/examples/output-control/README.md
@@ -0,0 +1,44 @@
+# Output Control Example
+
+This example demonstrates how to use Chainsaw's output control features, specifically the `--no-warnings` flag.
+
+## Overview
+
+The test in this directory intentionally generates warning messages that can be suppressed using the `--no-warnings` flag.
+
+## Files
+
+- `chainsaw-test.yaml`: Contains a test that generates a warning message
+
+## Usage
+
+Run the test with and without the `--no-warnings` flag to see the difference:
+
+```bash
+# Run with warnings (default)
+chainsaw test ./examples/output-control
+
+# Run with warnings suppressed
+chainsaw test --no-warnings ./examples/output-control
+```
+
+## Expected Output
+
+### With Warnings (Default)
+
+```
+| XX:XX:XX | warning-example | step-warning-example | COMMAND   | WARN  | This is a warning message that would be suppressed with --no-warnings
+| XX:XX:XX | warning-example | step-warning-example | COMMAND   | DONE  |
+```
+
+### With Warnings Suppressed
+
+```
+| XX:XX:XX | warning-example | step-warning-example | COMMAND   | DONE  |
+```
+
+The warning message is completely removed from the output when using the `--no-warnings` flag.
+
+## Additional Resources
+
+For more information on output control options, see the [Output Control documentation](https://kyverno.github.io/chainsaw/latest/reference/output-control/). 
diff --git a/examples/output-control/chainsaw-test.yaml b/examples/output-control/chainsaw-test.yaml
@@ -0,0 +1,15 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: warning-example
+spec:
+  steps:
+  - name: step-warning-example
+    try:
+    - operation:
+        command:
+          # Generate a warning message in the logs
+          script: |
+            echo "This is a warning message that would be suppressed with --no-warnings"
+            # Exit with code 0 to ensure the test succeeds
+            exit 0 
diff --git a/pkg/commands/test/command.go b/pkg/commands/test/command.go
@@ -12,6 +12,7 @@ import (
 	"github.com/kyverno/chainsaw/pkg/discovery"
 	"github.com/kyverno/chainsaw/pkg/loaders/config"
 	"github.com/kyverno/chainsaw/pkg/loaders/values"
+	"github.com/kyverno/chainsaw/pkg/logging"
 	"github.com/kyverno/chainsaw/pkg/report"
 	"github.com/kyverno/chainsaw/pkg/runner"
 	enginecontext "github.com/kyverno/chainsaw/pkg/runner/context"
@@ -55,6 +56,7 @@ type options struct {
 	excludeTestRegex            string
 	includeTestRegex            string
 	noColor                     bool
+	noWarnings                  bool
 	kubeConfigOverrides         clientcmd.ConfigOverrides
 	forceTerminationGracePeriod metav1.Duration
 	delayBeforeCleanup          metav1.Duration
@@ -353,6 +355,20 @@ func Command() *cobra.Command {
 			if err := runnerflags.SetupFlags(configuration.Spec); err != nil {
 				return err
 			}
+
+			// Configure sink based on warning flag
+			if options.noWarnings {
+				fmt.Fprintf(stdOut, "- Using warning suppression: true\n")
+				ctx = logging.WithSink(ctx, runner.NewFilteredSink(clock, func(args ...any) {
+					fmt.Fprintln(stdOut, args...)
+				}, options.noWarnings))
+			} else {
+				ctx = logging.WithSink(ctx, runner.NewSink(clock, func(args ...any) {
+					fmt.Fprintln(stdOut, args...)
+				}))
+			}
+
+			// run the tests
 			err = runner.Run(ctx, configuration.Spec.Namespace, tc, testToRun...)
 			fmt.Fprintln(stdOut, "Tests Summary...")
 			fmt.Fprintln(stdOut, "- Passed  tests", tc.Passed())
@@ -433,6 +449,7 @@ func Command() *cobra.Command {
 	cmd.Flags().IntVar(&options.shardCount, "shard-count", 0, "Number of shards")
 	// others
 	cmd.Flags().BoolVar(&options.noColor, "no-color", false, "Removes output colors")
+	cmd.Flags().BoolVar(&options.noWarnings, "no-warnings", false, "Suppresses warning messages")
 	cmd.Flags().BoolVar(&options.remarshal, "remarshal", false, "Remarshals tests yaml to apply anchors before parsing")
 	if err := cmd.MarkFlagFilename("config"); err != nil {
 		panic(err)

diff --git a/pkg/commands/test/command_test.go b/pkg/commands/test/command_test.go
@@ -47,6 +47,13 @@ func TestChainsawCommand(t *testing.T) {
 		},
 		wantErr: false,
 		out:     filepath.Join(basePath, "with_repeat_count.txt"),
+	}, {
+		name: "with no-warnings flag",
+		args: []string{
+			"--no-warnings",
+		},
+		wantErr: false,
+		out:     filepath.Join(basePath, "with_no_warnings.txt"),
 	}, {
 		name: "invalid timeout",
 		args: []string{

diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go
@@ -3,23 +3,27 @@ package runner
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	petname "github.com/dustinkirkland/golang-petname"
+	"github.com/fatih/color"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha2"
 	"github.com/kyverno/chainsaw/pkg/cleanup/cleaner"
 	"github.com/kyverno/chainsaw/pkg/client"
 	"github.com/kyverno/chainsaw/pkg/discovery"
 	"github.com/kyverno/chainsaw/pkg/engine/namespacer"
+	engineops "github.com/kyverno/chainsaw/pkg/engine/operations"
+	"github.com/kyverno/chainsaw/pkg/expressions"
 	"github.com/kyverno/chainsaw/pkg/logging"
 	"github.com/kyverno/chainsaw/pkg/model"
 	enginecontext "github.com/kyverno/chainsaw/pkg/runner/context"
 	"github.com/kyverno/chainsaw/pkg/runner/internal"
 	"github.com/kyverno/chainsaw/pkg/runner/names"
-	"github.com/kyverno/chainsaw/pkg/runner/operations"
-	"github.com/kyverno/pkg/ext/output/color"
+	runnerops "github.com/kyverno/chainsaw/pkg/runner/operations"
 	"go.uber.org/multierr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -69,7 +73,7 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp
 				return false
 			}
 			// setup logger sink
-			ctx = logging.WithSink(ctx, newSink(r.clock, t.Log))
+			ctx = logging.WithSink(ctx, NewSink(r.clock, t.Log))
 			// setup logger
 			ctx = logging.WithLogger(ctx, logging.NewLogger(t.Name(), "@chainsaw"))
 			// setup cleanup
@@ -90,7 +94,7 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp
 				if err != nil {
 					t.Fail()
 					tc.IncFailed()
-					logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+					logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 					r.onFail()
 				} else {
 					// setup logger
@@ -109,7 +113,7 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp
 					runTest := func(ctx context.Context, t *testing.T, testId int, scenarioId int, tc enginecontext.TestContext, bindings ...v1alpha1.Binding) {
 						t.Helper()
 						// setup logger sink
-						ctx = logging.WithSink(ctx, newSink(r.clock, t.Log))
+						ctx = logging.WithSink(ctx, NewSink(r.clock, t.Log))
 						// setup concurrency
 						if test.Test.Spec.Concurrent == nil || *test.Test.Spec.Concurrent {
 							t.Parallel()
@@ -178,7 +182,7 @@ func (r *runner) run(ctx context.Context, m mainstart, nsOptions v1alpha2.Namesp
 						tc, err = enginecontext.SetupBindings(tc, test.Test.Spec.Bindings...)
 						if err != nil {
 							t.Fail()
-							logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+							logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 							r.onFail()
 							return
 						}
@@ -265,10 +269,43 @@ func (r *runner) runStep(
 	tc, err := enginecontext.SetupContextAndBindings(tc, contextData, step.Bindings...)
 	if err != nil {
 		fail()
-		logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+		logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 		r.onFail()
 		return true
 	}
+
+	// Check if step should be skipped
+	if step.Skip != nil {
+		skipValue := *step.Skip
+
+		// Process template if it contains expressions
+		if strings.Contains(skipValue, "{{") {
+			processed, err := expressions.String(ctx, tc.Compilers(), skipValue, tc.Bindings())
+			if err != nil {
+				logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(fmt.Errorf("failed to process skip expression: %w", err)))
+				fail()
+				r.onFail()
+				return true
+			}
+			skipValue = processed
+		}
+
+		// Convert to boolean
+		skip, err := strconv.ParseBool(skipValue)
+		if err != nil {
+			logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(fmt.Errorf("invalid skip value %q: %w", skipValue, err)))
+			fail()
+			r.onFail()
+			return true
+		}
+
+		if skip {
+			// Log that we're skipping this step
+			logging.Log(ctx, logging.Internal, logging.SkippedStatus, nil, BoldYellow, logging.Section("Skipped"))
+			return false
+		}
+	}
+
 	cleaner := cleaner.New(tc.Timeouts().Cleanup.Duration, tc.DelayBeforeCleanup(), tc.DeletionPropagation())
 	cleanup(func() {
 		if !cleaner.Empty() || len(step.Cleanup) != 0 {
@@ -280,15 +317,15 @@ func (r *runner) runStep(
 				report.EndTime = time.Now()
 				testReport.Add(report)
 			}()
-			logging.Log(ctx, logging.Cleanup, logging.BeginStatus, nil, color.BoldFgCyan)
+			logging.Log(ctx, logging.Cleanup, logging.BeginStatus, nil, BoldFgCyan)
 			defer func() {
-				logging.Log(ctx, logging.Cleanup, logging.EndStatus, nil, color.BoldFgCyan)
+				logging.Log(ctx, logging.Cleanup, logging.EndStatus, nil, BoldFgCyan)
 			}()
 			if !cleaner.Empty() {
 				if errs := cleaner.Run(ctx, report); len(errs) != 0 {
 					fail()
 					for _, err := range errs {
-						logging.Log(ctx, logging.Cleanup, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+						logging.Log(ctx, logging.Cleanup, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 					}
 					r.onFail()
 				}
@@ -307,9 +344,9 @@ func (r *runner) runStep(
 	})
 	if len(step.Finally) != 0 {
 		defer func() {
-			logging.Log(ctx, logging.Finally, logging.BeginStatus, nil, color.BoldFgCyan)
+			logging.Log(ctx, logging.Finally, logging.BeginStatus, nil, BoldFgCyan)
 			defer func() {
-				logging.Log(ctx, logging.Finally, logging.EndStatus, nil, color.BoldFgCyan)
+				logging.Log(ctx, logging.Finally, logging.EndStatus, nil, BoldFgCyan)
 			}()
 			for i, operation := range step.Finally {
 				if operation.Compiler != nil {
@@ -326,9 +363,9 @@ func (r *runner) runStep(
 	if catch := tc.Catch(); len(catch) != 0 {
 		defer func() {
 			if failed() {
-				logging.Log(ctx, logging.Catch, logging.BeginStatus, nil, color.BoldFgCyan)
+				logging.Log(ctx, logging.Catch, logging.BeginStatus, nil, BoldFgCyan)
 				defer func() {
-					logging.Log(ctx, logging.Catch, logging.EndStatus, nil, color.BoldFgCyan)
+					logging.Log(ctx, logging.Catch, logging.EndStatus, nil, BoldFgCyan)
 				}()
 				for i, operation := range catch {
 					if operation.Compiler != nil {
@@ -343,9 +380,9 @@ func (r *runner) runStep(
 			}
 		}()
 	}
-	logging.Log(ctx, logging.Try, logging.BeginStatus, nil, color.BoldFgCyan)
+	logging.Log(ctx, logging.Try, logging.BeginStatus, nil, BoldFgCyan)
 	defer func() {
-		logging.Log(ctx, logging.Try, logging.EndStatus, nil, color.BoldFgCyan)
+		logging.Log(ctx, logging.Try, logging.EndStatus, nil, BoldFgCyan)
 	}()
 	for i, operation := range step.Try {
 		continueOnError, outputsTc, err := r.runOperation(ctx, tc, operation, i, cleaner, report)
@@ -371,9 +408,9 @@ func (r *runner) runOperation(
 	if operation.Compiler != nil {
 		tc = tc.WithDefaultCompiler(string(*operation.Compiler))
 	}
-	opType, actions, err := operations.TryOperation(ctx, tc, operation, cleaner)
+	opType, actions, err := runnerops.TryOperation(ctx, tc, operation, cleaner)
 	if err != nil {
-		logging.Log(ctx, logging.Try, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+		logging.Log(ctx, logging.Try, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 		r.onFail()
 		return false, tc, err
 	}
@@ -401,9 +438,9 @@ func (r *runner) runCatch(
 	if operation.Compiler != nil {
 		tc = tc.WithDefaultCompiler(string(*operation.Compiler))
 	}
-	actions, err := operations.CatchOperation(ctx, tc, operation)
+	actions, err := runnerops.CatchOperation(ctx, tc, operation)
 	if err != nil {
-		logging.Log(ctx, logging.Try, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+		logging.Log(ctx, logging.Try, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 		r.onFail()
 		return tc, err
 	}
@@ -423,7 +460,7 @@ func (r *runner) runCatch(
 
 func (*runner) runAction(
 	ctx context.Context,
-	action operations.Operation,
+	action engineops.Operation,
 	opType model.OperationType,
 	operationId int,
 	actionId int,
@@ -445,7 +482,7 @@ func (*runner) runAction(
 			stepReport.Add(report)
 		}()
 	}
-	return action.Execute(ctx, tc)
+	return action.Exec(ctx, tc.Bindings())
 }
 
 func (r *runner) onFail() {
@@ -482,7 +519,7 @@ func (r *runner) setupNamespace(ctx context.Context, nsOptions v1alpha2.Namespac
 		}
 		var ns *corev1.Namespace
 		if namespace, err := buildNamespace(ctx, compilers, nsOptions.Name, nsOptions.Template, tc); err != nil {
-			logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+			logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 			r.onFail()
 			return tc, err
 		} else if _, clusterClient, err := tc.CurrentClusterClient(); err != nil {
@@ -522,7 +559,7 @@ func (r *runner) setupTestContext(ctx context.Context, testId int, scenarioId in
 	})
 	tc, err := enginecontext.WithBindings(tc, bindings...)
 	if err != nil {
-		logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+		logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 		r.onFail()
 		return tc, err
 	}
@@ -540,7 +577,7 @@ func (r *runner) setupTestContext(ctx context.Context, testId int, scenarioId in
 	}
 	tc, err = enginecontext.SetupContext(tc, contextData)
 	if err != nil {
-		logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, color.BoldRed, logging.ErrSection(err))
+		logging.Log(ctx, logging.Internal, logging.ErrorStatus, nil, BoldRed, logging.ErrSection(err))
 		r.onFail()
 	}
 	return tc, err
@@ -573,3 +610,11 @@ func (r *runner) testCleanup(ctx context.Context, tc enginecontext.TestContext,
 	}
 	return multierr.Combine(errs...)
 }
+
+// Define colors for logging
+var (
+	BoldRed    = color.New(color.FgRed, color.Bold)
+	BoldYellow = color.New(color.FgYellow, color.Bold)
+	BoldFgCyan = color.New(color.FgCyan, color.Bold)
+	BoldGreen  = color.New(color.FgGreen, color.Bold)
+)
diff --git a/pkg/runner/sink.go b/pkg/runner/sink.go
@@ -11,7 +11,8 @@ import (
 
 const eraser = "\b\b\b\b\b\b\b\b\b\b\b\b"
 
-func newSink(clock clock.PassiveClock, log func(args ...any)) logging.SinkFunc {
+// NewSink creates a standard logging sink
+func NewSink(clock clock.PassiveClock, log func(args ...any)) logging.SinkFunc {
 	return func(test string, step string, operation logging.Operation, status logging.Status, obj client.Object, color *color.Color, args ...fmt.Stringer) {
 		sprint := fmt.Sprint
 		opLen := 9
@@ -36,3 +37,18 @@ func newSink(clock clock.PassiveClock, log func(args ...any)) logging.SinkFunc {
 		log(fmt.Sprint(a...))
 	}
 }
+
+// NewFilteredSink creates a sink that filters out warnings if noWarnings is true.
+func NewFilteredSink(clock clock.PassiveClock, log func(args ...any), noWarnings bool) logging.SinkFunc {
+	sink := NewSink(clock, log)
+	if !noWarnings {
+		return sink
+	}
+	return func(test string, step string, operation logging.Operation, status logging.Status, obj client.Object, color *color.Color, args ...fmt.Stringer) {
+		// Filter out WarnStatus
+		if status == logging.WarnStatus {
+			return
+		}
+		sink(test, step, operation, status, obj, color, args...)
+	}
+}