Add warn reason (#547)

* Update check.go

Added new warn_reason value which gives a brief explanation about why the not scored tests failed

* Update common.go

Changed when a not scored test fails because it has a wrong syntax audit command or just running something that can't be run the print the failure. but if the test just fails because it doesn't line up with the cis hardening recommendations then print the remediation text.

* Update check/check.go

fix typo

Co-Authored-By: Liz Rice <liz@lizrice.com>

* Update check.go

* Update common.go

* Update check.go

added back os.Exit(1) to  exitWithError

* Update job-master.data

Change some tests output to fit warn reason. (No change to the summary)

* Update job-node.data

Changed some tests output to fit warn reason. (No change to the summary)

* Update job.data

Change some tests output to fit warn reason. (No change to the summary)

* Update common.go

Keep to old way to print manual test output

Co-authored-by: Liz Rice <liz@lizrice.com>
Co-authored-by: Roberto Rojas <robertojrojas@gmail.com>

check/check.go

@@ -80,6 +80,7 @@ type Check struct {
 	ActualValue    string `json:"actual_value"`
 	Scored         bool   `json:"scored"`
 	ExpectedResult string `json:"expected_result"`
+	Reason     string `json:"reason,omitempty"`
 }
 
 // Runner wraps the basic Run method.
@@ -107,18 +108,21 @@ func (c *Check) run() State {
 	// without tests return a 'WARN' to alert
 	// the user that this check needs attention
 	if c.Scored && len(strings.TrimSpace(c.Type)) == 0 && c.Tests == nil {
+		c.Reason = "There are no tests"
 		c.State = WARN
 		return c.State
 	}
 
 	// If check type is skip, force result to INFO
 	if c.Type == "skip" {
+		c.Reason = "Test marked as skip"
 		c.State = INFO
 		return c.State
 	}
 
 	// If check type is manual force result to WARN
 	if c.Type == MANUAL {
+		c.Reason = "Test marked as a manual test"
 		c.State = WARN
 		return c.State
 	}
@@ -128,6 +132,7 @@ func (c *Check) run() State {
 
 	state, finalOutput, retErrmsgs := performTest(c.Audit, c.Commands, c.Tests)
 	if len(state) > 0 {
+		c.Reason = retErrmsgs
 		c.State = state
 		return c.State
 	}
@@ -163,6 +168,7 @@ func (c *Check) run() State {
 
 		state, finalOutput, retErrmsgs = performTest(c.AuditConfig, c.ConfigCommands, currentTests)
 		if len(state) > 0 {
+			c.Reason = retErrmsgs
 			c.State = state
 			return c.State
 		}
@@ -177,6 +183,7 @@ func (c *Check) run() State {
 		if c.Scored {
 			c.State = FAIL
 		} else {
+			c.Reason = errmsgs
 			c.State = WARN
 		}
 	}
@@ -256,13 +263,13 @@ func isShellCommand(s string) bool {
 
 func performTest(audit string, commands []*exec.Cmd, tests *tests) (State, *testOutput, string) {
 	if len(strings.TrimSpace(audit)) == 0 {
-		return "", failTestItem("missing command"), ""
+		return "", failTestItem("missing command"), "missing audit command"
 	}
 
 	var out bytes.Buffer
 	state, retErrmsgs := runExecCommands(audit, commands, &out)
 	if len(state) > 0 {
-		return state, nil, ""
+		return state, nil, retErrmsgs
 	}
 	errmsgs := retErrmsgs
 
@@ -281,6 +288,7 @@ func runExecCommands(audit string, commands []*exec.Cmd, out *bytes.Buffer) (Sta
 	// Check if command exists or exit with WARN.
 	for _, cmd := range commands {
 		if !isShellCommand(cmd.Path) {
+			errmsgs += fmt.Sprintf("Command '%s' not found\n", cmd.Path)
 			return WARN, errmsgs
 		}
 	}

cmd/common.go

@@ -180,9 +180,17 @@ func prettyPrint(r *check.Controls, summary check.Summary) {
 			colors[check.WARN].Printf("== Remediations ==\n")
 			for _, g := range r.Groups {
 				for _, c := range g.Checks {
-					if c.State == check.FAIL || c.State == check.WARN {
+					if c.State == check.FAIL {
 						fmt.Printf("%s %s\n", c.ID, c.Remediation)
 					}
+					if c.State == check.WARN {
+						// Print the error if test failed due to problem with the audit command
+						if c.Reason != "" && c.Type != "manual"{
+							fmt.Printf("%s audit test did not run: %s\n", c.ID, c.Reason)
+						} else {
+							fmt.Printf("%s %s\n", c.ID, c.Remediation)
+						}
+					}
 				}
 			}
 			fmt.Println()

integration/testdata/job-node.data

@@ -1,4 +1,4 @@
- [INFO] 2 Worker Node Security Configuration
+[INFO] 2 Worker Node Security Configuration
 [INFO] 2.1 Kubelet
 [PASS] 2.1.1 Ensure that the --anonymous-auth argument is set to false (Scored)
 [PASS] 2.1.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)