Support multiple values flag when check the audit output (#652)

check/check.go

@@ -74,6 +74,7 @@ type Check struct {
 	State          `json:"status"`
 	ActualValue    string `json:"actual_value"`
 	Scored         bool   `json:"scored"`
+	IsMultiple     bool   `yaml:"use_multiple_values"`
 	ExpectedResult string `json:"expected_result"`
 	Reason         string `json:"reason,omitempty"`
 }
@@ -125,7 +126,7 @@ func (c *Check) run() State {
 	lastCommand := c.Audit
 	hasAuditConfig := c.AuditConfig != ""
 
-	state, finalOutput, retErrmsgs := performTest(c.Audit, c.Tests)
+	state, finalOutput, retErrmsgs := performTest(c.Audit, c.Tests, c.IsMultiple)
 	if len(state) > 0 {
 		c.Reason = retErrmsgs
 		c.State = state
@@ -161,7 +162,7 @@ func (c *Check) run() State {
 			currentTests.TestItems[i] = nti
 		}
 
-		state, finalOutput, retErrmsgs = performTest(c.AuditConfig, currentTests)
+		state, finalOutput, retErrmsgs = performTest(c.AuditConfig, currentTests, c.IsMultiple)
 		if len(state) > 0 {
 			c.Reason = retErrmsgs
 			c.State = state
@@ -195,7 +196,7 @@ func (c *Check) run() State {
 	return c.State
 }
 
-func performTest(audit string, tests *tests) (State, *testOutput, string) {
+func performTest(audit string, tests *tests, isMultipleOutput bool) (State, *testOutput, string) {
 	if len(strings.TrimSpace(audit)) == 0 {
 		return "", failTestItem("missing command"), "missing audit command"
 	}
@@ -203,7 +204,7 @@ func performTest(audit string, tests *tests) (State, *testOutput, string) {
 	var out bytes.Buffer
 	errmsgs := runAudit(audit, &out)
 
-	finalOutput := tests.execute(out.String())
+	finalOutput := tests.execute(out.String(), isMultipleOutput)
 	if finalOutput == nil {
 		errmsgs += fmt.Sprintf("Final output is <<EMPTY>>. Failed to run: %s\n", audit)
 	}

check/check_test.go

@@ -103,6 +103,18 @@ func TestCheckAuditConfig(t *testing.T) {
 			controls.Groups[1].Checks[8],
 			"FAIL",
 		},
+		{
+			controls.Groups[1].Checks[9],
+			"PASS",
+		},
+		{
+			controls.Groups[1].Checks[10],
+			"FAIL",
+		},
+		{
+			controls.Groups[1].Checks[11],
+			"FAIL",
+		},
 	}
 
 	for _, c := range cases {

check/controls_test.go

@@ -184,7 +184,7 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
 			},
 			expect: []byte(`<testsuite name="" tests="0" failures="0" errors="0" time="0">
     <testcase name="check1id check1text" classname="" time="0">
-        <system-out>{&#34;test_number&#34;:&#34;check1id&#34;,&#34;test_desc&#34;:&#34;check1text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;PASS&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+        <system-out>{&#34;test_number&#34;:&#34;check1id&#34;,&#34;test_desc&#34;:&#34;check1text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;PASS&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
 </testsuite>`),
 		}, {
@@ -207,7 +207,7 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
 			},
 			expect: []byte(`<testsuite name="" tests="402" failures="99" errors="0" time="0">
     <testcase name="check1id check1text" classname="" time="0">
-        <system-out>{&#34;test_number&#34;:&#34;check1id&#34;,&#34;test_desc&#34;:&#34;check1text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;PASS&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+        <system-out>{&#34;test_number&#34;:&#34;check1id&#34;,&#34;test_desc&#34;:&#34;check1text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;PASS&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
 </testsuite>`),
 		}, {
@@ -227,19 +227,19 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
 			},
 			expect: []byte(`<testsuite name="" tests="0" failures="0" errors="0" time="0">
     <testcase name="check1id check1text" classname="" time="0">
-        <system-out>{&#34;test_number&#34;:&#34;check1id&#34;,&#34;test_desc&#34;:&#34;check1text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;PASS&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+        <system-out>{&#34;test_number&#34;:&#34;check1id&#34;,&#34;test_desc&#34;:&#34;check1text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;PASS&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
     <testcase name="check2id check2text" classname="" time="0">
         <skipped></skipped>
-        <system-out>{&#34;test_number&#34;:&#34;check2id&#34;,&#34;test_desc&#34;:&#34;check2text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;INFO&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+        <system-out>{&#34;test_number&#34;:&#34;check2id&#34;,&#34;test_desc&#34;:&#34;check2text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;INFO&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
     <testcase name="check3id check3text" classname="" time="0">
         <skipped></skipped>
-        <system-out>{&#34;test_number&#34;:&#34;check3id&#34;,&#34;test_desc&#34;:&#34;check3text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;WARN&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+        <system-out>{&#34;test_number&#34;:&#34;check3id&#34;,&#34;test_desc&#34;:&#34;check3text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;WARN&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
     <testcase name="check4id check4text" classname="" time="0">
         <failure type=""></failure>
-        <system-out>{&#34;test_number&#34;:&#34;check4id&#34;,&#34;test_desc&#34;:&#34;check4text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;FAIL&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
+        <system-out>{&#34;test_number&#34;:&#34;check4id&#34;,&#34;test_desc&#34;:&#34;check4text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;FAIL&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
     </testcase>
 </testsuite>`),
 		},

check/data

@@ -440,3 +440,39 @@ groups:
               value: "correct"
             set: true
       scored: true
+    - id: 9
+      text: "test use_multiple_values is correct -> pass"
+      audit: "printf 'permissions=600\npermissions=600\npermissions=600'"
+      use_multiple_values: true
+      tests:
+        test_items:
+          - flag: "permissions"
+            compare:
+              op: bitmask
+              value: "600"
+            set: true
+      scored: true
+    - id: 10
+      text: "test use_multiple_values is wrong -> fail"
+      audit: "printf 'permissions=600\npermissions=600\npermissions=644'"
+      use_multiple_values: true
+      tests:
+        test_items:
+          - flag: "permissions"
+            compare:
+              op: bitmask
+              value: "600"
+            set: true
+      scored: true
+    - id: 11
+      text: "test use_multiple_values include empty value -> fail"
+      audit: "printf 'permissions=600\n\npermissions=600'"
+      use_multiple_values: true
+      tests:
+        test_items:
+          - flag: "permissions"
+            compare:
+              op: bitmask
+              value: "600"
+            set: true
+      scored: true

check/test.go

@@ -67,7 +67,28 @@ func failTestItem(s string) *testOutput {
 	return &testOutput{testResult: false, actualResult: s}
 }
 
-func (t *testItem) execute(s string) *testOutput {
+func (t *testItem) execute(s string, isMultipleOutput bool) *testOutput {
+	result := &testOutput{}
+	s = strings.TrimRight(s, " \n")
+
+	// If the test has output that should be evaluated for each row
+	if isMultipleOutput {
+		output := strings.Split(s, "\n")
+		for _, op := range output {
+			result = t.evaluate(op)
+			// If the test failed for the current row, no need to keep testing for this output
+			if !result.testResult {
+				break
+			}
+		}
+	} else {
+		result = t.evaluate(s)
+	}
+
+	return result
+}
+
+func (t *testItem) evaluate(s string) *testOutput {
 	result := &testOutput{}
 	var match bool
 	var flagVal string
@@ -310,7 +331,7 @@ type tests struct {
 	BinOp     binOp       `yaml:"bin_op"`
 }
 
-func (ts *tests) execute(s string) *testOutput {
+func (ts *tests) execute(s string, isMultipleOutput bool) *testOutput {
 	finalOutput := &testOutput{}
 
 	// If no tests are defined return with empty finalOutput.
@@ -327,7 +348,7 @@ func (ts *tests) execute(s string) *testOutput {
 	expectedResultArr := make([]string, len(res))
 
 	for i, t := range ts.TestItems {
-		res[i] = *(t.execute(s))
+		res[i] = *(t.execute(s, isMultipleOutput))
 		expectedResultArr[i] = res[i].ExpectedResult
 	}
 

check/test_test.go

@@ -183,7 +183,7 @@ func TestTestExecute(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		res := c.Tests.execute(c.str).testResult
+		res := c.Tests.execute(c.str, c.IsMultiple).testResult
 		if !res {
 			t.Errorf("%s, expected:%v, got:%v\n", c.Text, true, res)
 		}
@@ -219,7 +219,7 @@ func TestTestExecuteExceptions(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		res := c.Tests.execute(c.str).testResult
+		res := c.Tests.execute(c.str, c.IsMultiple).testResult
 		if res {
 			t.Errorf("%s, expected:%v, got:%v\n", c.Text, false, res)
 		}