Add a total summary and always show all tests. (#759)

Whether the total summary is shown can be specified with an option.

Fixes #528

Signed-off-by: Christian Zunker <christian.zunker@codecentric.cloud>

check/controls.go

@@ -40,6 +40,11 @@ const (
 	TYPE = "Software and Configuration Checks/Industry and Regulatory Standards/CIS Kubernetes Benchmark"
 )
 
+type OverallControls struct {
+	Controls []*Controls
+	Totals   Summary
+}
+
 // Controls holds all controls to check for master nodes.
 type Controls struct {
 	ID      string   `yaml:"id" json:"id"`

cmd/common.go

@@ -148,7 +148,7 @@ func generateDefaultEnvAudit(controls *check.Controls, binSubs []string){
 }
 
 func parseSkipIds(skipIds string) map[string]bool {
-	var skipIdMap =  make(map[string]bool, 0)
+	var skipIdMap = make(map[string]bool, 0)
 	if skipIds != "" {
 		for _, id := range strings.Split(skipIds, ",") {
 			skipIdMap[strings.Trim(id, " ")] = true
@@ -185,7 +185,7 @@ func prettyPrint(r *check.Controls, summary check.Summary) {
 	// Print remediations.
 	if !noRemediations {
 		if summary.Fail > 0 || summary.Warn > 0 {
-			colors[check.WARN].Printf("== Remediations ==\n")
+			colors[check.WARN].Printf("== Remediations %s ==\n", r.Type)
 			for _, g := range r.Groups {
 				for _, c := range g.Checks {
 					if c.State == check.FAIL {
@@ -207,20 +207,24 @@ func prettyPrint(r *check.Controls, summary check.Summary) {
 
 	// Print summary setting output color to highest severity.
 	if !noSummary {
-		var res check.State
-		if summary.Fail > 0 {
-			res = check.FAIL
-		} else if summary.Warn > 0 {
-			res = check.WARN
-		} else {
-			res = check.PASS
-		}
+		printSummary(summary, string(r.Type))
+	}
+}
 
-		colors[res].Printf("== Summary ==\n")
-		fmt.Printf("%d checks PASS\n%d checks FAIL\n%d checks WARN\n%d checks INFO\n",
-			summary.Pass, summary.Fail, summary.Warn, summary.Info,
-		)
+func printSummary(summary check.Summary, sectionName string) {
+	var res check.State
+	if summary.Fail > 0 {
+		res = check.FAIL
+	} else if summary.Warn > 0 {
+		res = check.WARN
+	} else {
+		res = check.PASS
 	}
+
+	colors[res].Printf("== Summary %s ==\n", sectionName)
+	fmt.Printf("%d checks PASS\n%d checks FAIL\n%d checks WARN\n%d checks INFO\n\n",
+		summary.Pass, summary.Fail, summary.Warn, summary.Info,
+	)
 }
 
 // loadConfig finds the correct config dir based on the kubernetes version,
@@ -407,7 +411,16 @@ func writeOutput(controlsCollection []*check.Controls) {
 }
 
 func writeJSONOutput(controlsCollection []*check.Controls) {
-	out, err := json.Marshal(controlsCollection)
+	var out []byte
+	var err error
+	if !noTotals {
+		var totals check.OverallControls
+		totals.Controls = controlsCollection
+		totals.Totals = getSummaryTotals(controlsCollection)
+		out, err = json.Marshal(totals)
+	} else {
+		out, err = json.Marshal(controlsCollection)
+	}
 	if err != nil {
 		exitWithError(fmt.Errorf("failed to output in JSON format: %v", err))
 	}
@@ -451,6 +464,21 @@ func writeStdoutOutput(controlsCollection []*check.Controls) {
 		summary := controls.Summary
 		prettyPrint(controls, summary)
 	}
+	if !noTotals {
+		printSummary(getSummaryTotals(controlsCollection), "total")
+	}
+}
+
+func getSummaryTotals(controlsCollection []*check.Controls) check.Summary {
+	var totalSummary check.Summary
+	for _, controls := range controlsCollection {
+		summary := controls.Summary
+		totalSummary.Fail = totalSummary.Fail + summary.Fail
+		totalSummary.Warn = totalSummary.Warn + summary.Warn
+		totalSummary.Pass = totalSummary.Pass + summary.Pass
+		totalSummary.Info = totalSummary.Info + summary.Info
+	}
+	return totalSummary
 }
 
 func printRawOutput(output string) {

cmd/common_test.go

@@ -30,6 +30,15 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+type JsonOutputFormat struct {
+	Controls     []*check.Controls `json:"Controls"`
+	TotalSummary map[string]int    `json:"Totals"`
+}
+
+type JsonOutputFormatNoTotals struct {
+	Controls []*check.Controls `json:"Controls"`
+}
+
 func TestParseSkipIds(t *testing.T) {
 	skipMap := parseSkipIds("4.12,4.13,5")
 	_, fourTwelveExists := skipMap["4.12"]
@@ -527,13 +536,45 @@ func TestWriteResultToJsonFile(t *testing.T) {
 	}
 	writeOutput(controlsCollection)
 
+	var expect JsonOutputFormat
+	var result JsonOutputFormat
+	result, err = parseResultJsonFile(outputFile)
+	if err != nil {
+		t.Error(err)
+	}
+	expect, err = parseResultJsonFile("./testdata/result.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	assert.Equal(t, expect, result)
+}
+
+func TestWriteResultNoTotalsToJsonFile(t *testing.T) {
+	defer func() {
+		controlsCollection = []*check.Controls{}
+		jsonFmt = false
+		outputFile = ""
+	}()
+	var err error
+	jsonFmt = true
+	outputFile = path.Join(os.TempDir(), fmt.Sprintf("%d", time.Now().UnixNano()))
+
+	noTotals = true
+
+	controlsCollection, err = parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+	writeOutput(controlsCollection)
+
 	var expect []*check.Controls
 	var result []*check.Controls
-	result, err = parseControlsJsonFile(outputFile)
+	result, err = parseResultNoTotalsJsonFile(outputFile)
 	if err != nil {
 		t.Error(err)
 	}
-	expect, err = parseControlsJsonFile("./testdata/result.json")
+	expect, err = parseResultNoTotalsJsonFile("./testdata/result_no_totals.json")
 	if err != nil {
 		t.Error(err)
 	}
@@ -541,7 +582,7 @@ func TestWriteResultToJsonFile(t *testing.T) {
 	assert.Equal(t, expect, result)
 }
 
-func TestExitCodeSelection(t *testing.T){
+func TestExitCodeSelection(t *testing.T) {
 	exitCode = 10
 	controlsCollectionAllPassed, errPassed := parseControlsJsonFile("./testdata/passedControlsCollection.json")
 	if errPassed != nil {
@@ -587,13 +628,121 @@ groups:
 	controls, err := check.NewControls(check.MASTER, input)
 	assert.NoError(t, err)
 
-	binSubs := []string {"TestBinPath"}
+	binSubs := []string{"TestBinPath"}
 	generateDefaultEnvAudit(controls, binSubs)
 
 	expectedAuditEnv := fmt.Sprintf("cat \"/proc/$(/bin/ps -C %s -o pid= | tr -d ' ')/environ\" | tr '\\0' '\\n'", binSubs[0])
 	assert.Equal(t, expectedAuditEnv, controls.Groups[1].Checks[0].AuditEnv)
 }
 
+func TestGetSummaryTotals(t *testing.T) {
+	controlsCollection, err := parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	resultTotals := getSummaryTotals(controlsCollection)
+	assert.Equal(t, 12, resultTotals.Fail)
+	assert.Equal(t, 14, resultTotals.Warn)
+	assert.Equal(t, 0, resultTotals.Info)
+	assert.Equal(t, 49, resultTotals.Pass)
+}
+
+func TestPrintSummary(t *testing.T) {
+	controlsCollection, err := parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	resultTotals := getSummaryTotals(controlsCollection)
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	printSummary(resultTotals, "totals")
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = rescueStdout
+
+	assert.Contains(t, string(out), "49 checks PASS\n12 checks FAIL\n14 checks WARN\n0 checks INFO\n\n")
+}
+
+func TestPrettyPrintNoSummary(t *testing.T) {
+	controlsCollection, err := parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	resultTotals := getSummaryTotals(controlsCollection)
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	noSummary = true
+	prettyPrint(controlsCollection[0], resultTotals)
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = rescueStdout
+
+	assert.NotContains(t, string(out), "49 checks PASS")
+}
+
+func TestPrettyPrintSummary(t *testing.T) {
+	controlsCollection, err := parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	resultTotals := getSummaryTotals(controlsCollection)
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	noSummary = false
+	prettyPrint(controlsCollection[0], resultTotals)
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = rescueStdout
+
+	assert.Contains(t, string(out), "49 checks PASS")
+}
+
+func TestWriteStdoutOutputNoTotal(t *testing.T) {
+	controlsCollection, err := parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	noTotals = true
+	writeStdoutOutput(controlsCollection)
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = rescueStdout
+
+	assert.NotContains(t, string(out), "49 checks PASS")
+}
+
+func TestWriteStdoutOutputTotal(t *testing.T) {
+	controlsCollection, err := parseControlsJsonFile("./testdata/controlsCollection.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	rescueStdout := os.Stdout
+
+	r, w, _ := os.Pipe()
+
+	os.Stdout = w
+	noTotals = false
+	writeStdoutOutput(controlsCollection)
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+
+	os.Stdout = rescueStdout
+
+	assert.Contains(t, string(out), "49 checks PASS")
+}
+
 func parseControlsJsonFile(filepath string) ([]*check.Controls, error) {
 	var result []*check.Controls
 
@@ -609,6 +758,36 @@ func parseControlsJsonFile(filepath string) ([]*check.Controls, error) {
 	return result, nil
 }
 
+func parseResultJsonFile(filepath string) (JsonOutputFormat, error) {
+	var result JsonOutputFormat
+
+	d, err := ioutil.ReadFile(filepath)
+	if err != nil {
+		return result, err
+	}
+	err = json.Unmarshal(d, &result)
+	if err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func parseResultNoTotalsJsonFile(filepath string) ([]*check.Controls, error) {
+	var result []*check.Controls
+
+	d, err := ioutil.ReadFile(filepath)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(d, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
 func loadConfigForTest() (*viper.Viper, error) {
 	viperWithData := viper.New()
 	viperWithData.SetConfigFile("../cfg/config.yaml")

cmd/root.go

@@ -54,6 +54,7 @@ var (
 	noSummary           bool
 	noRemediations      bool
 	skipIds             string
+	noTotals            bool
 	filterOpts          FilterOpts
 	includeTestOutput   bool
 	outputFile          string
@@ -87,6 +88,8 @@ var RootCmd = &cobra.Command{
 				glog.V(1).Info("== Running control plane checks ==")
 				runChecks(check.CONTROLPLANE, loadConfig(check.CONTROLPLANE, bv))
 			}
+		} else {
+			glog.V(1).Info("== Skipping master checks ==")
 		}
 
 		// Etcd is only valid for CIS 1.5 and later,
@@ -98,6 +101,8 @@ var RootCmd = &cobra.Command{
 		if valid && isEtcd() {
 			glog.V(1).Info("== Running etcd checks ==")
 			runChecks(check.ETCD, loadConfig(check.ETCD, bv))
+		} else {
+			glog.V(1).Info("== Skipping etcd checks ==")
 		}
 
 		glog.V(1).Info("== Running node checks ==")
@@ -112,6 +117,8 @@ var RootCmd = &cobra.Command{
 		if valid {
 			glog.V(1).Info("== Running policies checks ==")
 			runChecks(check.POLICIES, loadConfig(check.POLICIES, bv))
+		} else {
+			glog.V(1).Info("== Skipping policies checks ==")
 		}
 
 		// Managedservices is only valid for GKE 1.0 and later,
@@ -123,6 +130,8 @@ var RootCmd = &cobra.Command{
 		if valid {
 			glog.V(1).Info("== Running managed services checks ==")
 			runChecks(check.MANAGEDSERVICES, loadConfig(check.MANAGEDSERVICES, bv))
+		} else {
+			glog.V(1).Info("== Skipping managed services checks ==")
 		}
 
 		writeOutput(controlsCollection)
@@ -154,6 +163,7 @@ func init() {
 	RootCmd.PersistentFlags().BoolVar(&noResults, "noresults", false, "Disable printing of results section")
 	RootCmd.PersistentFlags().BoolVar(&noSummary, "nosummary", false, "Disable printing of summary section")
 	RootCmd.PersistentFlags().BoolVar(&noRemediations, "noremediations", false, "Disable printing of remediations section")
+	RootCmd.PersistentFlags().BoolVar(&noTotals, "nototals", false, "Disable printing of totals for failed, passed, ... checks across all sections")
 	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Prints the results as JSON")
 	RootCmd.PersistentFlags().BoolVar(&junitFmt, "junit", false, "Prints the results as JUnit")
 	RootCmd.PersistentFlags().BoolVar(&pgSQL, "pgsql", false, "Save the results to PostgreSQL")

cmd/testdata/controlsCollection.json

@@ -111,4 +111,4 @@
     "total_warn": 11,
     "total_info": 0
   }
-]
+]

cmd/testdata/result.json

@@ -1,114 +1,122 @@
-[
-  {
-    "id": "1",
-    "version": "1.5",
-    "text": "Master Node Security Configuration",
-    "node_type": "master",
-    "tests": [
-      {
-        "section": "1.1",
-        "pass": 15,
-        "fail": 1,
-        "warn": 5,
-        "info": 0,
-        "desc": "Master Node Configuration Files",
-        "results": [
-          {
-            "test_number": "1.1.1",
-            "test_desc": "Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Scored)",
-            "audit": "/bin/sh -c 'if test -e /etc/kubernetes/manifests/kube-apiserver.yaml; then stat -c permissions=%a /etc/kubernetes/manifests/kube-apiserver.yaml; fi'",
-            "AuditConfig": "",
-            "type": "",
-            "remediation": "Run the below command (based on the file location on your system) on the\nmaster node.\nFor example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml\n",
-            "test_info": [
-              "Run the below command (based on the file location on your system) on the\nmaster node.\nFor example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml\n"
-            ],
-            "status": "PASS",
-            "actual_value": "permissions=600\n",
-            "scored": true,
-            "expected_result": "bitmask '600' AND '644'"
-          }
-        ]
-      }
-    ],
-    "total_pass": 42,
+{
+  "Controls": [
+    {
+      "id": "1",
+      "version": "1.5",
+      "text": "Master Node Security Configuration",
+      "node_type": "master",
+      "tests": [
+        {
+          "section": "1.1",
+          "pass": 15,
+          "fail": 1,
+          "warn": 5,
+          "info": 0,
+          "desc": "Master Node Configuration Files",
+          "results": [
+            {
+              "test_number": "1.1.1",
+              "test_desc": "Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Scored)",
+              "audit": "/bin/sh -c 'if test -e /etc/kubernetes/manifests/kube-apiserver.yaml; then stat -c permissions=%a /etc/kubernetes/manifests/kube-apiserver.yaml; fi'",
+              "AuditConfig": "",
+              "type": "",
+              "remediation": "Run the below command (based on the file location on your system) on the\nmaster node.\nFor example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml\n",
+              "test_info": [
+                "Run the below command (based on the file location on your system) on the\nmaster node.\nFor example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml\n"
+              ],
+              "status": "PASS",
+              "actual_value": "permissions=600\n",
+              "scored": true,
+              "expected_result": "bitmask '600' AND '644'"
+            }
+          ]
+        }
+      ],
+      "total_pass": 42,
+      "total_fail": 12,
+      "total_warn": 11,
+      "total_info": 0
+    },
+    {
+      "id": "2",
+      "version": "1.15",
+      "text": "Etcd Node Configuration",
+      "node_type": "etcd",
+      "tests": [
+        {
+          "section": "2",
+          "pass": 7,
+          "fail": 0,
+          "warn": 0,
+          "info": 0,
+          "desc": "Etcd Node Configuration Files",
+          "results": [
+            {
+              "test_number": "2.1",
+              "test_desc": "Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored)",
+              "audit": "/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep",
+              "AuditConfig": "",
+              "type": "",
+              "remediation": "Follow the etcd service documentation and configure TLS encryption.\nThen, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml\non the master node and set the below parameters.\n--cert-file=</path/to/ca-file>\n--key-file=</path/to/key-file>\n",
+              "test_info": [
+                "Follow the etcd service documentation and configure TLS encryption.\nThen, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml\non the master node and set the below parameters.\n--cert-file=</path/to/ca-file>\n--key-file=</path/to/key-file>\n"
+              ],
+              "status": "PASS",
+              "actual_value": "root      3277  3218  3 Apr19 ?        03:57:52 etcd --advertise-client-urls=https://192.168.64.4:2379 --cert-file=/var/lib/minikube/certs/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/minikube/etcd --initial-advertise-peer-urls=https://192.168.64.4:2380 --initial-cluster=minikube=https://192.168.64.4:2380 --key-file=/var/lib/minikube/certs/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://192.168.64.4:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://192.168.64.4:2380 --name=minikube --peer-cert-file=/var/lib/minikube/certs/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/var/lib/minikube/certs/etcd/peer.key --peer-trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt\nroot      4624  4605  8 Apr21 ?        04:55:10 kube-apiserver --advertise-address=192.168.64.4 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/var/lib/minikube/certs/ca.crt --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,PodSecurityPolicy --enable-bootstrap-token-auth=true --etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt --etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt --etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt --kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt --proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=8443 --service-account-key-file=/var/lib/minikube/certs/sa.pub --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/var/lib/minikube/certs/apiserver.crt --tls-private-key-file=/var/lib/minikube/certs/apiserver.key\n",
+              "scored": true,
+              "expected_result": "'--cert-file' is present AND '--key-file' is present"
+            }
+          ]
+        }
+      ],
+      "total_pass": 7,
+      "total_fail": 0,
+      "total_warn": 0,
+      "total_info": 0
+    },
+    {
+      "id": "3",
+      "version": "1.5",
+      "text": "Control Plane Configuration",
+      "node_type": "controlplane",
+      "tests": [
+        {
+          "section": "3.1",
+          "pass": 0,
+          "fail": 0,
+          "warn": 1,
+          "info": 0,
+          "desc": "Authentication and Authorization",
+          "results": [
+            {
+              "test_number": "3.1.1",
+              "test_desc": "Client certificate authentication should not be used for users (Not Scored)",
+              "audit": "",
+              "AuditConfig": "",
+              "type": "manual",
+              "remediation": "Alternative mechanisms provided by Kubernetes such as the use of OIDC should be\nimplemented in place of client certificates.\n",
+              "test_info": [
+                "Alternative mechanisms provided by Kubernetes such as the use of OIDC should be\nimplemented in place of client certificates.\n"
+              ],
+              "status": "WARN",
+              "actual_value": "",
+              "scored": false,
+              "expected_result": "",
+              "reason": "Test marked as a manual test"
+            }
+          ]
+        }
+      ],
+      "total_pass": 0,
+      "total_fail": 0,
+      "total_warn": 3,
+      "total_info": 0
+    }
+  ],
+  "Totals": {
+    "total_pass": 49,
     "total_fail": 12,
-    "total_warn": 11,
-    "total_info": 0
-  },
-  {
-    "id": "2",
-    "version": "1.15",
-    "text": "Etcd Node Configuration",
-    "node_type": "etcd",
-    "tests": [
-      {
-        "section": "2",
-        "pass": 7,
-        "fail": 0,
-        "warn": 0,
-        "info": 0,
-        "desc": "Etcd Node Configuration Files",
-        "results": [
-          {
-            "test_number": "2.1",
-            "test_desc": "Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored)",
-            "audit": "/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep",
-            "AuditConfig": "",
-            "type": "",
-            "remediation": "Follow the etcd service documentation and configure TLS encryption.\nThen, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml\non the master node and set the below parameters.\n--cert-file=</path/to/ca-file>\n--key-file=</path/to/key-file>\n",
-            "test_info": [
-              "Follow the etcd service documentation and configure TLS encryption.\nThen, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml\non the master node and set the below parameters.\n--cert-file=</path/to/ca-file>\n--key-file=</path/to/key-file>\n"
-            ],
-            "status": "PASS",
-            "actual_value": "root      3277  3218  3 Apr19 ?        03:57:52 etcd --advertise-client-urls=https://192.168.64.4:2379 --cert-file=/var/lib/minikube/certs/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/minikube/etcd --initial-advertise-peer-urls=https://192.168.64.4:2380 --initial-cluster=minikube=https://192.168.64.4:2380 --key-file=/var/lib/minikube/certs/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://192.168.64.4:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://192.168.64.4:2380 --name=minikube --peer-cert-file=/var/lib/minikube/certs/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/var/lib/minikube/certs/etcd/peer.key --peer-trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt\nroot      4624  4605  8 Apr21 ?        04:55:10 kube-apiserver --advertise-address=192.168.64.4 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/var/lib/minikube/certs/ca.crt --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,PodSecurityPolicy --enable-bootstrap-token-auth=true --etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt --etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt --etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt --kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt --proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=8443 --service-account-key-file=/var/lib/minikube/certs/sa.pub --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/var/lib/minikube/certs/apiserver.crt --tls-private-key-file=/var/lib/minikube/certs/apiserver.key\n",
-            "scored": true,
-            "expected_result": "'--cert-file' is present AND '--key-file' is present"
-          }
-        ]
-      }
-    ],
-    "total_pass": 7,
-    "total_fail": 0,
-    "total_warn": 0,
-    "total_info": 0
-  },
-  {
-    "id": "3",
-    "version": "1.5",
-    "text": "Control Plane Configuration",
-    "node_type": "controlplane",
-    "tests": [
-      {
-        "section": "3.1",
-        "pass": 0,
-        "fail": 0,
-        "warn": 1,
-        "info": 0,
-        "desc": "Authentication and Authorization",
-        "results": [
-          {
-            "test_number": "3.1.1",
-            "test_desc": "Client certificate authentication should not be used for users (Not Scored)",
-            "audit": "",
-            "AuditConfig": "",
-            "type": "manual",
-            "remediation": "Alternative mechanisms provided by Kubernetes such as the use of OIDC should be\nimplemented in place of client certificates.\n",
-            "test_info": [
-              "Alternative mechanisms provided by Kubernetes such as the use of OIDC should be\nimplemented in place of client certificates.\n"
-            ],
-            "status": "WARN",
-            "actual_value": "",
-            "scored": false,
-            "expected_result": "",
-            "reason": "Test marked as a manual test"
-          }
-        ]
-      }
-    ],
-    "total_pass": 0,
-    "total_fail": 0,
-    "total_warn": 3,
+    "total_warn": 14,
     "total_info": 0
   }
-]
+}

cmd/testdata/result_no_totals.json

@@ -0,0 +1,114 @@
+[
+  {
+    "id": "1",
+    "version": "1.5",
+    "text": "Master Node Security Configuration",
+    "node_type": "master",
+    "tests": [
+      {
+        "section": "1.1",
+        "pass": 15,
+        "fail": 1,
+        "warn": 5,
+        "info": 0,
+        "desc": "Master Node Configuration Files",
+        "results": [
+          {
+            "test_number": "1.1.1",
+            "test_desc": "Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Scored)",
+            "audit": "/bin/sh -c 'if test -e /etc/kubernetes/manifests/kube-apiserver.yaml; then stat -c permissions=%a /etc/kubernetes/manifests/kube-apiserver.yaml; fi'",
+            "AuditConfig": "",
+            "type": "",
+            "remediation": "Run the below command (based on the file location on your system) on the\nmaster node.\nFor example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml\n",
+            "test_info": [
+              "Run the below command (based on the file location on your system) on the\nmaster node.\nFor example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml\n"
+            ],
+            "status": "PASS",
+            "actual_value": "permissions=600\n",
+            "scored": true,
+            "expected_result": "bitmask '600' AND '644'"
+          }
+        ]
+      }
+    ],
+    "total_pass": 42,
+    "total_fail": 12,
+    "total_warn": 11,
+    "total_info": 0
+  },
+  {
+    "id": "2",
+    "version": "1.15",
+    "text": "Etcd Node Configuration",
+    "node_type": "etcd",
+    "tests": [
+      {
+        "section": "2",
+        "pass": 7,
+        "fail": 0,
+        "warn": 0,
+        "info": 0,
+        "desc": "Etcd Node Configuration Files",
+        "results": [
+          {
+            "test_number": "2.1",
+            "test_desc": "Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored)",
+            "audit": "/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep",
+            "AuditConfig": "",
+            "type": "",
+            "remediation": "Follow the etcd service documentation and configure TLS encryption.\nThen, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml\non the master node and set the below parameters.\n--cert-file=</path/to/ca-file>\n--key-file=</path/to/key-file>\n",
+            "test_info": [
+              "Follow the etcd service documentation and configure TLS encryption.\nThen, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml\non the master node and set the below parameters.\n--cert-file=</path/to/ca-file>\n--key-file=</path/to/key-file>\n"
+            ],
+            "status": "PASS",
+            "actual_value": "root      3277  3218  3 Apr19 ?        03:57:52 etcd --advertise-client-urls=https://192.168.64.4:2379 --cert-file=/var/lib/minikube/certs/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/minikube/etcd --initial-advertise-peer-urls=https://192.168.64.4:2380 --initial-cluster=minikube=https://192.168.64.4:2380 --key-file=/var/lib/minikube/certs/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://192.168.64.4:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://192.168.64.4:2380 --name=minikube --peer-cert-file=/var/lib/minikube/certs/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/var/lib/minikube/certs/etcd/peer.key --peer-trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt\nroot      4624  4605  8 Apr21 ?        04:55:10 kube-apiserver --advertise-address=192.168.64.4 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/var/lib/minikube/certs/ca.crt --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,PodSecurityPolicy --enable-bootstrap-token-auth=true --etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt --etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt --etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt --kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt --proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=8443 --service-account-key-file=/var/lib/minikube/certs/sa.pub --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/var/lib/minikube/certs/apiserver.crt --tls-private-key-file=/var/lib/minikube/certs/apiserver.key\n",
+            "scored": true,
+            "expected_result": "'--cert-file' is present AND '--key-file' is present"
+          }
+        ]
+      }
+    ],
+    "total_pass": 7,
+    "total_fail": 0,
+    "total_warn": 0,
+    "total_info": 0
+  },
+  {
+    "id": "3",
+    "version": "1.5",
+    "text": "Control Plane Configuration",
+    "node_type": "controlplane",
+    "tests": [
+      {
+        "section": "3.1",
+        "pass": 0,
+        "fail": 0,
+        "warn": 1,
+        "info": 0,
+        "desc": "Authentication and Authorization",
+        "results": [
+          {
+            "test_number": "3.1.1",
+            "test_desc": "Client certificate authentication should not be used for users (Not Scored)",
+            "audit": "",
+            "AuditConfig": "",
+            "type": "manual",
+            "remediation": "Alternative mechanisms provided by Kubernetes such as the use of OIDC should be\nimplemented in place of client certificates.\n",
+            "test_info": [
+              "Alternative mechanisms provided by Kubernetes such as the use of OIDC should be\nimplemented in place of client certificates.\n"
+            ],
+            "status": "WARN",
+            "actual_value": "",
+            "scored": false,
+            "expected_result": "",
+            "reason": "Test marked as a manual test"
+          }
+        ]
+      }
+    ],
+    "total_pass": 0,
+    "total_fail": 0,
+    "total_warn": 3,
+    "total_info": 0
+  }
+]

cmd/util.go

@@ -93,7 +93,7 @@ func getBinaries(v *viper.Viper, nodetype check.NodeType) (map[string]string, er
 		if len(bins) > 0 {
 			bin, err := findExecutable(bins)
 			if err != nil && !optional {
-				glog.Warning(buildComponentMissingErrorMessage(nodetype, component, bins))
+				glog.V(1).Info(buildComponentMissingErrorMessage(nodetype, component, bins))
 				return nil, fmt.Errorf("unable to detect running programs for component %q", component)
 			}
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d // indirect
 	github.com/jinzhu/now v1.0.1 // indirect
 	github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd // indirect
+	github.com/magiconair/properties v1.8.0
 	github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597 // indirect
 	github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect

integration/testdata/cis-1.5/job-master.data

@@ -69,7 +69,7 @@
 [FAIL] 1.4.1 Ensure that the --profiling argument is set to false (Scored)
 [PASS] 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Scored)
 
-== Remediations ==
+== Remediations master ==
 1.1.9 Run the below command (based on the file location on your system) on the master node.
 For example,
 chmod 644 <path/to/cni/files>
@@ -165,8 +165,14 @@ on the master node and set the below parameter.
 --profiling=false
 
 
-== Summary ==
+== Summary master ==
 45 checks PASS
 10 checks FAIL
 10 checks WARN
 0 checks INFO
+
+== Summary total ==
+45 checks PASS
+10 checks FAIL
+10 checks WARN
+0 checks INFO

integration/testdata/cis-1.5/job-node.data

@@ -25,7 +25,7 @@
 [PASS] 4.2.12 Ensure that the RotateKubeletServerCertificate argument is set to true (Scored)
 [PASS] 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Not Scored)
 
-== Remediations ==
+== Remediations node ==
 4.2.6 If using a Kubelet config file, edit the file to set protectKernelDefaults: true.
 If using command line arguments, edit the kubelet service file
 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and
@@ -56,8 +56,14 @@ systemctl daemon-reload
 systemctl restart kubelet.service
 
 
-== Summary ==
+== Summary node ==
 20 checks PASS
 2 checks FAIL
 1 checks WARN
 0 checks INFO
+
+== Summary total ==
+20 checks PASS
+2 checks FAIL
+1 checks WARN
+0 checks INFO

integration/testdata/cis-1.5/job.data

@@ -69,7 +69,7 @@
 [FAIL] 1.4.1 Ensure that the --profiling argument is set to false (Scored)
 [PASS] 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Scored)
 
-== Remediations ==
+== Remediations master ==
 1.1.9 Run the below command (based on the file location on your system) on the master node.
 For example,
 chmod 644 <path/to/cni/files>
@@ -165,11 +165,12 @@ on the master node and set the below parameter.
 --profiling=false
 
 
-== Summary ==
+== Summary master ==
 45 checks PASS
 10 checks FAIL
 10 checks WARN
 0 checks INFO
+
 [INFO] 2 Etcd Node Configuration
 [INFO] 2 Etcd Node Configuration Files
 [PASS] 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored)
@@ -180,11 +181,12 @@ on the master node and set the below parameter.
 [PASS] 2.6 Ensure that the --peer-auto-tls argument is not set to true (Scored)
 [PASS] 2.7 Ensure that a unique Certificate Authority is used for etcd (Not Scored)
 
-== Summary ==
+== Summary etcd ==
 7 checks PASS
 0 checks FAIL
 0 checks WARN
 0 checks INFO
+
 [INFO] 3 Control Plane Configuration
 [INFO] 3.1 Authentication and Authorization
 [WARN] 3.1.1 Client certificate authentication should not be used for users (Not Scored)
@@ -192,7 +194,7 @@ on the master node and set the below parameter.
 [FAIL] 3.2.1 Ensure that a minimal audit policy is created (Scored)
 [WARN] 3.2.2 Ensure that the audit policy covers key security concerns (Not Scored)
 
-== Remediations ==
+== Remediations controlplane ==
 3.1.1 Alternative mechanisms provided by Kubernetes such as the use of OIDC should be
 implemented in place of client certificates.
 
@@ -202,11 +204,12 @@ implemented in place of client certificates.
 minimum.
 
 
-== Summary ==
+== Summary controlplane ==
 0 checks PASS
 1 checks FAIL
 2 checks WARN
 0 checks INFO
+
 [INFO] 4 Worker Node Security Configuration
 [INFO] 4.1 Worker Node Configuration Files
 [PASS] 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Scored)
@@ -234,7 +237,7 @@ minimum.
 [PASS] 4.2.12 Ensure that the RotateKubeletServerCertificate argument is set to true (Scored)
 [PASS] 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Not Scored)
 
-== Remediations ==
+== Remediations node ==
 4.2.6 If using a Kubelet config file, edit the file to set protectKernelDefaults: true.
 If using command line arguments, edit the kubelet service file
 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and
@@ -265,11 +268,12 @@ systemctl daemon-reload
 systemctl restart kubelet.service
 
 
-== Summary ==
+== Summary node ==
 20 checks PASS
 2 checks FAIL
 1 checks WARN
 0 checks INFO
+
 [INFO] 5 Kubernetes Policies
 [INFO] 5.1 RBAC and Service Accounts
 [WARN] 5.1.1 Ensure that the cluster-admin role is only used where required (Not Scored)
@@ -302,7 +306,7 @@ systemctl restart kubelet.service
 [WARN] 5.7.3 Apply Security Context to Your Pods and Containers (Not Scored)
 [WARN] 5.7.4 The default namespace should not be used (Scored)
 
-== Remediations ==
+== Remediations policies ==
 5.1.1 Identify all clusterrolebindings to the cluster-admin role. Check if they are used and
 if they need this role or if they could use a role with fewer privileges.
 Where possible, first bind users to a lower privileged role and then remove the
@@ -399,8 +403,14 @@ Containers.
 resources and that all new resources are created in a specific namespace.
 
 
-== Summary ==
+== Summary policies ==
 0 checks PASS
 0 checks FAIL
 24 checks WARN
 0 checks INFO
+
+== Summary total ==
+72 checks PASS
+13 checks FAIL
+37 checks WARN
+0 checks INFO

integration/testdata/cis-1.6/job-master.data

@@ -69,7 +69,7 @@
 [FAIL] 1.4.1 Ensure that the --profiling argument is set to false (Automated)
 [PASS] 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated)
 
-== Remediations ==
+== Remediations master ==
 1.1.9 Run the below command (based on the file location on your system) on the master node.
 For example,
 chmod 644 <path/to/cni/files>
@@ -168,8 +168,14 @@ on the master node and set the below parameter.
 --profiling=false
 
 
-== Summary ==
+== Summary master ==
 45 checks PASS
 10 checks FAIL
 10 checks WARN
 0 checks INFO
+
+== Summary total ==
+45 checks PASS
+10 checks FAIL
+10 checks WARN
+0 checks INFO

integration/testdata/cis-1.6/job-node.data

@@ -25,7 +25,7 @@
 [PASS] 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual)
 [PASS] 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual)
 
-== Remediations ==
+== Remediations node ==
 4.2.6 If using a Kubelet config file, edit the file to set protectKernelDefaults: true.
 If using command line arguments, edit the kubelet service file
 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and
@@ -56,8 +56,14 @@ systemctl daemon-reload
 systemctl restart kubelet.service
 
 
-== Summary ==
+== Summary node ==
 20 checks PASS
 1 checks FAIL
 2 checks WARN
 0 checks INFO
+
+== Summary total ==
+20 checks PASS
+1 checks FAIL
+2 checks WARN
+0 checks INFO

integration/testdata/cis-1.6/job.data

@@ -69,7 +69,7 @@
 [FAIL] 1.4.1 Ensure that the --profiling argument is set to false (Automated)
 [PASS] 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated)
 
-== Remediations ==
+== Remediations master ==
 1.1.9 Run the below command (based on the file location on your system) on the master node.
 For example,
 chmod 644 <path/to/cni/files>
@@ -168,11 +168,12 @@ on the master node and set the below parameter.
 --profiling=false
 
 
-== Summary ==
+== Summary master ==
 45 checks PASS
 10 checks FAIL
 10 checks WARN
 0 checks INFO
+
 [INFO] 2 Etcd Node Configuration
 [INFO] 2 Etcd Node Configuration Files
 [PASS] 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated)
@@ -183,11 +184,12 @@ on the master node and set the below parameter.
 [PASS] 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated)
 [PASS] 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual)
 
-== Summary ==
+== Summary etcd ==
 7 checks PASS
 0 checks FAIL
 0 checks WARN
 0 checks INFO
+
 [INFO] 3 Control Plane Configuration
 [INFO] 3.1 Authentication and Authorization
 [WARN] 3.1.1 Client certificate authentication should not be used for users (Manual)
@@ -195,7 +197,7 @@ on the master node and set the below parameter.
 [WARN] 3.2.1 Ensure that a minimal audit policy is created (Manual)
 [WARN] 3.2.2 Ensure that the audit policy covers key security concerns (Manual)
 
-== Remediations ==
+== Remediations controlplane ==
 3.1.1 Alternative mechanisms provided by Kubernetes such as the use of OIDC should be
 implemented in place of client certificates.
 
@@ -205,11 +207,12 @@ implemented in place of client certificates.
 minimum.
 
 
-== Summary ==
+== Summary controlplane ==
 0 checks PASS
 0 checks FAIL
 3 checks WARN
 0 checks INFO
+
 [INFO] 4 Worker Node Security Configuration
 [INFO] 4.1 Worker Node Configuration Files
 [PASS] 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated)
@@ -237,7 +240,7 @@ minimum.
 [PASS] 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual)
 [PASS] 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual)
 
-== Remediations ==
+== Remediations node ==
 4.2.6 If using a Kubelet config file, edit the file to set protectKernelDefaults: true.
 If using command line arguments, edit the kubelet service file
 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and
@@ -268,11 +271,12 @@ systemctl daemon-reload
 systemctl restart kubelet.service
 
 
-== Summary ==
+== Summary node ==
 20 checks PASS
 1 checks FAIL
 2 checks WARN
 0 checks INFO
+
 [INFO] 5 Kubernetes Policies
 [INFO] 5.1 RBAC and Service Accounts
 [WARN] 5.1.1 Ensure that the cluster-admin role is only used where required (Manual)
@@ -305,7 +309,7 @@ systemctl restart kubelet.service
 [WARN] 5.7.3 Apply Security Context to Your Pods and Containers (Manual)
 [WARN] 5.7.4 The default namespace should not be used (Manual)
 
-== Remediations ==
+== Remediations policies ==
 5.1.1 Identify all clusterrolebindings to the cluster-admin role. Check if they are used and
 if they need this role or if they could use a role with fewer privileges.
 Where possible, first bind users to a lower privileged role and then remove the
@@ -402,8 +406,14 @@ Containers.
 resources and that all new resources are created in a specific namespace.
 
 
-== Summary ==
+== Summary policies ==
 0 checks PASS
 0 checks FAIL
 24 checks WARN
 0 checks INFO
+
+== Summary total ==
+72 checks PASS
+11 checks FAIL
+39 checks WARN
+0 checks INFO