Add kubeconfig variable substitution for kubelet and proxy.

There are checks for the kubeconfig for both kubelet and proxy which
the current kube-bench implementation does not check for properly.
kube-bench checks the wrong files.

This PR adds support for variable substitution for all the config file
types are that should be checked in the CIS benchmarks.

This PR also fixes a buggy in CIS 1.3.0 check 2.2.9, which checks for
ownership of the kubelet config file /var/lib/kubelet/config.yaml but
recommends changing ownership of kubelet kubeconfig file
/etc/kubernetes/kubelet.conf as remediation.

cfg/1.11/node.yaml

@@ -320,7 +320,7 @@ groups:
     - id: 2.2.1
       text: "Ensure that the kubelet.conf file permissions are set to 644 or
       more restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %a $kubeletconf; fi'"
+      audit: "/bin/sh -c 'if test -e $kubeletkubeconfig; then stat -c %a $kubeletkubeconfig; fi'"
       tests:
         bin_op: or
         test_items:
@@ -342,12 +342,12 @@ groups:
       remediation: |
         Run the below command (based on the file location on your system) on the each worker
         node. For example,
-        chmod 644 $kubeletconf
+        chmod 644 $kubeletkubeconfig
       scored: true
 
     - id: 2.2.2
       text: "Ensure that the kubelet.conf file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi'"
+      audit: "/bin/sh -c 'if test -e $kubeletkubeconfig; then stat -c %U:%G $kubeletkubeconfig; fi'"
       tests:
         test_items:
           - flag: "root:root"
@@ -358,7 +358,7 @@ groups:
       remediation: |
         Run the below command (based on the file location on your system) on the each worker
         node. For example,
-        chown root:root $kubeletconf
+        chown root:root $kubeletkubeconfig
       scored: true
 
     - id: 2.2.3
@@ -404,7 +404,7 @@ groups:
 
     - id: 2.2.5
       text: "Ensure that the proxy kubeconfig file permissions are set to 644 or more restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %a $proxyconf; fi'"
+      audit: "/bin/sh -c 'if test -e $proxykubeconfig; then stat -c %a $proxykubeconfig; fi'"
       tests:
         bin_op: or
         test_items:
@@ -426,12 +426,12 @@ groups:
       remediation: |
         Run the below command (based on the file location on your system) on the each worker
         node. For example,
-        chmod 644 $proxyconf
+        chmod 644 $proxykubeconfig
       scored: true
 
     - id: 2.2.6
       text: "Ensure that the proxy kubeconfig file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %U:%G $proxyconf; fi'"
+      audit: "/bin/sh -c 'if test -e $proxykubeconfig; then stat -c %U:%G $proxykubeconfig; fi'"
       tests:
         test_items:
         - flag: "root:root"
@@ -439,7 +439,7 @@ groups:
       remediation: |
           Run the below command (based on the file location on your system) on the each worker
           node. For example,
-          chown root:root $proxyconf
+          chown root:root $proxykubeconfig
       scored: true
 
     - id: 2.2.7
@@ -462,19 +462,19 @@ groups:
 
     - id: 2.2.9
       text: "Ensure that the kubelet configuration file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then stat -c %U:%G /var/lib/kubelet/config.yaml; fi'"
+      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi'"
       tests:
         test_items:
         - flag: "root:root"
           set: true
       remediation: |
         Run the following command (using the config file location identied in the Audit step)
-        chown root:root /etc/kubernetes/kubelet.conf
+        chown root:root $kubeletconf
       scored: true
 
     - id: 2.2.10
       text: "Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then stat -c %a /var/lib/kubelet/config.yaml; fi'"
+      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %a $kubeletconf; fi'"
       tests:
         bin_op: or
         test_items:
@@ -495,5 +495,5 @@ groups:
           set: true
       remediation: |
         Run the following command (using the config file location identied in the Audit step)
-        chmod 644 /var/lib/kubelet/config.yaml
+        chmod 644 $kubeletconf
       scored: true

cfg/1.8/config.yaml

@@ -9,36 +9,13 @@
 
 master:
   apiserver:
-    confs:
-      - /etc/kubernetes/manifests/kube-apiserver.yaml
-      - /etc/kubernetes/manifests/kube-apiserver.manifest
     defaultconf: /etc/kubernetes/manifests/kube-apiserver.yaml
 
   scheduler:
-    confs: 
-      - /etc/kubernetes/manifests/kube-scheduler.yaml
-      - /etc/kubernetes/manifests/kube-scheduler.manifest
     defaultconf: /etc/kubernetes/manifests/kube-scheduler.yaml
 
   controllermanager:
-    confs:
-      - /etc/kubernetes/manifests/kube-controller-manager.yaml
-      - /etc/kubernetes/manifests/kube-controller-manager.manifest
     defaultconf: /etc/kubernetes/manifests/kube-controller-manager.yaml
 
   etcd:
-    confs:
-      - /etc/kubernetes/manifests/etcd.yaml
-      - /etc/kubernetes/manifests/etcd.manifest
     defaultconf: /etc/kubernetes/manifests/etcd.yaml
-
-node:
-  kubelet:
-    confs:
-     - /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
-    defaultconf: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
-  
-  proxy:
-    confs:
-      - /etc/kubernetes/addons/kube-proxy-daemonset.yaml
-    defaultconf: /etc/kubernetes/addons/kube-proxy-daemonset.yaml

cfg/1.8/node.yaml

@@ -297,7 +297,7 @@ groups:
     - id: 2.2.1
       text: "Ensure that the kubelet.conf file permissions are set to 644 or
       more restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %a $kubeletconf; fi'"
+      audit: "/bin/sh -c 'if test -e $kubeletkubeconfig; then stat -c %a $kubeletkubeconfig; fi'"
       tests:
         bin_op: or
         test_items:
@@ -319,12 +319,12 @@ groups:
       remediation: |
         Run the below command (based on the file location on your system) on the each worker
         node. For example,
-        chmod 644 $kubeletconf
+        chmod 644 $kubeletkubeconfig
       scored: true
 
     - id: 2.2.2
       text: "Ensure that the kubelet.conf file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi'"
+      audit: "/bin/sh -c 'if test -e $kubeletkubeconfig; then stat -c %U:%G $kubeletkubeconfig; fi'"
       tests:
         test_items:
           - flag: "root:root"
@@ -335,7 +335,7 @@ groups:
       remediation: |
         Run the below command (based on the file location on your system) on the each worker
         node. For example,
-        chown root:root $kubeletconf
+        chown root:root $kubeletkubeconfig
       scored: true
 
     - id: 2.2.3
@@ -382,7 +382,7 @@ groups:
     - id: 2.2.5
       text: "Ensure that the proxy kubeconfig file permissions are set to 644 or more
       restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %a $proxyconf; fi'"
+      audit: "/bin/sh -c 'if test -e $proxykubeconfig; then stat -c %a $proxykubeconfig; fi'"
       tests:
         bin_op: or
         test_items:
@@ -404,12 +404,12 @@ groups:
       remediation: |
         Run the below command (based on the file location on your system) on the each worker
         node. For example,
-        chmod 644 $proxyconf
+        chmod 644 $proxykubeconfig
       scored: true
 
     - id: 2.2.6
       text: "Ensure that the proxy kubeconfig file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %U:%G $proxyconf; fi'"
+      audit: "/bin/sh -c 'if test -e $proxykubeconfig; then stat -c %U:%G $proxykubeconfig; fi'"
       tests:
         test_items:
         - flag: "root:root"
@@ -417,7 +417,7 @@ groups:
       remediation: |
           Run the below command (based on the file location on your system) on the each worker
           node. For example,
-          chown root:root $proxyconf
+          chown root:root $proxykubeconfig
       scored: true
 
     - id: 2.2.7

cfg/config.yaml

@@ -78,11 +78,9 @@ node:
     bins:
       - "hyperkube kubelet"
       - "kubelet"
-    confs:
-      - /etc/kubernetes/kubelet.conf
-      - /etc/kubernetes/kubelet 
-    defaultconf: "/etc/kubernetes/kubelet.conf"
+    defaultconf: "/var/lib/kubelet/config.yaml"
     defaultsvc: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+    defaultkubeconfig: "/etc/kubernetes/kubelet.conf"
 
   proxy:
     bins:
@@ -90,9 +88,9 @@ node:
       - "hyperkube proxy"
       - "proxy"
     confs:
-      - /etc/kubernetes/proxy.conf
       - /etc/kubernetes/proxy
       - /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    defaultkubeconfig: "/etc/kubernetes/proxy.conf"
 
 federated:
   components:

cmd/common.go

@@ -83,12 +83,14 @@ func runChecks(nodetype check.NodeType) {
 	binmap := getBinaries(typeConf)
 	confmap := getConfigFiles(typeConf)
 	svcmap := getServiceFiles(typeConf)
+	kubeconfmap := getKubeConfigFiles(typeConf)
 
 	// Variable substitutions. Replace all occurrences of variables in controls files.
 	s := string(in)
 	s = makeSubstitutions(s, "bin", binmap)
 	s = makeSubstitutions(s, "conf", confmap)
 	s = makeSubstitutions(s, "svc", svcmap)
+	s = makeSubstitutions(s, "kubeconfig", kubeconfmap)
 
 	controls, err := check.NewControls(nodetype, []byte(s))
 	if err != nil {

cmd/util.go

@@ -219,6 +219,37 @@ func getServiceFiles(v *viper.Viper) map[string]string {
 	return svcmap
 }
 
+// getKubeConfigFiles finds which of the set of candidate kubeconfig files exist
+func getKubeConfigFiles(v *viper.Viper) map[string]string {
+	kubeconfigmap := make(map[string]string)
+
+	for _, component := range v.GetStringSlice("components") {
+		s := v.Sub(component)
+		if s == nil {
+			continue
+		}
+
+		// See if any of the candidate config files exist
+		kubeconfig := findConfigFile(s.GetStringSlice("kubeconfig"))
+		if kubeconfig == "" {
+			if s.IsSet("defaultkubeconfig") {
+				kubeconfig = s.GetString("defaultkubeconfig")
+				glog.V(2).Info(fmt.Sprintf("Using default kubeconfig file name '%s' for component %s", kubeconfig, component))
+			} else {
+				// Default the service file name that we'll substitute to the name of the component
+				glog.V(2).Info(fmt.Sprintf("Missing service file for %s", component))
+				kubeconfig = component
+			}
+		} else {
+			glog.V(2).Info(fmt.Sprintf("Component %s uses service file '%s'", component, kubeconfig))
+		}
+
+		kubeconfigmap[component] = kubeconfig
+	}
+
+	return kubeconfigmap
+}
+
 // verifyBin checks that the binary specified is running
 func verifyBin(bin string) bool {