Merge remote-tracking branch 'origin/master' into support for multiple

Kubernetes versions.

README.md

@@ -47,7 +47,10 @@ Flags:
 ## Configuration
 Kubernetes config and binary file locations and names can vary from installation to installation, so these are configurable in the `cfg/config.yaml` file. 
 
-They also tend to vary according to which tool was used to install Kubernetes. You can use the `--installation` flag to pick up a different default set of file names and locations. Again these defaults are configurable through `cfg/config.yaml` (and pull requests to correct or add default file locations are especially welcome). 
+For each type of node (*master*, *node* or *federated*) there is a list of components, and for each component there is a set of binaries (*bins*) and config files (*confs*) that kube-bench will look for (in the order they are listed). If your installation uses a different binary name or config file location for a Kubernetes component, you can add it to `cfg/config.yaml`.  
+
+* **bins** - If there is a *bins* list for a component, at least one of these binaries must be running. The tests will consider the parameters for the first binary in the list found to be running. 
+* **confs** - If one of the listed config files is found, this will be considered for the test. Tests can continue even if no config file is found. If no file is found at any of the listed locations, and a *defaultconf* location is given for the component, the test will give remediation advice using the *defaultconf* location. 
 
 ## Test config YAML representation
 The tests are represented as YAML documents (installed by default into ./cfg).

cfg/1.7/master.yaml

@@ -637,7 +637,7 @@ groups:
 
     - id: 1.4.3
       text: "Ensure that the config file permissions are set to 644 or more restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e $config; then stat -c %a $config; fi'"
+      audit: "/bin/sh -c 'if test -e $kubernetesconf; then stat -c %a $kubernetesconf; fi'"
       tests:
         bin_op: or
         test_items:
@@ -657,12 +657,12 @@ groups:
             value: "600"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chmod 644 $config"
+              \nFor example, chmod 644 $kubernetesconf"
       scored: true
 
     - id: 1.4.4
       text: "Ensure that the config file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e $config; then stat -c %U:%G $config; fi'"
+      audit: "/bin/sh -c 'if test -e $kubernetesconf; then stat -c %U:%G $kubernetesconf; fi'"
       tests:
         test_items:
         - flag: "root:root"
@@ -671,7 +671,7 @@ groups:
             value: "root:root"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chown root:root $config"
+              \nFor example, chown root:root $kubernetesconf"
       scored: true
 
     - id: 1.4.5

cfg/1.7/node.yaml

@@ -18,7 +18,7 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $config file on each node and set the KUBE_ALLOW_PRIV 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBE_ALLOW_PRIV 
               parameter to \"--allow-privileged=false\""
       scored: true
 
@@ -200,7 +200,7 @@ groups:
               op: eq
               value: true
             set: true
-      remediation: "Edit the /etc/kubernetes/kubelet file on each node and set the KUBELET_ARGS parameter
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS parameter
               to a value to include \"--feature-gates=RotateKubeletClientCertificate=true\"."
       scored: true
 
@@ -214,7 +214,7 @@ groups:
               op: eq
               value: true
             set: true
-      remediation: "Edit the /etc/kubernetes/kubelet file on each node and set the KUBELET_ARGS parameter
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS parameter
               to a value to include \"--feature-gates=RotateKubeletServerCertificate=true\"."
       scored: true
 
@@ -223,7 +223,7 @@ groups:
   checks:
     - id: 2.2.1
       text: "Ensure that the config file permissions are set to 644 or more restrictive (Scored)"
-      audit: "/bin/sh -c 'if test -e $config; then stat -c %a $config; fi'"
+      audit: "/bin/sh -c 'if test -e $kubernetesconf; then stat -c %a $kubernetesconf; fi'"
       tests:
         bin_op: or
         test_items:
@@ -243,12 +243,12 @@ groups:
               value: "600"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $config"
+              \nFor example, chmod 644 $kubernetesconf"
       scored: true
 
     - id: 2.2.2
       text: "Ensure that the config file ownership is set to root:root (Scored)"
-      audit: "/bin/sh -c 'if test -e $config; then stat -c %U:%G $config; fi'"
+      audit: "/bin/sh -c 'if test -e $kubernetesconf; then stat -c %U:%G $kubernetesconf; fi'"
       tests:
         test_items:
           - flag: "root:root"
@@ -257,7 +257,7 @@ groups:
               value: root:root
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $config"
+              \nFor example, chown root:root $kubernetesconf"
       scored: true
 
     - id: 2.2.3

cfg/config.yaml

@@ -7,106 +7,112 @@
 # nodeControls: ./cfg/node.yaml
 # federatedControls: ./cfg/federated.yaml
 
-## Support components
+master:
+  components:
+    - apiserver
+    - scheduler
+    - controllermanager
+    - etcd 
+    - flanneld
+    # kubernetes is a component to cover the config file /etc/kubernetes/config that is referred to in the benchmark
+    - kubernetes
+
+  kubernetes:
+    defaultconf: /etc/kubernetes/config
+
+  apiserver:
+    bins:
+      - "kube-apiserver"
+      - "hyperkube apiserver"
+      - "apiserver"
+    confs:
+      - /etc/kubernetes/manifests/kube-apiserver.yaml
+      - /etc/kubernetes/apiserver.conf
+      - /etc/kubernetes/apiserver
+    defaultconf: /etc/kubernetes/apiserver
+
+  scheduler:
+    bins:
+      - "kube-scheduler"
+      - "hyperkube scheduler"
+      - "scheduler"
+    confs: 
+      - /etc/kubernetes/manifests/kube-scheduler.yaml
+      - /etc/kubernetes/scheduler.conf
+      - /etc/kubernetes/scheduler
+    defaultconf: /etc/kubernetes/scheduler
+
+  controllermanager:
+    bins:
+      - "kube-controller-manager"
+      - "hyperkube controller-manager"
+      - "controller-manager"
+    confs:
+      - /etc/kubernetes/manifests/kube-controller-manager.yaml
+      - /etc/kubernetes/controller-manager.conf
+      - /etc/kubernetes/controller-manager
+    defaultconf: /etc/kubernetes/controller-manager
+
   etcd:
-  bin: etcd
+    optional: true
-  conf: /etc/etcd/etcd.conf
+    bins:
+      - "etcd"
+    confs:
+      - /etc/kubernetes/manifests/etcd.yaml
+      - /etc/etcd/etcd.conf
+    defaultconf: /etc/etcd/etcd.conf
 
   flanneld:
-  bin: flanneld
+    optional: true
-  conf: /etc/sysconfig/flanneld
+    bins:
+      - flanneld
+    defaultconf: /etc/sysconfig/flanneld
 
-# Installation
-# Configure kubernetes component binaries and paths to their configuration files.
-installation:
-  default:
-    config: /etc/kubernetes/config
-    master:
-      bin:
-        apiserver: apiserver
-        scheduler: scheduler
-        controller-manager: controller-manager
-      conf:
-        apiserver: /etc/kubernetes/apiserver
-        scheduler: /etc/kubernetes/scheduler
-        controller-manager: /etc/kubernetes/controller-manager
-    node:
-      bin:
-        kubelet: kubelet
-        proxy: proxy
-      conf:
-        kubelet: /etc/kubernetes/kubelet
-        proxy: /etc/kubernetes/proxy
-    federated:
-      bin:
-        apiserver: federation-apiserver
-        controller-manager: federation-controller-manager
 
-  kops:
-    config: /etc/kubernetes/config
-    master:
-      bin:
-        apiserver: apiserver
-        scheduler: scheduler
-        controller-manager: controller-manager
-      conf:
-        apiserver: /etc/kubernetes/apiserver
-        scheduler: /etc/kubernetes/scheduler
-        controller-manager: /etc/kubernetes/apiserver
 node:
-      bin:
+  components:
-        kubelet: kubelet
+    - kubelet
-        proxy: proxy
+    - proxy
-      conf:
+    # kubernetes is a component to cover the config file /etc/kubernetes/config that is referred to in the benchmark
-        kubelet: /etc/kubernetes/kubelet
+    - kubernetes
-        proxy: /etc/kubernetes/proxy
-    federated:
-      bin:
-        apiserver: federation-apiserver
-        controller-manager: federation-controller-manager
 
-  hyperkube:
+  kubernetes:
-    config: /etc/kubernetes/config
+    defaultconf: /etc/kubernetes/config    
-    master:
+
-      bin:
+  kubelet:
-        apiserver: hyperkube apiserver
+    bins:
-        scheduler: hyperkube scheduler
+      - "hyperkube kubelet"
-        controller-manager: hyperkube controller-manager
+      - "kubelet"
-      conf:
+    confs:
-        apiserver: /etc/kubernetes/manifests/kube-apiserver.yaml
+      - /etc/kubernetes/kubelet.conf
-        scheduler: /etc/kubernetes/manifests/kube-scheduler.yaml
+      - /etc/kubernetes/kubelet 
-        controller-manager: /etc/kubernetes/manifests/kube-controller-manager.yaml
+    defaultconf: "/etc/kubernetes/kubelet.conf"
-    node:
+  
-      bin:
+  proxy:
-        kubelet: hyperkube kubelet
+    bins:
-        proxy: hyperkube proxy
+      - "kube-proxy"
-      conf:
+      - "hyperkube proxy"
-        kubelet: /etc/kubernetes/kubelet
+      - "proxy"
-        proxy: /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    confs:
-    federated:
+      - /etc/kubernetes/proxy.conf
-      bin:
+      - /etc/kubernetes/proxy
-        apiserver: hyperkube federation-apiserver
+      - /etc/kubernetes/addons/kube-proxy-daemonset.yaml
-        controller-manager: hyperkube federation-controller-manager
 
-  kubeadm:
-    config: /etc/kubernetes/config
-    master:
-      bin:
-        apiserver: kube-apiserver
-        scheduler: kube-scheduler
-        controller-manager: kube-controller-manager
-      conf:
-        apiserver: /etc/kubernetes/admin.conf
-        scheduler: /etc/kubernetes/scheduler.conf
-        controller-manager: /etc/kubernetes/controller-manager.conf
-    node:
-      bin:
-        kubelet: kubelet
-        proxy: kube-proxy
-      conf:
-        kubelet: /etc/kubernetes/kubelet.conf
-        proxy: /etc/kubernetes/proxy.conf
 federated:
-      bin:
+  components:
-        apiserver: kube-federation-apiserver
+    - fedapiserver
-        controller-manager: kube-federation-controller-manager
+    - fedcontrollermanager
+
+  fedapiserver:
+    bins:
+      - "hyperkube federation-apiserver"
+      - "kube-federation-apiserver"
+      - "federation-apiserver"
+
+  fedcontrollermanager:
+    bins:
+      - "hyperkube federation-controller-manager"
+      - "kube-federation-controller-manager"
+      - "federation-controller-manager"
+
+

check/check.go

@@ -156,7 +156,9 @@ func (c *Check) Run() {
 		i++
 	}
 
-	glog.V(2).Info("%s\n", errmsgs)
+	if errmsgs != "" {
+		glog.V(2).Info(errmsgs)
+	}
 
 	res := c.Tests.execute(out.String())
 	if res {

check/data

@@ -116,3 +116,45 @@ groups:
             op: eq
             value: "600"
           set: true
+
+    - id: 10
+      text: "flag value includes some value in a comma-separated list, value is last in list"
+      tests:
+        test_items:
+          - flag: "--admission-control"
+            compare:
+              op: has
+              value: RBAC
+            set: true
+
+    - id: 11
+      text: "flag value includes some value in a comma-separated list, value is first in list"
+      tests:
+        test_items:
+          - flag: "--admission-control"
+            compare:
+              op: has
+              value: WebHook
+            set: true
+
+    - id: 12
+      text: "flag value includes some value in a comma-separated list, value middle of list"
+      tests:
+        test_items:
+          - flag: "--admission-control"
+            compare:
+              op: has
+              value: Something
+            set: true
+
+    - id: 13
+      text: "flag value includes some value in a comma-separated list, value only one in list"
+      tests:
+        test_items:
+          - flag: "--admission-control"
+            compare:
+              op: has
+              value: Something
+            set: true
+
+

check/test.go

@@ -62,7 +62,8 @@ func (t *testItem) execute(s string) (result bool) {
 			// --flag=somevalue
 			// --flag
 			// somevalue
-			pttn := `(` + t.Flag + `)(=)*([^\s,]*) *`
+			//pttn := `(` + t.Flag + `)(=)*([^\s,]*) *`
+			pttn := `(` + t.Flag + `)(=)*([^\s]*) *`
 			flagRe := regexp.MustCompile(pttn)
 			vals := flagRe.FindStringSubmatch(s)
 

check/test_test.go

@@ -94,6 +94,22 @@ func TestTestExecute(t *testing.T) {
 			controls.Groups[0].Checks[9],
 			"600",
 		},
+		{
+			controls.Groups[0].Checks[10],
+			"2:45 ../kubernetes/kube-apiserver --option --admission-control=WebHook,RBAC ---audit-log-maxage=40",
+		},
+		{
+			controls.Groups[0].Checks[11],
+			"2:45 ../kubernetes/kube-apiserver --option --admission-control=WebHook,RBAC ---audit-log-maxage=40",
+		},
+		{
+			controls.Groups[0].Checks[12],
+			"2:45 ../kubernetes/kube-apiserver --option --admission-control=WebHook,Something,RBAC ---audit-log-maxage=40",
+		},
+		{
+			controls.Groups[0].Checks[13],
+			"2:45 ../kubernetes/kube-apiserver --option --admission-control=Something ---audit-log-maxage=40",
+		},
 	}
 
 	for _, c := range cases {

cmd/common.go

@@ -17,9 +17,9 @@ package cmd
 import (
 	"fmt"
 	"io/ioutil"
-	"os"
 
 	"github.com/aquasecurity/kube-bench/check"
+	"github.com/golang/glog"
 	"github.com/spf13/viper"
 )
 
@@ -52,34 +52,30 @@ var (
 func runChecks(t check.NodeType) {
 	var summary check.Summary
 	var file string
+	var err error
+	var typeConf *viper.Viper
 
-	// Master variables
+	glog.V(1).Info(fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
-	apiserverBin = viper.GetString("installation." + installation + ".master.bin.apiserver")
-	apiserverConf = viper.GetString("installation." + installation + ".master.conf.apiserver")
-	schedulerBin = viper.GetString("installation." + installation + ".master.bin.scheduler")
-	schedulerConf = viper.GetString("installation." + installation + ".master.conf.scheduler")
-	controllerManagerBin = viper.GetString("installation." + installation + ".master.bin.controller-manager")
-	controllerManagerConf = viper.GetString("installation." + installation + ".master.conf.controller-manager")
-	config = viper.GetString("installation." + installation + ".config")
 
-	etcdBin = viper.GetString("etcd.bin")
+	switch t {
-	etcdConf = viper.GetString("etcd.conf")
+	case check.MASTER:
-	flanneldBin = viper.GetString("flanneld.bin")
+		file = masterFile
-	flanneldConf = viper.GetString("flanneld.conf")
+		typeConf = viper.Sub("master")
+	case check.NODE:
+		file = nodeFile
+		typeConf = viper.Sub("node")
+	case check.FEDERATED:
+		file = federatedFile
+		typeConf = viper.Sub("federated")
+	}
 
-	// Node variables
+	// Get the set of exectuables and config files we care about on this type of node. This also
-	kubeletBin = viper.GetString("installation." + installation + ".node.bin.kubelet")
+	// checks that the executables we need for the node type are running.
-	kubeletConf = viper.GetString("installation." + installation + ".node.conf.kubelet")
+	binmap := getBinaries(typeConf)
-	proxyBin = viper.GetString("installation." + installation + ".node.bin.proxy")
+	confmap := getConfigFiles(typeConf)
-	proxyConf = viper.GetString("installation." + installation + ".node.conf.proxy")
-
-	// Federated
-	fedApiserverBin = viper.GetString("installation." + installation + ".federated.bin.apiserver")
-	fedControllerManagerBin = viper.GetString("installation." + installation + ".federated.bin.controller-manager")
 
 	// Run kubernetes installation validation checks.
 	verifyKubeVersion(kubeMajorVersion, kubeMinorVersion)
-	verifyNodeType(t)
 
 	switch t {
 	case check.MASTER:
@@ -98,26 +94,9 @@ func runChecks(t check.NodeType) {
 	}
 
 	// Variable substitutions. Replace all occurrences of variables in controls files.
-	s := multiWordReplace(string(in), "$apiserverbin", apiserverBin)
+	s := string(in)
-	s = multiWordReplace(s, "$apiserverconf", apiserverConf)
+	s = makeSubstitutions(s, "bin", binmap)
-	s = multiWordReplace(s, "$schedulerbin", schedulerBin)
+	s = makeSubstitutions(s, "conf", confmap)
-	s = multiWordReplace(s, "$schedulerconf", schedulerConf)
-	s = multiWordReplace(s, "$controllermanagerbin", controllerManagerBin)
-	s = multiWordReplace(s, "$controllermanagerconf", controllerManagerConf)
-	s = multiWordReplace(s, "$config", config)
-
-	s = multiWordReplace(s, "$etcdbin", etcdBin)
-	s = multiWordReplace(s, "$etcdconf", etcdConf)
-	s = multiWordReplace(s, "$flanneldbin", flanneldBin)
-	s = multiWordReplace(s, "$flanneldconf", flanneldConf)
-
-	s = multiWordReplace(s, "$kubeletbin", kubeletBin)
-	s = multiWordReplace(s, "$kubeletconf", kubeletConf)
-	s = multiWordReplace(s, "$proxybin", proxyBin)
-	s = multiWordReplace(s, "$proxyconf", proxyConf)
-
-	s = multiWordReplace(s, "$fedapiserverbin", fedApiserverBin)
-	s = multiWordReplace(s, "$fedcontrollermanagerbin", fedControllerManagerBin)
 
 	controls, err := check.NewControls(t, []byte(s))
 	if err != nil {
@@ -149,41 +128,6 @@ func runChecks(t check.NodeType) {
 	}
 }
 
-// verifyNodeType checks the executables and config files are as expected
-// for the specified tests (master, node or federated).
-func verifyNodeType(t check.NodeType) {
-	var bins []string
-	var confs []string
-
-	switch t {
-	case check.MASTER:
-		bins = []string{apiserverBin, schedulerBin, controllerManagerBin}
-		confs = []string{apiserverConf, schedulerConf, controllerManagerConf}
-	case check.NODE:
-		bins = []string{kubeletBin, proxyBin}
-		confs = []string{kubeletConf, proxyConf}
-	case check.FEDERATED:
-		bins = []string{fedApiserverBin, fedControllerManagerBin}
-	}
-
-	for _, bin := range bins {
-		if !verifyBin(bin, ps) {
-			printlnWarn(fmt.Sprintf("%s is not running", bin))
-		}
-	}
-
-	for _, conf := range confs {
-		_, err := os.Stat(conf)
-		if err != nil {
-			if os.IsNotExist(err) {
-				printlnWarn(fmt.Sprintf("Missing kubernetes config file: %s", conf))
-			} else {
-				exitWithError(fmt.Errorf("error looking for file %s: %v", conf, err))
-			}
-		}
-	}
-}
-
 // colorPrint outputs the state in a specific colour, along with a message string
 func colorPrint(state check.State, s string) {
 	colors[state].Printf("[%s] ", state)
@@ -192,8 +136,6 @@ func colorPrint(state check.State, s string) {
 
 // prettyPrint outputs the results to stdout in human-readable format
 func prettyPrint(r *check.Controls, summary check.Summary) {
-	colorPrint(check.INFO, fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
-
 	colorPrint(check.INFO, fmt.Sprintf("%s %s\n", r.ID, r.Text))
 	for _, g := range r.Groups {
 		colorPrint(check.INFO, fmt.Sprintf("%s %s\n", g.ID, g.Text))

cmd/root.go

@@ -34,14 +34,6 @@ var (
 	masterFile    string
 	nodeFile      string
 	federatedFile string
-
-	loud bool
-
-	kubeConfDir     string
-	etcdConfDir     string
-	flanneldConfDir string
-
-	installation string
 )
 
 // RootCmd represents the base command when called without any subcommands
@@ -67,12 +59,6 @@ func init() {
 	cobra.OnInitialize(initConfig)
 
 	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Prints the results as JSON")
-	RootCmd.PersistentFlags().StringVar(
-		&installation,
-		"installation",
-		"default",
-		"Specify how kubernetes cluster was installed. Possible values are default,hyperkube,kops,kubeadm",
-	)
 	RootCmd.PersistentFlags().StringVarP(
 		&checkList,
 		"check",

cmd/util.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aquasecurity/kube-bench/check"
 	"github.com/fatih/color"
 	"github.com/golang/glog"
+	"github.com/spf13/viper"
 )
 
 var (
@@ -22,6 +23,14 @@ var (
 	}
 )
 
+var psFunc func(string) string
+var statFunc func(string) (os.FileInfo, error)
+
+func init() {
+	psFunc = ps
+	statFunc = os.Stat
+}
+
 func printlnWarn(msg string) {
 	fmt.Fprintf(os.Stderr, "[%s] %s\n",
 		colors[check.WARN].Sprintf("%s", check.WARN),
@@ -43,7 +52,7 @@ func exitWithError(err error) {
 
 func continueWithError(err error, msg string) string {
 	if err != nil {
-		glog.V(1).Info(err)
+		glog.V(2).Info(err)
 	}
 
 	if msg != "" {
@@ -75,8 +84,71 @@ func ps(proc string) string {
 	return string(out)
 }
 
+// getBinaries finds which of the set of candidate executables are running
+func getBinaries(v *viper.Viper) map[string]string {
+	binmap := make(map[string]string)
+
+	for _, component := range v.GetStringSlice("components") {
+		s := v.Sub(component)
+		if s == nil {
+			continue
+		}
+
+		optional := s.GetBool("optional")
+		bins := s.GetStringSlice("bins")
+		if len(bins) > 0 {
+			bin, err := findExecutable(bins)
+			if err != nil && !optional {
+				exitWithError(fmt.Errorf("need %s executable but none of the candidates are running", component))
+			}
+
+			// Default the executable name that we'll substitute to the name of the component
+			if bin == "" {
+				bin = component
+				glog.V(2).Info(fmt.Sprintf("Component %s not running", component))
+			} else {
+				glog.V(2).Info(fmt.Sprintf("Component %s uses running binary %s", component, bin))
+			}
+			binmap[component] = bin
+		}
+	}
+
+	return binmap
+}
+
+// getConfigFiles finds which of the set of candidate config files exist
+func getConfigFiles(v *viper.Viper) map[string]string {
+	confmap := make(map[string]string)
+
+	for _, component := range v.GetStringSlice("components") {
+		s := v.Sub(component)
+		if s == nil {
+			continue
+		}
+
+		// See if any of the candidate config files exist
+		conf := findConfigFile(s.GetStringSlice("confs"))
+		if conf == "" {
+			if s.IsSet("defaultconf") {
+				conf = s.GetString("defaultconf")
+				glog.V(2).Info(fmt.Sprintf("Using default config file name '%s' for component %s", conf, component))
+			} else {
+				// Default the config file name that we'll substitute to the name of the component
+				printlnWarn(fmt.Sprintf("Missing config file for %s", component))
+				conf = component
+			}
+		} else {
+			glog.V(2).Info(fmt.Sprintf("Component %s uses config file '%s'", component, conf))
+		}
+
+		confmap[component] = conf
+	}
+
+	return confmap
+}
+
 // verifyBin checks that the binary specified is running
-func verifyBin(bin string, psFunc func(string) string) bool {
+func verifyBin(bin string) bool {
 
 	// Strip any quotes
 	bin = strings.Trim(bin, "'\"")
@@ -87,7 +159,47 @@ func verifyBin(bin string, psFunc func(string) string) bool {
 	proc := strings.Fields(bin)[0]
 	out := psFunc(proc)
 
-	return strings.Contains(out, bin)
+	// There could be multiple lines in the ps output
+	// The binary needs to be the first word in the ps output, except that it could be preceded by a path
+	// e.g. /usr/bin/kubelet is a match for kubelet
+	// but apiserver is not a match for kube-apiserver
+	reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin)
+	lines := strings.Split(out, "\n")
+	for _, l := range lines {
+		if reFirstWord.Match([]byte(l)) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// fundConfigFile looks through a list of possible config files and finds the first one that exists
+func findConfigFile(candidates []string) string {
+	for _, c := range candidates {
+		_, err := statFunc(c)
+		if err == nil {
+			return c
+		}
+		if !os.IsNotExist(err) {
+			exitWithError(fmt.Errorf("error looking for file %s: %v", c, err))
+		}
+	}
+
+	return ""
+}
+
+// findExecutable looks through a list of possible executable names and finds the first one that's running
+func findExecutable(candidates []string) (string, error) {
+	for _, c := range candidates {
+		if verifyBin(c) {
+			return c, nil
+		} else {
+			glog.V(1).Info(fmt.Sprintf("executable '%s' not running", c))
+		}
+	}
+
+	return "", fmt.Errorf("no candidates running")
 }
 
 func verifyKubeVersion(major string, minor string) {
@@ -194,3 +306,17 @@ func getKubeVersion() *version {
 
 	return ver
 }
+
+func makeSubstitutions(s string, ext string, m map[string]string) string {
+	for k, v := range m {
+		subst := "$" + k + ext
+		if v == "" {
+			glog.V(2).Info(fmt.Sprintf("No subsitution for '%s'\n", subst))
+			continue
+		}
+		glog.V(1).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v))
+		s = multiWordReplace(s, subst, v)
+	}
+
+	return s
+}

cmd/util_test.go

@@ -15,9 +15,13 @@
 package cmd
 
 import (
+	"os"
+	"reflect"
 	"regexp"
 	"strconv"
 	"testing"
+
+	"github.com/spf13/viper"
 )
 
 func TestCheckVersion(t *testing.T) {
@@ -78,10 +82,19 @@ func TestVersionMatch(t *testing.T) {
 }
 
 var g string
+var e []error
+var eIndex int
 
 func fakeps(proc string) string {
 	return g
 }
+
+func fakestat(file string) (os.FileInfo, error) {
+	err := e[eIndex]
+	eIndex++
+	return nil, err
+}
+
 func TestVerifyBin(t *testing.T) {
 	cases := []struct {
 		proc  string
@@ -95,14 +108,110 @@ func TestVerifyBin(t *testing.T) {
 		{proc: "cmd", psOut: "cmd param1 param2", exp: true},
 		{proc: "cmd param", psOut: "cmd param1 param2", exp: true},
 		{proc: "cmd param", psOut: "cmd", exp: false},
+		{proc: "cmd", psOut: "cmd x \ncmd y", exp: true},
+		{proc: "cmd y", psOut: "cmd x \ncmd y", exp: true},
+		{proc: "cmd", psOut: "/usr/bin/cmd", exp: true},
+		{proc: "cmd", psOut: "kube-cmd", exp: false},
+		{proc: "cmd", psOut: "/usr/bin/kube-cmd", exp: false},
 	}
 
+	psFunc = fakeps
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			g = c.psOut
+			v := verifyBin(c.proc)
+			if v != c.exp {
+				t.Fatalf("Expected %v got %v", c.exp, v)
+			}
+		})
+	}
+}
+
+func TestFindExecutable(t *testing.T) {
+	cases := []struct {
+		candidates []string // list of executables we'd consider
+		psOut      string   // fake output from ps
+		exp        string   // the one we expect to find in the (fake) ps output
+		expErr     bool
+	}{
+		{candidates: []string{"one", "two", "three"}, psOut: "two", exp: "two"},
+		{candidates: []string{"one", "two", "three"}, psOut: "two three", exp: "two"},
+		{candidates: []string{"one double", "two double", "three double"}, psOut: "two double is running", exp: "two double"},
+		{candidates: []string{"one", "two", "three"}, psOut: "blah", expErr: true},
+		{candidates: []string{"one double", "two double", "three double"}, psOut: "two", expErr: true},
+		{candidates: []string{"apiserver", "kube-apiserver"}, psOut: "kube-apiserver", exp: "kube-apiserver"},
+		{candidates: []string{"apiserver", "kube-apiserver", "hyperkube-apiserver"}, psOut: "kube-apiserver", exp: "kube-apiserver"},
+	}
+
+	psFunc = fakeps
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			g = c.psOut
+			e, err := findExecutable(c.candidates)
+			if e != c.exp {
+				t.Fatalf("Expected %v got %v", c.exp, e)
+			}
+
+			if err == nil && c.expErr {
+				t.Fatalf("Expected error")
+			}
+
+			if err != nil && !c.expErr {
+				t.Fatalf("Didn't expect error: %v", err)
+			}
+		})
+	}
+}
+
+func TestGetBinaries(t *testing.T) {
+	cases := []struct {
+		config map[string]interface{}
+		psOut  string
+		exp    map[string]string
+	}{
+		{
+			config: map[string]interface{}{"components": []string{"apiserver"}, "apiserver": map[string]interface{}{"bins": []string{"apiserver", "kube-apiserver"}}},
+			psOut:  "kube-apiserver",
+			exp:    map[string]string{"apiserver": "kube-apiserver"},
+		},
+		{
+			// "thing" is not in the list of components
+			config: map[string]interface{}{"components": []string{"apiserver"}, "apiserver": map[string]interface{}{"bins": []string{"apiserver", "kube-apiserver"}}, "thing": map[string]interface{}{"bins": []string{"something else", "thing"}}},
+			psOut:  "kube-apiserver thing",
+			exp:    map[string]string{"apiserver": "kube-apiserver"},
+		},
+		{
+			// "anotherthing" in list of components but doesn't have a defintion
+			config: map[string]interface{}{"components": []string{"apiserver", "anotherthing"}, "apiserver": map[string]interface{}{"bins": []string{"apiserver", "kube-apiserver"}}, "thing": map[string]interface{}{"bins": []string{"something else", "thing"}}},
+			psOut:  "kube-apiserver thing",
+			exp:    map[string]string{"apiserver": "kube-apiserver"},
+		},
+		{
+			// more than one component
+			config: map[string]interface{}{"components": []string{"apiserver", "thing"}, "apiserver": map[string]interface{}{"bins": []string{"apiserver", "kube-apiserver"}}, "thing": map[string]interface{}{"bins": []string{"something else", "thing"}}},
+			psOut:  "kube-apiserver \nthing",
+			exp:    map[string]string{"apiserver": "kube-apiserver", "thing": "thing"},
+		},
+		{
+			// default binary to component name
+			config: map[string]interface{}{"components": []string{"apiserver", "thing"}, "apiserver": map[string]interface{}{"bins": []string{"apiserver", "kube-apiserver"}}, "thing": map[string]interface{}{"bins": []string{"something else", "thing"}, "optional": true}},
+			psOut:  "kube-apiserver \notherthing some params",
+			exp:    map[string]string{"apiserver": "kube-apiserver", "thing": "thing"},
+		},
+	}
+
+	v := viper.New()
+	psFunc = fakeps
+
 	for id, c := range cases {
 		t.Run(strconv.Itoa(id), func(t *testing.T) {
 			g = c.psOut
-			v := verifyBin(c.proc, fakeps)
+			for k, val := range c.config {
-			if v != c.exp {
+				v.Set(k, val)
-				t.Fatalf("Expected %v got %v", c.exp, v)
+			}
+			m := getBinaries(v)
+			if !reflect.DeepEqual(m, c.exp) {
+				t.Fatalf("Got %v\nExpected %v", m, c.exp)
 			}
 		})
 	}
@@ -142,5 +251,118 @@ func TestGetKubeVersion(t *testing.T) {
 		if ok, err := regexp.MatchString(`\d+.\d+`, ver.Server); !ok && err != nil {
 			t.Logf("Expected:%v got %v\n", "n.m", ver.Server)
 		}
+
+	}
+}
+
+func TestFindConfigFile(t *testing.T) {
+	cases := []struct {
+		input       []string
+		statResults []error
+		exp         string
+	}{
+		{input: []string{"myfile"}, statResults: []error{nil}, exp: "myfile"},
+		{input: []string{"thisfile", "thatfile"}, statResults: []error{os.ErrNotExist, nil}, exp: "thatfile"},
+		{input: []string{"thisfile", "thatfile"}, statResults: []error{os.ErrNotExist, os.ErrNotExist}, exp: ""},
+	}
+
+	statFunc = fakestat
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			e = c.statResults
+			eIndex = 0
+			conf := findConfigFile(c.input)
+			if conf != c.exp {
+				t.Fatalf("Got %s expected %s", conf, c.exp)
+			}
+		})
+	}
+}
+
+func TestGetConfigFiles(t *testing.T) {
+	cases := []struct {
+		config      map[string]interface{}
+		exp         map[string]string
+		statResults []error
+	}{
+		{
+			config:      map[string]interface{}{"components": []string{"apiserver"}, "apiserver": map[string]interface{}{"confs": []string{"apiserver", "kube-apiserver"}}},
+			statResults: []error{os.ErrNotExist, nil},
+			exp:         map[string]string{"apiserver": "kube-apiserver"},
+		},
+		{
+			// Component "thing" isn't included in the list of components
+			config: map[string]interface{}{
+				"components": []string{"apiserver"},
+				"apiserver":  map[string]interface{}{"confs": []string{"apiserver", "kube-apiserver"}},
+				"thing":      map[string]interface{}{"confs": []string{"/my/file/thing"}}},
+			statResults: []error{os.ErrNotExist, nil},
+			exp:         map[string]string{"apiserver": "kube-apiserver"},
+		},
+		{
+			// More than one component
+			config: map[string]interface{}{
+				"components": []string{"apiserver", "thing"},
+				"apiserver":  map[string]interface{}{"confs": []string{"apiserver", "kube-apiserver"}},
+				"thing":      map[string]interface{}{"confs": []string{"/my/file/thing"}}},
+			statResults: []error{os.ErrNotExist, nil, nil},
+			exp:         map[string]string{"apiserver": "kube-apiserver", "thing": "/my/file/thing"},
+		},
+		{
+			// Default thing to specified default config
+			config: map[string]interface{}{
+				"components": []string{"apiserver", "thing"},
+				"apiserver":  map[string]interface{}{"confs": []string{"apiserver", "kube-apiserver"}},
+				"thing":      map[string]interface{}{"confs": []string{"/my/file/thing"}, "defaultconf": "another/thing"}},
+			statResults: []error{os.ErrNotExist, nil, os.ErrNotExist},
+			exp:         map[string]string{"apiserver": "kube-apiserver", "thing": "another/thing"},
+		},
+		{
+			// Default thing to component name
+			config: map[string]interface{}{
+				"components": []string{"apiserver", "thing"},
+				"apiserver":  map[string]interface{}{"confs": []string{"apiserver", "kube-apiserver"}},
+				"thing":      map[string]interface{}{"confs": []string{"/my/file/thing"}}},
+			statResults: []error{os.ErrNotExist, nil, os.ErrNotExist},
+			exp:         map[string]string{"apiserver": "kube-apiserver", "thing": "thing"},
+		},
+	}
+
+	v := viper.New()
+	statFunc = fakestat
+
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			for k, val := range c.config {
+				v.Set(k, val)
+			}
+			e = c.statResults
+			eIndex = 0
+
+			m := getConfigFiles(v)
+			if !reflect.DeepEqual(m, c.exp) {
+				t.Fatalf("Got %v\nExpected %v", m, c.exp)
+			}
+		})
+	}
+}
+
+func TestMakeSubsitutions(t *testing.T) {
+	cases := []struct {
+		input string
+		subst map[string]string
+		exp   string
+	}{
+		{input: "Replace $thisbin", subst: map[string]string{"this": "that"}, exp: "Replace that"},
+		{input: "Replace $thisbin", subst: map[string]string{"this": "that", "here": "there"}, exp: "Replace that"},
+		{input: "Replace $thisbin and $herebin", subst: map[string]string{"this": "that", "here": "there"}, exp: "Replace that and there"},
+	}
+	for _, c := range cases {
+		t.Run(c.input, func(t *testing.T) {
+			s := makeSubstitutions(c.input, "bin", c.subst)
+			if s != c.exp {
+				t.Fatalf("Got %s expected %s", s, c.exp)
+			}
+		})
 	}
 }