From 772839fc92b90aa5c3e74830a3912b8f43d5f51d Mon Sep 17 00:00:00 2001 From: Liz Rice Date: Sun, 30 Aug 2020 08:16:21 +0100 Subject: [PATCH] move target mapping to config.yaml - updated version (#682) * move target mapping to config.yaml * Update config.yaml * Update common.go * Add support for eks-1.0 Add also eks-1.0 to map * chore: merge correction * Move file only used for testing * Tidier logs * Add target mapping for GKE and EKS * fingers cross this finishes target mapping Co-authored-by: Murali Paluru Co-authored-by: Roberto Rojas Co-authored-by: yoavrotems --- cfg/config.yaml | 30 ++++++++++++++++++++ cmd/common.go | 54 ++++++++++++++++++++---------------- cmd/common_test.go | 16 +++++++---- cmd/database.go | 2 +- cmd/root.go | 37 +++++++++++++++++------- cmd/run.go | 22 ++++++++++----- cmd/util.go | 5 ++-- cmd/version.go | 1 - {cfg => hack}/node_only.yaml | 0 integration/docker.go | 6 ++-- 10 files changed, 119 insertions(+), 54 deletions(-) rename {cfg => hack}/node_only.yaml (100%) diff --git a/cfg/config.yaml b/cfg/config.yaml index 5c2aaf9..f772e5e 100644 --- a/cfg/config.yaml +++ b/cfg/config.yaml @@ -201,3 +201,33 @@ version_mapping: "gke-1.0": "gke-1.0" "ocp-3.10": "rh-0.7" "ocp-3.11": "rh-0.7" + +target_mapping: + "cis-1.3": + - "master" + - "node" + "cis-1.4": + - "master" + - "node" + "cis-1.5": + - "master" + - "node" + - "controlplane" + - "etcd" + - "policies" + "gke-1.0": + - "master" + - "node" + - "controlplane" + - "etcd" + - "policies" + - "managedservices" + "eks-1.0": + - "master" + - "node" + - "controlplane" + - "policies" + - "managedservices" + "rh-0.7": + - "master" + - "node" diff --git a/cmd/common.go b/cmd/common.go index 0bc815e..b36205e 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -249,7 +249,7 @@ func mapToBenchmarkVersion(kubeToBenchmarkMap map[string]string, kv string) (str if !found { glog.V(1).Info(fmt.Sprintf("mapToBenchmarkVersion unable to find a match for: %q", kvOriginal)) - glog.V(3).Info(fmt.Sprintf("mapToBenchmarkVersion kubeToBenchmarkSMap: %#v", kubeToBenchmarkMap)) + glog.V(3).Info(fmt.Sprintf("mapToBenchmarkVersion kubeToBenchmarkMap: %#v", kubeToBenchmarkMap)) return "", fmt.Errorf("unable to find a matching Benchmark Version match for kubernetes version: %s", kvOriginal) } @@ -265,6 +265,15 @@ func loadVersionMapping(v *viper.Viper) (map[string]string, error) { return kubeToBenchmarkMap, nil } +func loadTargetMapping(v *viper.Viper) (map[string][]string, error) { + benchmarkVersionToTargetsMap := v.GetStringMapStringSlice("target_mapping") + if len(benchmarkVersionToTargetsMap) == 0 { + return nil, fmt.Errorf("config file is missing 'target_mapping' section") + } + + return benchmarkVersionToTargetsMap, nil +} + func getBenchmarkVersion(kubeVersion, benchmarkVersion string, v *viper.Viper) (bv string, err error) { if !isEmpty(kubeVersion) && !isEmpty(benchmarkVersion) { return "", fmt.Errorf("It is an error to specify both --version and --benchmark flags") @@ -306,16 +315,16 @@ func isEtcd() bool { } func isThisNodeRunning(nodeType check.NodeType) bool { - glog.V(2).Infof("Checking if the current node is running %s components", nodeType) - etcdConf := viper.Sub(string(nodeType)) - if etcdConf == nil { - glog.V(2).Infof("No %s components found to be running", nodeType) + glog.V(3).Infof("Checking if the current node is running %s components", nodeType) + nodeTypeConf := viper.Sub(string(nodeType)) + if nodeTypeConf == nil { + glog.V(2).Infof("No config for %s components found", nodeType) return false } - components, err := getBinariesFunc(etcdConf, nodeType) + components, err := getBinariesFunc(nodeTypeConf, nodeType) if err != nil { - glog.V(2).Info(err) + glog.V(2).Infof("Failed to find %s binaries: %v", nodeType, err) return false } if len(components) == 0 { @@ -323,6 +332,7 @@ func isThisNodeRunning(nodeType check.NodeType) bool { return false } + glog.V(2).Infof("Node is running %s components", nodeType) return true } @@ -337,7 +347,7 @@ func writeOutput(controlsCollection []*check.Controls) { return } if jsonFmt { - writeJsonOutput(controlsCollection) + writeJSONOutput(controlsCollection) return } if pgSQL { @@ -347,12 +357,12 @@ func writeOutput(controlsCollection []*check.Controls) { writeStdoutOutput(controlsCollection) } -func writeJsonOutput(controlsCollection []*check.Controls) { +func writeJSONOutput(controlsCollection []*check.Controls) { out, err := json.Marshal(controlsCollection) if err != nil { exitWithError(fmt.Errorf("failed to output in JSON format: %v", err)) } - PrintOutput(string(out), outputFile) + printOutput(string(out), outputFile) } func writeJunitOutput(controlsCollection []*check.Controls) { @@ -361,7 +371,7 @@ func writeJunitOutput(controlsCollection []*check.Controls) { if err != nil { exitWithError(fmt.Errorf("failed to output in JUnit format: %v", err)) } - PrintOutput(string(out), outputFile) + printOutput(string(out), outputFile) } } @@ -400,7 +410,7 @@ func writeOutputToFile(output string, outputFile string) error { return w.Flush() } -func PrintOutput(output string, outputFile string) { +func printOutput(output string, outputFile string) { if len(outputFile) == 0 { fmt.Println(output) } else { @@ -411,20 +421,16 @@ func PrintOutput(output string, outputFile string) { } } -var benchmarkVersionToTargetsMap = map[string][]string{ - "cis-1.3": []string{string(check.MASTER), string(check.NODE)}, - "cis-1.4": []string{string(check.MASTER), string(check.NODE)}, - "cis-1.5": []string{string(check.MASTER), string(check.NODE), string(check.CONTROLPLANE), string(check.ETCD), string(check.POLICIES)}, - "gke-1.0": []string{string(check.MASTER), string(check.NODE), string(check.CONTROLPLANE), string(check.ETCD), string(check.POLICIES), string(check.MANAGEDSERVICES)}, - "eks-1.0": []string{string(check.NODE), string(check.CONTROLPLANE), string(check.POLICIES), string(check.MANAGEDSERVICES)}, -} - // validTargets helps determine if the targets // are legitimate for the benchmarkVersion. -func validTargets(benchmarkVersion string, targets []string) bool { +func validTargets(benchmarkVersion string, targets []string, v *viper.Viper) (bool, error) { + benchmarkVersionToTargetsMap, err := loadTargetMapping(v) + if err != nil { + return false, err + } providedTargets, found := benchmarkVersionToTargetsMap[benchmarkVersion] if !found { - return false + return false, fmt.Errorf("No targets configured for %s", benchmarkVersion) } for _, pt := range targets { @@ -437,9 +443,9 @@ func validTargets(benchmarkVersion string, targets []string) bool { } if !f { - return false + return false, nil } } - return true + return true, nil } diff --git a/cmd/common_test.go b/cmd/common_test.go index 1a230fd..909c46e 100644 --- a/cmd/common_test.go +++ b/cmd/common_test.go @@ -154,7 +154,7 @@ func TestIsMaster(t *testing.T) { }, { name: "valid config, does not include master", - cfgFile: "../cfg/node_only.yaml", + cfgFile: "../hack/node_only.yaml", isMaster: false, }, } @@ -364,6 +364,10 @@ func TestGetBenchmarkVersion(t *testing.T) { } func TestValidTargets(t *testing.T) { + viperWithData, err := loadConfigForTest() + if err != nil { + t.Fatalf("Unable to load config file %v", err) + } cases := []struct { name string benchmark string @@ -410,7 +414,10 @@ func TestValidTargets(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - ret := validTargets(c.benchmark, c.targets) + ret, err := validTargets(c.benchmark, c.targets, viperWithData) + if err != nil { + t.Fatalf("Expected nil error, got: %v", err) + } if ret != c.expected { t.Fatalf("Expected %t, got %t", c.expected, ret) } @@ -451,7 +458,7 @@ func TestIsEtcd(t *testing.T) { }, { name: "valid config, does not include etcd", - cfgFile: "../cfg/node_only.yaml", + cfgFile: "../hack/node_only.yaml", isEtcd: false, }, } @@ -532,11 +539,10 @@ func parseControlsJsonFile(filepath string) ([]*check.Controls, error) { func loadConfigForTest() (*viper.Viper, error) { viperWithData := viper.New() - viperWithData.SetConfigFile(filepath.Join("..", cfgDir, "config.yaml")) + viperWithData.SetConfigFile("../cfg/config.yaml") if err := viperWithData.ReadInConfig(); err != nil { return nil, err } - return viperWithData, nil } diff --git a/cmd/database.go b/cmd/database.go index a12778e..369290c 100644 --- a/cmd/database.go +++ b/cmd/database.go @@ -53,7 +53,7 @@ func savePgsql(jsonInfo string) { exitWithError(fmt.Errorf("received error connecting to database: %s", err)) } defer db.Close() - + db.Debug().AutoMigrate(&ScanResult{}) db.Save(&ScanResult{ScanHost: hostname, ScanTime: timestamp, ScanInfo: jsonInfo}) glog.V(2).Info(fmt.Sprintf("successfully stored result to: %s", envVars["PGSQL_HOST"])) diff --git a/cmd/root.go b/cmd/root.go index 702ab15..39b850c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -68,40 +68,57 @@ var RootCmd = &cobra.Command{ if err != nil { exitWithError(fmt.Errorf("unable to determine benchmark version: %v", err)) } + glog.V(1).Infof("Running checks for benchmark %v", bv) if isMaster() { - glog.V(1).Info("== Running master checks ==\n") + glog.V(1).Info("== Running master checks ==") runChecks(check.MASTER, loadConfig(check.MASTER, bv)) // Control Plane is only valid for CIS 1.5 and later, // this a gatekeeper for previous versions - if validTargets(bv, []string{string(check.CONTROLPLANE)}) { - glog.V(1).Info("== Running control plane checks ==\n") + valid, err := validTargets(bv, []string{string(check.CONTROLPLANE)}, viper.GetViper()) + if err != nil { + exitWithError(fmt.Errorf("error validating targets: %v", err)) + } + if valid { + glog.V(1).Info("== Running control plane checks ==") runChecks(check.CONTROLPLANE, loadConfig(check.CONTROLPLANE, bv)) } } // Etcd is only valid for CIS 1.5 and later, // this a gatekeeper for previous versions. - if validTargets(bv, []string{string(check.ETCD)}) && isEtcd() { - glog.V(1).Info("== Running etcd checks ==\n") + valid, err := validTargets(bv, []string{string(check.ETCD)}, viper.GetViper()) + if err != nil { + exitWithError(fmt.Errorf("error validating targets: %v", err)) + } + if valid && isEtcd() { + glog.V(1).Info("== Running etcd checks ==") runChecks(check.ETCD, loadConfig(check.ETCD, bv)) } - glog.V(1).Info("== Running node checks ==\n") + glog.V(1).Info("== Running node checks ==") runChecks(check.NODE, loadConfig(check.NODE, bv)) // Policies is only valid for CIS 1.5 and later, // this a gatekeeper for previous versions. - if validTargets(bv, []string{string(check.POLICIES)}) { - glog.V(1).Info("== Running policies checks ==\n") + valid, err = validTargets(bv, []string{string(check.POLICIES)}, viper.GetViper()) + if err != nil { + exitWithError(fmt.Errorf("error validating targets: %v", err)) + } + if valid { + glog.V(1).Info("== Running policies checks ==") runChecks(check.POLICIES, loadConfig(check.POLICIES, bv)) } // Managedservices is only valid for GKE 1.0 and later, // this a gatekeeper for previous versions. - if validTargets(bv, []string{string(check.MANAGEDSERVICES)}) { - glog.V(1).Info("== Running managed services checks ==\n") + valid, err = validTargets(bv, []string{string(check.MANAGEDSERVICES)}, viper.GetViper()) + if err != nil { + exitWithError(fmt.Errorf("error validating targets: %v", err)) + } + if valid { + glog.V(1).Info("== Running managed services checks ==") runChecks(check.MANAGEDSERVICES, loadConfig(check.MANAGEDSERVICES, bv)) } diff --git a/cmd/run.go b/cmd/run.go index 72a2f23..20199c5 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -16,7 +16,7 @@ func init() { RootCmd.AddCommand(runCmd) runCmd.Flags().StringSliceP("targets", "s", []string{}, `Specify targets of the benchmark to run. These names need to match the filenames in the cfg/ directory. - For example, to run the tests specified in master.yaml and etcd.yaml, specify --targets=master,etcd + For example, to run the tests specified in master.yaml and etcd.yaml, specify --targets=master,etcd If no targets are specified, run tests from all files in the cfg/ directory. `) } @@ -32,21 +32,29 @@ var runCmd = &cobra.Command{ exitWithError(fmt.Errorf("unable to get `targets` from command line :%v", err)) } - benchmarkVersion, err := getBenchmarkVersion(kubeVersion, benchmarkVersion, viper.GetViper()) + bv, err := getBenchmarkVersion(kubeVersion, benchmarkVersion, viper.GetViper()) if err != nil { exitWithError(fmt.Errorf("unable to get benchmark version. error: %v", err)) } - glog.V(2).Infof("Checking targets %v for %v", targets, benchmarkVersion) - if len(targets) > 0 && !validTargets(benchmarkVersion, targets) { - exitWithError(fmt.Errorf(fmt.Sprintf(`The specified --targets "%s" does not apply to the CIS Benchmark %s \n Valid targets %v`, strings.Join(targets, ","), benchmarkVersion, benchmarkVersionToTargetsMap[benchmarkVersion]))) + glog.V(2).Infof("Checking targets %v for %v", targets, bv) + benchmarkVersionToTargetsMap, err := loadTargetMapping(viper.GetViper()) + if err != nil { + exitWithError(fmt.Errorf("error loading targets: %v", err)) + } + valid, err := validTargets(bv, targets, viper.GetViper()) + if err != nil { + exitWithError(fmt.Errorf("error validating targets: %v", err)) + } + if len(targets) > 0 && !valid { + exitWithError(fmt.Errorf(fmt.Sprintf(`The specified --targets "%s" are not configured for the CIS Benchmark %s\n Valid targets %v`, strings.Join(targets, ","), bv, benchmarkVersionToTargetsMap[bv]))) } // Merge version-specific config if any. - path := filepath.Join(cfgDir, benchmarkVersion) + path := filepath.Join(cfgDir, bv) mergeConfig(path) - err = run(targets, benchmarkVersion) + err = run(targets, bv) if err != nil { fmt.Printf("Error in run: %v\n", err) } diff --git a/cmd/util.go b/cmd/util.go index 4dd7f87..82b2928 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -72,7 +72,7 @@ func ps(proc string) string { if err != nil { glog.V(2).Info(fmt.Errorf("%s: %s", cmd.Args, err)) } - + glog.V(2).Info(fmt.Sprintf("ps - returning: %q", string(out))) return string(out) } @@ -214,9 +214,8 @@ func verifyBin(bin string) bool { // but apiserver is not a match for kube-apiserver reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin) lines := strings.Split(out, "\n") - glog.V(2).Info(fmt.Sprintf("verifyBin - lines(%d)", len(lines))) for _, l := range lines { - glog.V(2).Info(fmt.Sprintf("reFirstWord.Match(%s)\n\n\n\n", l)) + glog.V(3).Info(fmt.Sprintf("reFirstWord.Match(%s)", l)) if reFirstWord.Match([]byte(l)) { return true } diff --git a/cmd/version.go b/cmd/version.go index e6a4f12..beb0469 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -20,4 +20,3 @@ var versionCmd = &cobra.Command{ func init() { RootCmd.AddCommand(versionCmd) } - diff --git a/cfg/node_only.yaml b/hack/node_only.yaml similarity index 100% rename from cfg/node_only.yaml rename to hack/node_only.yaml diff --git a/integration/docker.go b/integration/docker.go index fa7fddd..7cf2f39 100644 --- a/integration/docker.go +++ b/integration/docker.go @@ -14,7 +14,7 @@ import ( ) func loadImageFromDocker(imageName string, kindCtx *cluster.Context) error { - + // Check that the image exists locally and gets its ID, if not return error _, err := docker.ImageID(imageName) if err != nil { @@ -25,7 +25,7 @@ func loadImageFromDocker(imageName string, kindCtx *cluster.Context) error { if err != nil { return err } - + // Save the image into a tar dir, err := fs.TempDir("", "image-tar") if err != nil { @@ -58,4 +58,4 @@ func loadImage(imageTarName string, node *clusternodes.Node) error { } defer f.Close() return node.LoadImageArchive(f) -} \ No newline at end of file +}