From df3577519cffe77350d4ef60945cc233bef9b97c Mon Sep 17 00:00:00 2001 From: Liz Rice Date: Thu, 30 May 2019 22:55:48 +0100 Subject: [PATCH 1/8] Document version-specific config files Values in the version-specific files override the main file --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8eb2b98..ed8852d 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,8 @@ kube-bench includes a set of test files for Red Hat's OpenShift hardening guide Kubernetes config and binary file locations and names can vary from installation to installation, so these are configurable in the `cfg/config.yaml` file. +Any settings in the version-specific config file `cfg//config.yaml` take precedence over settings in the main `cfg/config.yaml` file. + For each type of node (*master*, *node* or *federated*) there is a list of components, and for each component there is a set of binaries (*bins*) and config files (*confs*) that kube-bench will look for (in the order they are listed). If your installation uses a different binary name or config file location for a Kubernetes component, you can add it to `cfg/config.yaml`. * **bins** - If there is a *bins* list for a component, at least one of these binaries must be running. The tests will consider the parameters for the first binary in the list found to be running. From 9d577d94b4aa2b03a3246f615ade212340c8b652 Mon Sep 17 00:00:00 2001 From: Liz Rice Date: Thu, 30 May 2019 23:04:44 +0100 Subject: [PATCH 2/8] Update openshift executables --- cfg/ocp-3.10/config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cfg/ocp-3.10/config.yaml b/cfg/ocp-3.10/config.yaml index c63b27b..df15172 100644 --- a/cfg/ocp-3.10/config.yaml +++ b/cfg/ocp-3.10/config.yaml @@ -4,7 +4,18 @@ master: apiserver: bins: + - openshift start master api - hypershift openshift-kube-apiserver + + scheduler: + bins: + - "openshift start master controllers" + confs: + - /etc/origin/master/scheduler.json + + controllermanager: + bins: + - "openshift start master controllers" etcd: bins: From 9d0e3491a03e4721a47248018526bbbcb459fcc7 Mon Sep 17 00:00:00 2001 From: 030 Date: Sat, 1 Jun 2019 16:40:45 +0200 Subject: [PATCH 3/8] [GH-191] explained that master nodes cannot be inspected in managed k8s --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8eb2b98..fbd261c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ kube-bench logo -kube-bench is a Go application that checks whether Kubernetes is deployed securely by running the checks documented in the [CIS Kubernetes Benchmark](https://www.cisecurity.org/benchmark/kubernetes/). +kube-bench is a Go application that checks whether Kubernetes is deployed securely by running the checks documented in the [CIS Kubernetes Benchmark](https://www.cisecurity.org/benchmark/kubernetes/). Note that it is impossible to inspect master nodes of managed clusters, e.g. GKE, EKS and AKS, using kube-bench as one does not have access to such nodes. Tests are configured with YAML files, making this tool easy to update as test specifications evolve. From 27df1f60ed4523f2a0b254d8e5ab16323e492781 Mon Sep 17 00:00:00 2001 From: Liz Rice Date: Sat, 1 Jun 2019 18:17:09 +0200 Subject: [PATCH 4/8] Clarification about worker nodes in managed k8s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because we don’t want to put people off running kube-bench altogether in these environments --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fbd261c..c4ddf29 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ kube-bench logo -kube-bench is a Go application that checks whether Kubernetes is deployed securely by running the checks documented in the [CIS Kubernetes Benchmark](https://www.cisecurity.org/benchmark/kubernetes/). Note that it is impossible to inspect master nodes of managed clusters, e.g. GKE, EKS and AKS, using kube-bench as one does not have access to such nodes. +kube-bench is a Go application that checks whether Kubernetes is deployed securely by running the checks documented in the [CIS Kubernetes Benchmark](https://www.cisecurity.org/benchmark/kubernetes/). + +Note that it is impossible to inspect the master nodes of managed clusters, e.g. GKE, EKS and AKS, using kube-bench as one does not have access to such nodes, although it is still possible to use kube-bench to check worker node configuration in these environments. Tests are configured with YAML files, making this tool easy to update as test specifications evolve. From 43caaab00acb2eb07671d652d07a967b3fd18292 Mon Sep 17 00:00:00 2001 From: Daniel Sagi Date: Tue, 4 Jun 2019 17:14:43 +0300 Subject: [PATCH 5/8] added another kubelet config file to paths, in the main config yaml file. default location for gke cluster --- cfg/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cfg/config.yaml b/cfg/config.yaml index 912ca89..4eccf7c 100644 --- a/cfg/config.yaml +++ b/cfg/config.yaml @@ -86,6 +86,7 @@ node: confs: - "/var/lib/kubelet/config.yaml" - "/etc/kubernetes/kubelet/kubelet-config.json" + - "/home/kubernetes/kubelet-config.yaml" defaultconf: "/var/lib/kubelet/config.yaml" defaultsvc: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" defaultkubeconfig: "/etc/kubernetes/kubelet.conf" From 85849a3c1ff1e287db4114c90a7430636db44dc8 Mon Sep 17 00:00:00 2001 From: Abubakr-Sadik Nii Nai Davis Date: Tue, 9 Apr 2019 06:10:12 +0000 Subject: [PATCH 6/8] Add detailed kube-bench config documentation. --- docs/README.md | 397 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..51dc57e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,397 @@ +# Overview + +`kube-bench` runs checks specified in `controls` files that are a YAML +representation of the CIS Kubernetes Benchmark checks. There is a +`controls` file per kubernetes version and node type. + +kube-bench automatically selects which `controls` to use based on the detected +node type and the version of kubernetes a cluster is running. This behaviour +can be overridden by specifying the `master` or `node` subcommand and the +`--version` flag on the command line. + +For example: +run kube-bench against a master with version auto-detection: + +``` +kube-bench master +``` + +or run kube-bench against a node with the node `controls` for kubernetes +version 1.12: +``` +kube-bench node --version 1.12 +``` + +`controls` for the various versions of kubernetes can be found in directories +with same name as the kubernetes versions under `cfg/`, for example `cfg/1.12`. +`controls` are also organized by distribution under the `cfg` directory for +example `cfg/ocp-3.10`. + + +## Controls + +`controls` is a YAML document that contains checks that must be run against a +specific kubernetes node type, master or node and version. + +`controls` is the fundamental input to `kube-bench`. The following is an example +of a basic `controls`: + +``` +--- +controls: +id: 1 +text: "Master Node Security Configuration" +type: "master" +groups: +- id: 1.1 + text: API Server + checks: + - id: 1.1.1 + text: "Ensure that the --allow-privileged argument is set (Scored)" + audit: "ps -ef | grep kube-apiserver | grep -v grep" + tests: + bin_op: or + test_items: + - flag: "--allow-privileged" + set: true + - flag: "--some-other-flag" + set: false + remediation: "Edit the /etc/kubernetes/config file on the master node and + set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'" + scored: true +- id: 1.2 + text: Scheduler + checks: + - id: 1.2.1 + text: "Ensure that the --profiling argument is set to false (Scored)" + audit: "ps -ef | grep kube-scheduler | grep -v grep" + tests: + bin_op: or + test_items: + - flag: "--profiling" + set: true + - flag: "--some-other-flag" + set: false + remediation: "Edit the /etc/kubernetes/config file on the master node and + set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'" + scored: true +``` + +`controls` is composed of a hierachy of groups, sub-groups and checks. Each of +the `controls` components have an id and a text description which are displayed +in the `kube-bench` output. + +`type` specifies what kubernetes node type a `controls` is for. Possible values +for `type` are `master` and `node`. + +## Groups + +`groups` is list of subgroups which test the various kubernetes components +that run on the node type specified in the `controls`. + +For example one subgroup checks parameters passed to the apiserver binary, while +another subgroup checks parameters passed to the controller-manager binary. + +``` +groups: +- id: 1.1 + text: API Server + ... +- id: 1.2 + text: Scheduler + ... +``` + +These subgroups have `id`, `text` fields which serve the same purposes described +in the previous paragraphs. The most important part of the subgroup is the +`checks` field which is the collection of actual `check`s that form the subgroup. + +This is an example of a subgroup and checks in the subgroup. + +``` +id: 1.1 +text: API Server +checks: + - id: 1.1.1 + text: "Ensure that the --allow-privileged argument is set (Scored)" + audit: "ps -ef | grep kube-apiserver | grep -v grep" + tests: + ... + - id: 1.1.2 + text: "Ensure that the --anonymous-auth argument is set to false (Not Scored)" + audit: "ps -ef | grep kube-apiserver | grep -v grep" + tests: + ... +``` + +`kube-bench` supports running a subgroup by specifying the subgroup `id` on the +command line, with the flag `--group` or `-g`. + +## Check + +The CIS Kubernetes Benchmark recommends configurations to harden kubernetes +components. These recommendations are usually configuration options, and can be +specified by flags to kubernetes binaries, or in configuration files. + +The Benchmark also provides commands to audit a kubernetes installation, identify +places where the cluster security can be improved, and steps to remediate these +identified problems. + +In `kube-bench`, `check` objects embody these recommendations. This an example +`check` object: + +``` +id: 1.1.1 +text: "Ensure that the --anonymous-auth argument is set to false (Not Scored)" +audit: "ps -ef | grep kube-apiserver | grep -v grep" +tests: + test_items: + - flag: "--anonymous-auth" + compare: + op: eq + value: false + set: true +remediation: | + Edit the API server pod specification file kube-apiserver + on the master node and set the below parameter. + --anonymous-auth=false +scored: false +``` + +A `check` object has an `id`, a `text`, an `audit` , a `tests`,`remediation` +and `scored` fields. + +`kube-bench` supports running individual checks by specifying the check's `id` +as a comma-delimited list on the command line with the `--check` flag. + +The `audit` field specifies the command to run for a check. The output of this +command is then evaluated for conformance with the CIS Kubernetes Benchmark +recommendation. + +The audit is evaluated against a criteria specified by the `tests` +object. `tests` contain `bin_op` and `test_items`. + +`test_items` specify the criteria(s) the `audit` command's output should meet to +pass a check. This criteria is made up of keywords extracted from the output of +the `audit` command and operations that compare the these keywords against +values expected by the CIS Kubernetes Benchmark. + +The are two ways to extract keywords from the output of the `audit` command, +`flag` and `path`. + +`flag` is used when the keyword is a command line flag. The associated `audit` +command is usually a `ps` command and a `grep` for the binary whose flag we are +checking: + +``` +ps -ef | grep somebinary | grep -v grep +``` + +Here is an example usage of the `flag` option: + +``` +... +audit: "ps -ef | grep kube-apiserver | grep -v grep" +tests: + test_items: + - flag: "--anonymous-auth" + ... +``` + +`path` is used when the keyword is an option set in a JSON or YAML config file. +The associated `audit` command is usually `cat /path/to/config-yaml-or-json`. +For example: + +``` +... + +text: "Ensure that the --anonymous-auth argument is set to false (Not Scored)" +audit: "cat /path/to/some/config" +tests: + test_items: + - path: "{.someoption.value}" + ... +``` + +`test_item` compares the output of the audit command and keywords using the +`set` and `compare` fields. + +``` + test_items: + - flag: "--anonymous-auth" + compare: + op: eq + value: false + set: true +``` + +`set` checks if a keyword is present in the output of the audit command or in +a config file. The possible values for `set` are true and false. + +If `set` is true, the check passes only if the keyword is present in the output +of the audit command, or config file. If `set` is false, the check passes only +if the keyword is not present in the output of the audit command, or config file. + +`compare` has two fields `op` and `value` to compare keywords with expected +value. `op` specifies which operation is used for the comparison , and `value` +specifies the value to compare against. + +> To use `compare`, `set` must true. The comparison will be ignored if `set` is +> false + +The `op` (operations) currently supported in `kube-bench` are: +- `eq`: tests if the keyword is equal to the compared value. +- `noteq`: tests if the keyword is unequal to the compared value. +- `gt`: tests if the keyword is greater than the compared value. +- `gte`: tests if the keyword is greater than or equal to the compared value. +- `lt`: tests if the keyword is less than the compared value. +- `lte`: tests if the keyword is less than or equal to the compared value. +- `has`: tests if the keyword contains the compared value. +- `nothave`: tests if the keyword does not contain the compared value. + +## Configuration and Variables + +Kubernetes component configuration and binary file locations and names +vary based on cluster deployment methods and kubernetes distribution used. +For this reason, the locations of these binaries and config files are configurable +by editing the `cfg/config.yaml` file and these binaries and files can be +referenced in a `controls` file via variables. + +The `cfg/config.yaml` file is a global configuration file. Configuration files +can be created for specific Kubernetes versions (distributions). Values in the +version specific config overwrite similar values in `cfg/config.yaml`. + +For example, the kube-apiserver in Redhat OCP distribution is run as +`hypershift openshift-kube-apiserver` instead of the default `kube-apiserver`. +This difference can be specified by editing the `master.apiserver.defaultbin` +entry `cfg/ocp-3.10/config.yaml`. + +Below is the structure of `cfg/config.yaml`: + +``` +nodetype + |-- components + |-- component1 + |-- component1 + |-- bins + |-- defaultbin (optional) + |-- confs + |-- defaultconf (optional) + |-- svcs + |-- defaultsvc (optional) + |-- kubeconfig + |-- defaultkubeconfig (optional) +``` + +Every node type has a subsection that specifies the main configurations items. + +- `components`: A list of components for the node type. For example master + will have an entry for **apiserver**, **scheduler** and **controllermanager**. + + Each component has the following entries: + +- `bins`: A list of candidate binaries for a component. `kube-bench` checks this + list and selects the first binary that is running on the node, if none is + running, `kube-bench` terminates. + + If `defaultbin` is specified, `kube-bench` ignores the `bins` list (if it is + specified) and verifies the binary specified with `defaultbin` is running on + the node. `kube-bench` terminates if this binary is not running. + + The selected binary for a component can be referenced in `controls` using a + variable in the form `$bin`. In the example below, we reference + the selected API server binary with the variable `$apiserverbin` in an `audit` + command. + + ``` + id: 1.1.1 + text: "Ensure that the --anonymous-auth argument is set to false (Scored)" + audit: "ps -ef | grep $apiserverbin | grep -v grep" + ... + ``` + +- `confs`: A list of candidate configuration files for a component. `kube-bench` + checks this list and selects the first config fille that is found on the node, + if none of the config files exists `kube-bench` terminates. + + If `defaultconf`is specified for a component, `kube-bench` ignores the `confs` + list (if it is specified) and verifies the config specified by `defaultconf` + exists on the node. `kube-bench` terminates if this file does not exist. + + The selected config for a component can be referenced in `controls` using a + variable in the form `$conf`. In the example below we reference the + selected API server config file with the variable `$apiserverconf` in an `audit` + command. + + ``` + id: 1.4.1 + text: "Ensure that the API server pod specification file permissions are + set to 644 or more restrictive (Scored)" + audit: "/bin/sh -c 'if test -e $apiserverconf; then stat -c %a $apiserverconf; fi'" + + ``` + +- `svcs`: A list of candidates unitfiles for a component. `kube-bench` checks this + list and selects the first unitfile that is found on the node, if none of the + unitfiles exists `kube-bench` terminates. + + If `defaultsvc`is specified for a component, `kube-bench` ignores the `svcs` + list (if it is specified) and verifies the unitfile specified by `defaultsvc` + exists on the node. `kube-bench` terminates if this file does not exist. + + The selected unitfile for a component can be referenced in `controls` via a + variable in the form `$svc`. In the example below, the selected + kubelet unitfile is referenced with `$kubeletsvc` in the `remediation` of the + `check`. + + ``` + id: 2.1.1 + ... + remediation: | + Edit the kubelet service file $kubeletsvc + on each worker node and set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. + --allow-privileged=false + Based on your system, restart the kubelet service. For example: + systemctl daemon-reload + systemctl restart kubelet.service + ... + ``` + + - `kubeconfig`: A list of candidate kubeconfig files for a component. `kube-bench` + checks this list and selects the first file that is found on the node, if none + of the files exists `kube-bench` terminates. + + If `defaultkubeconfig` is specified for a component, `kube-bench` ignores the + `kubeconfig` list (if it is specified) and verifies the kubeconfig file exists on + the node. `kube-bench` terminates if this file does not exist. + + The selected kubeconfig for a component can be referenced in `controls` with + a variable in the form `$kubeconfig`. In the example below, the + selected kubelet kubeconfig is referenced with `$kubeletkubeconfig` in the + `audit` command. + + ``` + id: 2.2.1 + text: "Ensure that the kubelet.conf file permissions are set to 644 or + more restrictive (Scored)" + audit: "/bin/sh -c 'if test -e $kubeletkubeconfig; then stat -c %a $kubeletkubeconfig; fi'" + ... + ``` + +## Versions and distributions + +`kube-bench` has `controls` files for multiple Kubernetes versions and +distributions. The supported versions and distributions can be found under the +`cfg/` directory in the project root. + +The versions listed in `cfg` are kubernetes versions not CIS Kubernetes Benchmark +versions and they are not the same. Please refer to the version matrix below to +see how kubernetes versions map to CIS Kubernetes Benchmarks versions. + +| CIS Kubernetes Benchmark | kube-bench config | Kubernetes versions | +|---|---|---| +| 1.0.0| 1.6 | 1.6 | +| 1.1.0| 1.7 | 1.7 | +| 1.2.0| 1.8 | 1.8-1.10 | +| 1.3.0| 1.11 | 1.11-1.12 | +| 1.4.0| 1.13 | 1.13- | From 87820b9775912b3428b5acd431f1c0f8ac328e50 Mon Sep 17 00:00:00 2001 From: Liz Rice Date: Wed, 5 Jun 2019 10:28:11 +0100 Subject: [PATCH 7/8] Remove duplicate versions section That info is important enough that it needs to stay in the main README. I also changed the file title --- docs/README.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docs/README.md b/docs/README.md index 51dc57e..6214882 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -# Overview +# Test and config files `kube-bench` runs checks specified in `controls` files that are a YAML representation of the CIS Kubernetes Benchmark checks. There is a @@ -377,21 +377,3 @@ Every node type has a subsection that specifies the main configurations items. audit: "/bin/sh -c 'if test -e $kubeletkubeconfig; then stat -c %a $kubeletkubeconfig; fi'" ... ``` - -## Versions and distributions - -`kube-bench` has `controls` files for multiple Kubernetes versions and -distributions. The supported versions and distributions can be found under the -`cfg/` directory in the project root. - -The versions listed in `cfg` are kubernetes versions not CIS Kubernetes Benchmark -versions and they are not the same. Please refer to the version matrix below to -see how kubernetes versions map to CIS Kubernetes Benchmarks versions. - -| CIS Kubernetes Benchmark | kube-bench config | Kubernetes versions | -|---|---|---| -| 1.0.0| 1.6 | 1.6 | -| 1.1.0| 1.7 | 1.7 | -| 1.2.0| 1.8 | 1.8-1.10 | -| 1.3.0| 1.11 | 1.11-1.12 | -| 1.4.0| 1.13 | 1.13- | From 5df39eed029608ba2ad649e32adeb13ee98a4d72 Mon Sep 17 00:00:00 2001 From: Simarpreet Singh Date: Mon, 10 Jun 2019 13:37:29 -0700 Subject: [PATCH 8/8] ocp-3.10: Fix malformed yaml and improve TestControls_RunChecks This improves the TestControls_RunChecks() test by making more comprehensive assertions on a more fully fledged input yaml Fixes: https://github.com/aquasecurity/kube-bench/issues/304 Signed-off-by: Simarpreet Singh --- cfg/ocp-3.10/node.yaml | 2 +- check/controls_test.go | 26 ++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/cfg/ocp-3.10/node.yaml b/cfg/ocp-3.10/node.yaml index fc27642..cc894c5 100644 --- a/cfg/ocp-3.10/node.yaml +++ b/cfg/ocp-3.10/node.yaml @@ -196,7 +196,7 @@ groups: - id: 7.15 text: "Verify that the RotateKubeletServerCertificate argument is set to true" audit: "grep -B1 RotateKubeletServerCertificate=true /etc/origin/node/node-config.yaml" - test: + tests: test_items: - flag: "RotateKubeletServerCertificate=true" compare: diff --git a/check/controls_test.go b/check/controls_test.go index 18e92cb..bcf14d5 100644 --- a/check/controls_test.go +++ b/check/controls_test.go @@ -103,13 +103,26 @@ type: "master" groups: - id: G1 checks: - - id: G1/C1 + - id: G1/C1 - id: G2 checks: - - id: G2/C1 + - id: G2/C1 + text: "Verify that the SomeSampleFlag argument is set to true" + audit: "grep -B1 SomeSampleFlag=true /this/is/a/file/path" + tests: + test_items: + - flag: "SomeSampleFlag=true" + compare: + op: has + value: "true" + set: true + remediation: | + Edit the config file /this/is/a/file/path and set SomeSampleFlag to true. + scored: true `) // and - controls, _ := NewControls(MASTER, in) + controls, err := NewControls(MASTER, in) + assert.NoError(t, err) // and runner.On("Run", controls.Groups[0].Checks[0]).Return(PASS) runner.On("Run", controls.Groups[1].Checks[0]).Return(FAIL) @@ -130,6 +143,12 @@ groups: G2 := controls.Groups[1] assert.Equal(t, "G2", G2.ID) assert.Equal(t, "G2/C1", G2.Checks[0].ID) + assert.Equal(t, "has", G2.Checks[0].Tests.TestItems[0].Compare.Op) + assert.Equal(t, "true", G2.Checks[0].Tests.TestItems[0].Compare.Value) + assert.Equal(t, true, G2.Checks[0].Tests.TestItems[0].Set) + assert.Equal(t, "SomeSampleFlag=true", G2.Checks[0].Tests.TestItems[0].Flag) + assert.Equal(t, "Edit the config file /this/is/a/file/path and set SomeSampleFlag to true.\n", G2.Checks[0].Remediation) + assert.Equal(t, true, G2.Checks[0].Scored) assertEqualGroupSummary(t, 0, 1, 0, 0, G2) // and assert.Equal(t, 1, controls.Summary.Pass) @@ -139,7 +158,6 @@ groups: // and runner.AssertExpectations(t) }) - } func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn int, actual *Group) {