From f2386c03860c44ca3375d3eefd49d06792391908 Mon Sep 17 00:00:00 2001 From: Yoav Rotem Date: Wed, 24 Mar 2021 18:06:54 +0200 Subject: [PATCH] Update ocp 3.11 (#849) * Add OCP auto-detection * Add test for openshift * update and fix bugs update file to match with new kube-bench features and fix bugs * Update file and fix bugs update file to match with new kube-bench features and fix bugs * Remove specific configs Those configs could be set in main config.yaml * Update to include openshift files * fix typos * fix typo * Remove trailing spaces * Update util.go * Add tests for getOcpValidVersion --- cfg/config.yaml | 16 +- cfg/rh-0.7/config.yaml | 30 --- cfg/rh-0.7/master.yaml | 406 +++++++++++++++-------------------------- cfg/rh-0.7/node.yaml | 114 +++++------- cmd/util.go | 49 +++++ cmd/util_test.go | 34 ++++ 6 files changed, 284 insertions(+), 365 deletions(-) diff --git a/cfg/config.yaml b/cfg/config.yaml index 0003cbb..328590d 100644 --- a/cfg/config.yaml +++ b/cfg/config.yaml @@ -25,12 +25,15 @@ master: - "hyperkube apiserver" - "hyperkube kube-apiserver" - "apiserver" + - "openshift start master api" + - "hypershift openshift-kube-apiserver" confs: - /etc/kubernetes/manifests/kube-apiserver.yaml - /etc/kubernetes/manifests/kube-apiserver.yml - /etc/kubernetes/manifests/kube-apiserver.manifest - /var/snap/kube-apiserver/current/args - /var/snap/microk8s/current/args/kube-apiserver + - /etc/origin/master/master-config.yaml defaultconf: /etc/kubernetes/manifests/kube-apiserver.yaml scheduler: @@ -39,12 +42,14 @@ master: - "hyperkube scheduler" - "hyperkube kube-scheduler" - "scheduler" + - "openshift start master controllers" confs: - /etc/kubernetes/manifests/kube-scheduler.yaml - /etc/kubernetes/manifests/kube-scheduler.yml - /etc/kubernetes/manifests/kube-scheduler.manifest - /var/snap/kube-scheduler/current/args - /var/snap/microk8s/current/args/kube-scheduler + - /etc/origin/master/scheduler.json defaultconf: /etc/kubernetes/manifests/kube-scheduler.yaml kubeconfig: - /etc/kubernetes/scheduler.conf @@ -57,6 +62,8 @@ master: - "hyperkube controller-manager" - "hyperkube kube-controller-manager" - "controller-manager" + - "openshift start master controllers" + - "hypershift openshift-controller-manager" confs: - /etc/kubernetes/manifests/kube-controller-manager.yaml - /etc/kubernetes/manifests/kube-controller-manager.yml @@ -72,6 +79,7 @@ master: optional: true bins: - "etcd" + - "openshift start etcd" confs: - /etc/kubernetes/manifests/etcd.yaml - /etc/kubernetes/manifests/etcd.yml @@ -81,7 +89,6 @@ master: - /var/snap/etcd/common/etcd.conf.yaml - /var/snap/microk8s/current/args/etcd - /usr/lib/systemd/system/etcd.service - - /etc/kubernetes/manifests defaultconf: /etc/kubernetes/manifests/etcd.yaml flanneld: @@ -114,6 +121,8 @@ node: - "/lib/systemd/system/kubelet.service" - "/etc/systemd/system/snap.kubelet.daemon.service" - "/etc/systemd/system/snap.microk8s.daemon-kubelet.service" + - "/etc/systemd/system/atomic-openshift-node.service" + - "/etc/systemd/system/origin-node.service" bins: - "hyperkube kubelet" - "kubelet" @@ -121,13 +130,11 @@ node: - "/etc/kubernetes/kubelet.conf" - "/var/lib/kubelet/kubeconfig" - "/etc/kubernetes/kubelet-kubeconfig" - - "/etc/kubernetes/kubelet/kubeconfig" - "/var/snap/microk8s/current/credentials/kubelet.config" confs: - "/var/lib/kubelet/config.yaml" - "/var/lib/kubelet/config.yml" - "/etc/kubernetes/kubelet/kubelet-config.json" - - "/etc/kubernetes/kubelet/config" - "/home/kubernetes/kubelet-config.yaml" - "/home/kubernetes/kubelet-config.yml" - "/etc/default/kubelet" @@ -156,16 +163,15 @@ node: - "hyperkube proxy" - "hyperkube kube-proxy" - "proxy" + - "openshift start network" confs: - /etc/kubernetes/proxy - /etc/kubernetes/addons/kube-proxy-daemonset.yaml - /etc/kubernetes/addons/kube-proxy-daemonset.yml - /var/snap/kube-proxy/current/args - /var/snap/microk8s/current/args/kube-proxy - - /var/lib/kube-proxy/config.conf kubeconfig: - "/etc/kubernetes/kubelet-kubeconfig" - - "/etc/kubernetes/kubelet/config" - "/var/lib/kubelet/kubeconfig" - "/var/snap/microk8s/current/credentials/proxy.config" svc: diff --git a/cfg/rh-0.7/config.yaml b/cfg/rh-0.7/config.yaml index 4d4f563..b783945 100644 --- a/cfg/rh-0.7/config.yaml +++ b/cfg/rh-0.7/config.yaml @@ -1,32 +1,2 @@ --- ## Version-specific settings that override the values in cfg/config.yaml - -master: - apiserver: - bins: - - openshift start master api - - hypershift openshift-kube-apiserver - - scheduler: - bins: - - "openshift start master controllers" - - "hyperkube kube-scheduler" - confs: - - /etc/origin/master/scheduler.json - - controllermanager: - bins: - - "openshift start master controllers" - - "hypershift openshift-controller-manager" - - etcd: - bins: - - openshift start etcd - -node: - svcs: - - /etc/systemd/system/atomic-openshift-node.service - - /etc/systemd/system/origin-node.service - proxy: - bins: - - openshift start network diff --git a/cfg/rh-0.7/master.yaml b/cfg/rh-0.7/master.yaml index a60162b..3517568 100644 --- a/cfg/rh-0.7/master.yaml +++ b/cfg/rh-0.7/master.yaml @@ -16,13 +16,10 @@ groups: - id: 1.2 text: "Verify that the basic-auth-file method is not enabled" - audit: "grep -A2 basic-auth-file /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "--basic-auth-file" - compare: - op: eq - value: "" + - path: "{.kubernetesMasterConfig.apiServerArguments.basic-auth-file}" set: false remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml and @@ -41,35 +38,27 @@ groups: - id: 1.4 text: "Secure communications between the API server and master nodes" - audit: "grep -A4 kubeletClientInfo /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - - flag: "kubeletClientInfo:" - compare: - op: eq - value: "kubeletClientInfo:" - set: true - - flag: "ca" + - path: "{.kubeletClientInfo.ca}" compare: op: has value: "ca-bundle.crt" - set: true - - flag: "certFile" + - path: "{.kubeletClientInfo.certFile}" compare: op: has value: "master.kubelet-client.crt" - set: true - - flag: "keyFile" + - path: "{.kubeletClientInfo.keyFile}" compare: op: has value: "master.kubelet-client.key" set: true - - flag: "port: 10250" + - path: "{.kubeletClientInfo.port}" compare: op: eq - value: "port: 10250" - set: true + value: "10250" remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml and change it to match the below. @@ -83,10 +72,10 @@ groups: - id: 1.5 text: "Prevent insecure bindings" - audit: "grep -A2 insecure-bind-address /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "insecure-bind-address" + - path: "{.kubernetesMasterConfig.insecure-bind-address}" set: false remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml @@ -100,10 +89,10 @@ groups: - id: 1.6 text: "Prevent insecure port access" - audit: "grep -A2 insecure-port /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "insecure-port" + - path: "{.kubernetesMasterConfig.apiServerArguments.insecure-port}" set: false remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml @@ -117,17 +106,16 @@ groups: - id: 1.7 text: "Use Secure Ports for API Server Traffic" - audit: "grep -A2 secure-port /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: or test_items: - - flag: "secure-port" + - path: "{.kubernetesMasterConfig.apiServerArguments.secure-port}" set: false - - flag: "secure-port" + - path: "{.kubernetesMasterConfig.apiServerArguments.secure-port}" compare: - op: nothave + op: noteq value: "0" - set: true remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml and either remove the secure-port parameter or set it to a different (non-zero) @@ -146,17 +134,16 @@ groups: - id: 1.9 text: "Verify repair-malformed-updates argument for API compatibility" - audit: "grep -A2 repair-malformed-updates /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: or test_items: - - flag: "repair-malformed-updates" + - path: "{.repair-malformed-updates}" set: false - - flag: "repair-malformed-updates" + - path: "{.repair-malformed-updates}" compare: - op: has + op: eq value: "true" - set: true remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml and remove the repair-malformed-updates entry or set repair-malformed-updates=true. @@ -164,10 +151,10 @@ groups: - id: 1.10 text: "Verify that the AlwaysAdmit admission controller is disabled" - audit: "grep -A4 AlwaysAdmit /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "AlwaysAdmit" + - path: "{.AlwaysAdmit}" set: false remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml @@ -182,14 +169,13 @@ groups: - id: 1.11 text: "Manage the AlwaysPullImages admission controller" - audit: "grep -A4 AlwaysPullImages /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "disable" + - path: "{.admissionConfig.pluginConfig.configuration.disable}" compare: - op: has + op: eq value: "false" - set: true remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml and add the entry below. @@ -215,10 +201,10 @@ groups: - id: 1.14 text: "Manage the NamespaceLifecycle admission controller" - audit: "grep -A4 NamespaceLifecycle /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "NamespaceLifecycle" + - path: "{.NamespaceLifecycle}" set: false remediation: | Edit the kubernetes master config file /etc/origin/master/master-config.yaml @@ -233,14 +219,13 @@ groups: - id: 1.15 text: "Configure API server auditing - audit log file path" - audit: "grep -A5 auditConfig /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "enabled" + - path: "{.auditConfig.enabled}" compare: - op: has + op: eq value: "true" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml, update the following entry and restart the API server. @@ -257,14 +242,13 @@ groups: - id: 1.16 text: "Configure API server auditing - audit log retention" - audit: "grep -A5 auditConfig /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "maximumFileRetentionDays: 30" + - path: "{.auditConfig.maximumFileRetentionDays}" compare: - op: has - value: "maximumFileRetentionDays" - set: true + op: gte + value: 30 remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml, update the maximumFileRetentionDays entry and restart the API server. @@ -282,14 +266,13 @@ groups: - id: 1.17 text: "Configure API server auditing - audit log backup retention" - audit: "grep -A5 auditConfig /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "maximumRetainedFiles: 10" + - path: "{.auditConfig.maximumRetainedFiles}" compare: - op: has - value: "maximumRetainedFiles" - set: true + op: eq + value: "10" remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml, update the maximumRetainedFiles entry, set enabled to true and restart the API server. @@ -307,14 +290,13 @@ groups: - id: 1.18 text: "Configure audit log file size" - audit: "grep -A5 auditConfig /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "maximumFileSizeMegabytes: 30" + - path: "{.auditConfig.maximumFileSizeMegabytes}" compare: - op: has - value: "maximumFileSizeMegabytes" - set: true + op: eq + value: "30" remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml, update the maximumFileSizeMegabytes entry, set enabled to true and restart the API server. @@ -332,10 +314,10 @@ groups: - id: 1.19 text: "Verify that authorization-mode is not set to AlwaysAllow" - audit: "grep -A1 authorization-mode /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "authorization-mode" + - path: "{.kubernetesMasterConfig.apiServerArguments.authorization-mode}" set: false remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and remove the authorization-mode @@ -349,10 +331,10 @@ groups: - id: 1.20 text: "Verify that the token-auth-file flag is not set" - audit: "grep token-auth-file /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "token-auth-file" + - path: "{.kubernetesMasterConfig.apiServerArguments.token-auth-file}" set: false remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and remove the token-auth-file @@ -366,10 +348,10 @@ groups: - id: 1.21 text: "Verify the API server certificate authority" - audit: "grep -A1 kubelet-certificate-authority /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "kubelet-certificate-authority" + - path: "{.kubernetesMasterConfig.apiServerArguments.kubelet-certificat-authority}" set: false remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and remove the following @@ -383,20 +365,18 @@ groups: - id: 1.22 text: "Verify the API server client certificate and client key" - audit: "grep -A4 kubeletClientInfo /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - - flag: "keyFile" + - path: "{.kubeletClientInfo. keyFile}" compare: - op: has + op: eq value: "master.kubelet-client.key" - set: true - - flag: "certFile" + - path: "{.kubeletClientInfo.certFile}" compare: - op: has + op: eq value: "master.kubelet-client.crt" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and add the following configuration under kubeletClientInfo @@ -420,20 +400,18 @@ groups: - id: 1.25 text: "Verify that the service account key file argument is not set" - audit: "grep -A9 serviceAccountConfig /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - - flag: "privateKeyFile" + - path: "{.serviceAccountConfig.privateKeyFile}" compare: - op: has + op: eq value: "serviceaccounts.private.key" - set: true - - flag: "serviceaccounts.public.key" + - path: "{.serviceAccountConfig. publicKeyFiles}" compare: - op: has + op: eq value: "serviceaccounts.public.key" - set: true remediation: | OpenShift API server does not use the service-account-key-file argument. Even if value is set in master-config.yaml, it will not be used to verify @@ -460,20 +438,18 @@ groups: - id: 1.26 text: "Verify the certificate and key used for communication with etcd" - audit: "grep -A3 etcdClientInfo /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - - flag: "certFile" + - path: "{.etcdClientInfo.certFile}" compare: - op: has + op: eq value: "master.etcd-client.crt" - set: true - - flag: "keyFile" + - path: "{.etcdClientInfo.keyFile}" compare: - op: has + op: eq value: "master.etcd-client.key" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and set keyFile and certFile under etcdClientInfo like below. @@ -486,17 +462,16 @@ groups: - id: 1.27 text: "Verify that the ServiceAccount admission controller is enabled" - audit: "grep -A4 ServiceAccount /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: or test_items: - - flag: "ServiceAccount" + - path: "{.ServiceAccount.configuration.disable}" set: false - - flag: "disable" + - path: "{.ServiceAccount.configuration.disable}" compare: - op: has + op: eq value: "false" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and enable ServiceAccount admission control policy. @@ -510,20 +485,18 @@ groups: - id: 1.28 text: "Verify the certificate and key used to encrypt API server traffic" - audit: "grep -A7 servingInfo /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - - flag: "certFile" + - path: "{.servingInfo.certFile}" compare: - op: has + op: eq value: "master.server.crt" - set: true - - flag: "keyFile" + - path: "{.servingInfo.keyFile}" compare: - op: has + op: eq value: "master.server.key" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and set keyFile and certFile under servingInfo. @@ -539,10 +512,10 @@ groups: - id: 1.29 text: "Verify that the --client-ca-file argument is not set" - audit: "grep client-ca-file /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "clientCA: ca.crt" + - path: "{.servingInfo.clientCA}" set: false remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and set clientCA under servingInfo. @@ -559,14 +532,13 @@ groups: - id: 1.30 text: "Verify the CA used for communication with etcd" - audit: "grep -A3 etcdClientInfo /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "ca" + - path: "{.etcdClientInfo.ca}" compare: - op: has + op: eq value: "master.etcd-ca.crt" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and set ca under etcdClientInfo. @@ -583,17 +555,16 @@ groups: - id: 1.32 text: "Verify that the NodeRestriction admission controller is enabled" - audit: "grep -A4 NodeRestriction /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: or test_items: - - flag: "NodeRestriction" + - path: "{.NodeRestriction.configuration.disable}" set: false - - flag: "disable" + - path: "{.NodeRestriction.configuration.disable}" compare: - op: has + op: eq value: "false" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and enable NodeRestriction ca under etcdClientInfo. @@ -606,14 +577,10 @@ groups: - id: 1.33 text: "Configure encryption of data at rest in etcd datastore" - audit: "grep -A1 experimental-encryption-provider-config /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "experimental-encryption-provider-config:" - compare: - op: has - value: "experimental-encryption-provider-config:" - set: true + - path: "{.kubernetesMasterConfig.apiServerArguments.experimental-encryption-provider-config}" remediation: | Follow the instructions in the documentation to configure encryption. https://docs.openshift.com/container-platform/3.10/admin_guide/encrypting_data.html @@ -621,14 +588,14 @@ groups: - id: 1.34 text: "Set the encryption provider to aescbc for etcd data at rest" - audit: "grep -A1 experimental-encryption-provider-config /etc/origin/master/master-config.yaml | sed -n '2p' | awk '{ print $2 }' | xargs grep -A1 providers" + + audit_config: "grep -A1 experimental-encryption-provider-config /etc/origin/master/master-config.yaml | sed -n '2p' | awk '{ print $2 }' | xargs cat" tests: test_items: - - flag: "aescbc:" + - path: "{.providers.aescbc.experimental-encryption-provider-config}" compare: op: has - value: "aescbc:" - set: true + value: "aescbc" remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and set aescbc as the first provider in encryption provider config. See https://docs.openshift.com/container-platform/3.10/admin_guide/encrypting_data.html. @@ -636,14 +603,13 @@ groups: - id: 1.35 text: "Enable the EventRateLimit plugin" - audit: "grep -A4 EventRateLimit /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "disable" + - path: "{.admissionConfig.pluginConfig.AlwaysPullImages.configuration.disable}" compare: - op: has + op: eq value: "false" - set: true remediation: | Follow the documentation to enable the EventRateLimit plugin. https://docs.openshift.com/container-platform/3.10/architecture/additional_concepts/admission_controllers.html#admission-controllers-general-admission-rules @@ -651,16 +617,15 @@ groups: - id: 1.36 text: "Configure advanced auditing" - audit: "grep AdvancedAuditing /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: or test_items: - - flag: "AdvancedAuditing" + - path: "{.kubernetesMasterConfig.apiServerArguments.feature-gates.AdvancedAuditing}" compare: op: eq value: "true" - set: true - - flag: "AdvancedAuditing" + - path: "{.kubernetesMasterConfig.apiServerArguments.feature-gates.AdvancedAuditing}" set: false remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and enable AdvancedAuditing, @@ -696,14 +661,13 @@ groups: checks: - id: 3.1 text: "Adjust the terminated-pod-gc-threshold argument as needed" - audit: "grep terminated-pod-gc-threshold -A1 /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "terminated-pod-gc-threshold:" + - path: "{.kubernetesMasterConfig.controllerArguments.terminated-pod-gc-threshold}" compare: - op: has + op: eq value: "12500" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and enable terminated-pod-gc-threshold. @@ -722,17 +686,16 @@ groups: - id: 3.3 text: "Verify that the --use-service-account-credentials argument is set to true" - audit: "grep -A2 use-service-account-credentials /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: or test_items: - - flag: "use-service-account-credentials" + - path: "{.kubernetesMasterConfig.controllerArguments.use-service-account-credentials}" set: false - - flag: "true" + - path: "{.kubernetesMasterConfig.controllerArguments.use-service-account-credentials}" compare: - op: has + op: eq value: "true" - set: true remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and set use-service-account-credentials to true under controllerArguments section. @@ -743,45 +706,45 @@ groups:     - true scored: true - # Review 3.4 + - id: 3.4 text: "Verify that the --service-account-private-key-file argument is set as appropriate" audit: | grep -A9 serviceAccountConfig /etc/origin/master/master-config.yaml | grep privateKeyFile; grep -A2 service-account-private-key-file /etc/origin/master/master-config.yaml + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - flag: "privateKeyFile: serviceaccounts.private.key" + path: "{.serviceAccountConfig.privateKeyFile}" compare: - op: has - value: "privateKeyFile" + op: eq + value: "serviceaccounts.private.key" - flag: "service-account-private-key-file" set: false - remediation: + remediation: | Edit the Openshift master config file /etc/origin/master/master-config.yaml and remove service-account-private-key-file scored: true - # Review 3.5 + - id: 3.5 text: "Verify that the --root-ca-file argument is set as appropriate" - audit: "/bin/sh -c 'grep root-ca-file /etc/origin/master/master-config.yaml; grep -A9 serviceAccountConfig /etc/origin/master/master-config.yaml'" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: bin_op: and test_items: - - flag: "root-ca-file=/etc/origin/master/ca-bundle.crt" + - path: "{.serviceAccountConfig.root-ca-file}" compare: - op: has + op: eq value: "/etc/origin/master/ca-bundle.crt" - set: true - - flag: "masterCA" + - path: "{.serviceAccountConfig.masterCA}" compare: - op: has + op: eq value: "ca-bundle.crt" - set: true - remediation: + remediation: | Reset to OpenShift defaults OpenShift starts kube-controller-manager with - root-ca-file=/etc/origin/master/ca-bundle.crt by default.  OpenShift Advanced + root-ca-file=/etc/origin/master/ca-bundle.crt by default. OpenShift Advanced Installation creates this certificate authority and configuration without any configuration required. @@ -795,15 +758,14 @@ groups: - id: 3.7 text: "Manage certificate rotation" - audit: "grep -B3 RotateKubeletServerCertificate=true /etc/origin/master/master-config.yaml" + audit_config: "cat /etc/origin/master/master-config.yaml" tests: test_items: - - flag: "RotateKubeletServerCertificate" + - path: "{.kubeletArguments.feature-gates.RotateKubeletServerCertificate}" compare: op: eq value: "true" - set: true - remediation: + remediation: | If you decide not to enable the RotateKubeletServerCertificate feature, be sure to use the Ansible playbooks provided with the OpenShift installer to automate re-deploying certificates. @@ -822,7 +784,6 @@ groups: compare: op: bitmask value: "600" - set: true remediation: | Run the below command. @@ -835,10 +796,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -854,7 +811,6 @@ groups: compare: op: bitmask value: "600" - set: true remediation: | Run the below command on the master node. @@ -867,10 +823,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -886,7 +838,6 @@ groups: compare: op: bitmask value: "600" - set: true remediation: | Run the below command. @@ -899,10 +850,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -918,7 +865,6 @@ groups: compare: op: bitmask value: "600" - set: true remediation: | Run the below command. @@ -931,10 +877,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -944,13 +886,13 @@ groups: - id: 4.9 text: "Verify the default OpenShift Container Network Interface file permissions" audit: "stat -c permissions=%a /etc/origin/openvswitch/ /etc/cni/net.d/" + use_multiple_values: true tests: test_items: - flag: "permissions" compare: op: bitmask value: "644" - set: true remediation: | Run the below command. @@ -960,13 +902,10 @@ groups: - id: 4.10 text: "Verify the default OpenShift Container Network Interface file ownership" audit: "stat -c %U:%G /etc/origin/openvswitch/ /etc/cni/net.d/" + use_multiple_values: true tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -982,7 +921,6 @@ groups: compare: op: bitmask value: "700" - set: true remediation: | On the etcd server node, get the etcd data directory, passed as an argument --data-dir , from the below command: @@ -997,10 +935,6 @@ groups: tests: test_items: - flag: "etcd:etcd" - compare: - op: eq - value: "etcd:etcd" - set: true remediation: | Run the below command on the master node. @@ -1016,7 +950,6 @@ groups: compare: op: bitmask value: "644" - set: true remediation: | Run the below command. @@ -1029,10 +962,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -1048,7 +977,6 @@ groups: compare: op: bitmask value: "644" - set: true remediation: | Run the below command. @@ -1061,10 +989,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: "root:root" - set: true remediation: | Run the below command on the master node. @@ -1080,7 +1004,6 @@ groups: compare: op: bitmask value: "644" - set: true remediation: | Run the below command. @@ -1114,40 +1037,29 @@ groups: bin_op: and test_items: - flag: "Binary file /proc/1/environ matches" + - flag: "ETCD_CERT_FILE" compare: - op: has - value: "Binary file /proc/1/environ matches" - set: true - - flag: "ETCD_CERT_FILE=/etc/etcd/server.crt" + op: eq + value: "/etc/etcd/server.crt" + - flag: "ETCD_KEY_FILE" compare: - op: has - value: "ETCD_CERT_FILE=/etc/etcd/server.crt" - set: true - - flag: "ETCD_KEY_FILE=/etc/etcd/server.key" - compare: - op: has - value: "ETCD_KEY_FILE=/etc/etcd/server.key" - set: true + op: eq + value: "/etc/etcd/server.key" remediation: | Reset to the OpenShift default configuration. scored: true - id: 5.2 text: "Verify the default OpenShift setting for the client-cert-auth argument" - audit: "/bin/sh -c'/usr/local/bin/master-exec etcd etcd grep ETCD_CLIENT_CERT_AUTH=true /proc/1/environ; grep ETCD_CLIENT_CERT_AUTH /etc/etcd/etcd.conf'" + audit: "/bin/sh -c '/usr/local/bin/master-exec etcd etcd grep ETCD_CLIENT_CERT_AUTH=true /proc/1/environ; grep ETCD_CLIENT_CERT_AUTH /etc/etcd/etcd.conf'" tests: bin_op: and test_items: - flag: "Binary file /proc/1/environ matches" + - flag: "ETCD_CLIENT_CERT_AUTH" compare: - op: has - value: "Binary file /proc/1/environ matches" - set: true - - flag: "ETCD_CLIENT_CERT_AUTH=true" - compare: - op: has - value: "ETCD_CLIENT_CERT_AUTH=true" - set: true + op: eq + value: "true" remediation: | Reset to the OpenShift default configuration. scored: true @@ -1158,41 +1070,29 @@ groups: tests: bin_op: or test_items: - - flag: "ETCD_AUTO_TLS=false" + - flag: "ETCD_AUTO_TLS" compare: - op: has - value: "ETCD_AUTO_TLS=false" - set: true - - flag: "#ETCD_AUTO_TLS" - compare: - op: has - value: "#ETCD_AUTO_TLS" - set: true + op: eq + value: "false" remediation: | Reset to the OpenShift default configuration. scored: true - id: 5.4 text: "Verify the OpenShift default peer-cert-file and peer-key-file arguments for etcd" - audit: "/bin/sh -c'/usr/local/bin/master-exec etcd etcd grep ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt /proc/1/environ; /usr/local/bin/master-exec etcd etcd grep ETCD_PEER_KEY_FILE=/etc/etcd/peer.key /proc/1/environ; grep ETCD_PEER_CERT_FILE /etc/etcd/etcd.conf; grep ETCD_PEER_KEY_FILE /etc/etcd/etcd.conf'" + audit: "/bin/sh -c '/usr/local/bin/master-exec etcd etcd grep ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt /proc/1/environ; /usr/local/bin/master-exec etcd etcd grep ETCD_PEER_KEY_FILE=/etc/etcd/peer.key /proc/1/environ; grep ETCD_PEER_CERT_FILE /etc/etcd/etcd.conf; grep ETCD_PEER_KEY_FILE /etc/etcd/etcd.conf'" tests: bin_op: and test_items: - flag: "Binary file /proc/1/environ matches" + - flag: "ETCD_PEER_CERT_FILE" compare: - op: has - value: "Binary file /proc/1/environ matches" - set: true - - flag: "ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt" + op: eq + value: "/etc/etcd/peer.crt" + - flag: "ETCD_PEER_KEY_FILE" compare: - op: has - value: "ETCD_PEER_CERT_FILE=/etc/etcd/peer.crt" - set: true - - flag: "ETCD_PEER_KEY_FILE=/etc/etcd/peer.key" - compare: - op: has - value: "ETCD_PEER_KEY_FILE=/etc/etcd/peer.key" - set: true + op: eq + value: "/etc/etcd/peer.key" remediation: | Reset to the OpenShift default configuration. scored: true @@ -1204,15 +1104,10 @@ groups: bin_op: and test_items: - flag: "Binary file /proc/1/environ matches" + - flag: "ETCD_PEER_CLIENT_CERT_AUTH" compare: - op: has - value: "Binary file /proc/1/environ matches" - set: true - - flag: "ETCD_PEER_CLIENT_CERT_AUTH=true" - compare: - op: has - value: "ETCD_PEER_CLIENT_CERT_AUTH=true" - set: true + op: eq + value: "true" remediation: | Reset to the OpenShift default configuration. scored: true @@ -1221,18 +1116,11 @@ groups: text: "Verify the OpenShift default configuration for the peer-auto-tls argument" audit: "/bin/sh -c '/usr/local/bin/master-exec etcd etcd grep ETCD_PEER_AUTO_TLS /proc/1/environ; grep ETCD_PEER_AUTO_TLS /etc/etcd/etcd.conf'" tests: - bin_op: and test_items: - - flag: "Binary file /proc/1/environ matches" + - flag: "ETCD_PEER_AUTO_TLS" compare: - op: has - value: "Binary file /proc/1/environ matches" - set: true - - flag: "#ETCD_PEER_AUTO_TLS=false" - compare: - op: has - value: "#ETCD_PEER_AUTO_TLS=false" - set: true + op: noteq + value: "true" remediation: | Reset to the OpenShift default configuration. scored: true @@ -1253,10 +1141,6 @@ groups: tests: test_items: - flag: "issuer= /CN=etcd-signer" - compare: - op: has - value: "issuer= /CN=etcd-signer" - set: true remediation: | Reset to the OpenShift default configuration. scored: false diff --git a/cfg/rh-0.7/node.yaml b/cfg/rh-0.7/node.yaml index 62df9ce..4436047 100644 --- a/cfg/rh-0.7/node.yaml +++ b/cfg/rh-0.7/node.yaml @@ -19,17 +19,16 @@ groups: - id: 7.3 text: "Verify that the --authorization-mode argument is set to WebHook" - audit: "grep -A1 authorization-mode /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: bin_op: or test_items: - - flag: "authorization-mode" + - path: "{.kubeletArguments.authorization-mode}" set: false - - flag: "authorization-mode" + - path: "{.kubeletArguments.authorization-mode}" compare: op: has value: "Webhook" - set: true remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and remove authorization-mode under kubeletArguments in /etc/origin/node/node-config.yaml or set it to "Webhook". @@ -37,10 +36,10 @@ groups: - id: 7.4 text: "Verify the OpenShift default for the client-ca-file argument" - audit: "grep -A1 client-ca-file /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: test_items: - - flag: "client-ca-file" + - path: "{.PodManifestConfig.client-ca-file}" set: false remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and remove any configuration returned by the following: @@ -53,31 +52,32 @@ groups: - id: 7.5 text: "Verify the OpenShift default setting for the read-only-port argument" - audit: "grep -A1 read-only-port /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: bin_op: or test_items: - - flag: "read-only-port" + - path: "{.kubeletArguments.read-only-port}" set: false - - flag: "read-only-port" + - path: "{.kubeletArguments.read-only-port}" compare: - op: has + op: eq value: "0" - set: true remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and removed so that the OpenShift default is applied. scored: true - id: 7.6 text: "Adjust the streaming-connection-idle-timeout argument" - audit: "grep -A1 streaming-connection-idle-timeout /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: bin_op: or test_items: - - flag: "streaming-connection-idle-timeout" - set: false - - flag: "5m" + - path: "{.kubeletArguments.streaming-connection-idle-timeout}" set: false + - path: "{.kubeletArguments.streaming-connection-idle-timeout}" + compare: + op: eq + value: "5m" remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and set the streaming-connection-timeout value like the following in node-config.yaml. @@ -94,17 +94,16 @@ groups: - id: 7.8 text: "Verify the OpenShift default value of true for the make-iptables-util-chains argument" - audit: "grep -A1 make-iptables-util-chains /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: bin_op: or test_items: - - flag: "make-iptables-util-chains" + - path: "{.kubeletArguments.make-iptables-util-chains}" set: false - - flag: "make-iptables-util-chains" + - path: "{.kubeletArguments.make-iptables-util-chains}" compare: - op: has + op: eq value: "true" - set: true remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and reset make-iptables-util-chains to the OpenShift default value of true. @@ -112,14 +111,16 @@ groups: - id: 7.9 text: "Verify that the --keep-terminated-pod-volumes argument is set to false" - audit: "grep -A1 keep-terminated-pod-volumes /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: + bin_op: or test_items: - - flag: "keep-terminated-pod-volumes" + - path: "{.kubeletArguments.keep-terminated-pod-volumes}" + set: false + - path: "{.kubeletArguments.keep-terminated-pod-volumes}" compare: - op: has + op: eq value: "false" - set: true remediation: | Reset to the OpenShift defaults scored: true @@ -131,17 +132,16 @@ groups: - id: 7.11 text: "Set the --event-qps argument to 0" - audit: "grep -A1 event-qps /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: bin_op: or test_items: - - flag: "event-qps" + - path: "{.kubeletArguments.event-qps}" set: false - - flag: "event-qps" + - path: "{.kubeletArguments.event-qps}" compare: - op: has + op: eq value: "0" - set: true remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml set the event-qps argument to 0 in the kubeletArguments section of. @@ -149,31 +149,29 @@ groups: - id: 7.12 text: "Verify the OpenShift cert-dir flag for HTTPS traffic" - audit: "grep -A1 cert-dir /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: test_items: - - flag: "/etc/origin/node/certificates" + - path: "{.kubeletArguments.cert-dir}" compare: op: has value: "/etc/origin/node/certificates" - set: true remediation: | Reset to the OpenShift default values. scored: true - id: 7.13 text: "Verify the OpenShift default of 0 for the cadvisor-port argument" - audit: "grep -A1 cadvisor-port /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: bin_op: or test_items: - - flag: "cadvisor-port" + - path: "{.kubeletArguments.cadvisor-port}" set: false - - flag: "cadvisor-port" + - path: "{.kubeletArguments.cadvisor-port}" compare: - op: has + op: eq value: "0" - set: true remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and remove the cadvisor-port flag if it is set in the kubeletArguments section. @@ -181,28 +179,26 @@ groups: - id: 7.14 text: "Verify that the RotateKubeletClientCertificate argument is set to true" - audit: "grep -B1 RotateKubeletClientCertificate=true /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: test_items: - - flag: "RotateKubeletClientCertificate=true" + - path: "{.kubeletArguments.feature-gates}" compare: op: has - value: "true" - set: true + value: "RotateKubeletClientCertificate=true" remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and set RotateKubeletClientCertificate to true. scored: true - id: 7.15 text: "Verify that the RotateKubeletServerCertificate argument is set to true" - audit: "grep -B1 RotateKubeletServerCertificate=true /etc/origin/node/node-config.yaml" + audit_config: "cat /etc/origin/node/node-config.yaml" tests: test_items: - - flag: "RotateKubeletServerCertificate=true" + - path: "{.kubeletArguments.feature-gates}" compare: op: has - value: "true" - set: true + value: "RotateKubeletServerCertificate=true" remediation: | Edit the Openshift node config file /etc/origin/node/node-config.yaml and set RotateKubeletServerCertificate to true. scored: true @@ -220,7 +216,6 @@ groups: compare: op: bitmask value: "644" - set: true remediation: | Run the below command on each worker node. chmod 644 /etc/origin/node/node.kubeconfig @@ -232,10 +227,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: root:root - set: true remediation: | Run the below command on each worker node. chown root:root /etc/origin/node/node.kubeconfig @@ -243,32 +234,27 @@ groups: - id: 8.3 text: "Verify the kubelet service file permissions of 644" - audit: "stat -c permissions=%a $nodesvc" + audit: "stat -c permissions=%a $kubeletsvc" tests: test_items: - flag: "permissions" compare: op: bitmask value: "644" - set: true remediation: | Run the below command on each worker node. - chmod 644 $nodesvc + chmod 644 $kubeletsvc scored: true - id: 8.4 text: "Verify the kubelet service file ownership of root:root" - audit: "stat -c %U:%G $nodesvc" + audit: "stat -c %U:%G $kubeletsvc" tests: test_items: - flag: "root:root" - compare: - op: eq - value: root:root - set: true remediation: | Run the below command on each worker node. - chown root:root $nodesvc + chown root:root $kubeletsvc scored: true - id: 8.5 @@ -280,7 +266,6 @@ groups: compare: op: bitmask value: "644" - set: true remediation: | Run the below command on each worker node. chmod 644 /etc/origin/node/node.kubeconfig @@ -292,10 +277,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: root:root - set: true remediation: | Run the below command on each worker node. chown root:root /etc/origin/node/node.kubeconfig @@ -310,7 +291,6 @@ groups: compare: op: bitmask value: "644" - set: true remediation: | Run the below command on each worker node. chmod 644 /etc/origin/node/client-ca.crt @@ -322,10 +302,6 @@ groups: tests: test_items: - flag: "root:root" - compare: - op: eq - value: root:root - set: true remediation: | Run the below command on each worker node. chown root:root /etc/origin/node/client-ca.crt diff --git a/cmd/util.go b/cmd/util.go index e143a12..9f05699 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -428,6 +428,12 @@ These program names are provided in the config.yaml, section '%s.%s.bins' } func getPlatformName() string { + + openShiftVersion := getOpenShiftVersion() + if openShiftVersion != ""{ + return openShiftVersion + } + kv, err := getKubeVersion() if err != nil { glog.V(2).Info(err) @@ -446,11 +452,54 @@ func getPlatformNameFromVersion(s string) string { } func getPlatformBenchmarkVersion(platform string) string { + glog.V(3).Infof("getPlatformBenchmarkVersion platform: %s", platform) switch platform { case "eks": return "eks-1.0" case "gke": return "gke-1.0" + case "ocp-3.10": + return "rh-0.7" } return "" } + +func getOpenShiftVersion() string{ + glog.V(1).Info("Checking for oc") + _, err := exec.LookPath("oc") + + if err == nil { + cmd := exec.Command("oc", "version") + out, err := cmd.CombinedOutput() + + if err == nil { + versionRe := regexp.MustCompile(`oc v(\d+\.\d+)`) + subs := versionRe.FindStringSubmatch(string(out)) + if len(subs) > 1 { + glog.V(2).Infof("OCP output '%s' \nplatform is %s \nocp %v",string(out),getPlatformNameFromVersion(string(out)),subs[1]) + ocpBenchmarkVersion, err := getOcpValidVersion(subs[1]) + if err == nil{ + return fmt.Sprintf("ocp-%s", ocpBenchmarkVersion) + } + } + } + + } + return "" +} + +func getOcpValidVersion(ocpVer string) (string, error) { + ocpOriginal := ocpVer + + for (!isEmpty(ocpVer)) { + glog.V(3).Info(fmt.Sprintf("getOcpBenchmarkVersion check for ocp: %q \n", ocpVer)) + if ocpVer == "3.10"{ + glog.V(1).Info(fmt.Sprintf("getOcpBenchmarkVersion found valid version for ocp: %q \n", ocpVer)) + return ocpVer, nil + } + ocpVer = decrementVersion(ocpVer) + } + + glog.V(1).Info(fmt.Sprintf("getOcpBenchmarkVersion unable to find a match for: %q", ocpOriginal)) + return "", fmt.Errorf("unable to find a matching Benchmark Version match for ocp version: %s", ocpOriginal) +} diff --git a/cmd/util_test.go b/cmd/util_test.go index ffb2239..6636cc0 100644 --- a/cmd/util_test.go +++ b/cmd/util_test.go @@ -593,6 +593,13 @@ func Test_getPlatformBenchmarkVersion(t *testing.T) { }, want: "", }, + { + name: "open shift", + args: args{ + platform: "ocp-3.10", + }, + want: "rh-0.7", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -602,3 +609,30 @@ func Test_getPlatformBenchmarkVersion(t *testing.T) { }) } } + +func Test_getOcpValidVersion(t *testing.T) { + + cases := []struct { + openShiftVersion string + succeed bool + exp string + }{ + {openShiftVersion: "3.11", succeed: true, exp: "3.10"}, + {openShiftVersion: "3.10", succeed: true, exp: "3.10"}, + {openShiftVersion: "2.9", succeed: false, exp: ""}, + {openShiftVersion: "4.1", succeed: false, exp: ""}, + {openShiftVersion: "invalid", succeed: false, exp: ""}, + } + for _, c := range cases { + ocpVer,_ := getOcpValidVersion(c.openShiftVersion) + if c.succeed { + if c.exp != ocpVer { + t.Fatalf("getOcpValidVersion(%q) - Got %q expected %s", c.openShiftVersion, ocpVer, c.exp) + } + } else { + if len(ocpVer) > 0 { + t.Fatalf("getOcpValidVersion(%q) - Expected empty string but Got %s", c.openShiftVersion, ocpVer) + } + } + } +}