From 47c5661034268fe2bcf9be52774816062e7a5857 Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Sat, 16 Nov 2019 09:39:47 -0500 Subject: [PATCH] Fixes issue #439: Adds integration testing using KIND (#520) * Fixes issue #439: Adds integration testing using KIND * try integration tests * started using ticker and timeouts * trying built container image * adds load image into KIND * adds comparison * fixes as per PR review --- .travis.yml | 1 + go.mod | 19 +- go.sum | 139 ++++++- integration/docker.go | 61 +++ integration/integration.go | 180 +++++++++ integration/integration_test.go | 70 ++++ integration/testdata/add-tls-kind.yaml | 13 + integration/testdata/job-master.data | 426 ++++++++++++++++++++ integration/testdata/job-node.data | 92 +++++ integration/testdata/job.data | 518 +++++++++++++++++++++++++ makefile | 3 + 11 files changed, 1502 insertions(+), 20 deletions(-) create mode 100644 integration/docker.go create mode 100644 integration/integration.go create mode 100644 integration/integration_test.go create mode 100644 integration/testdata/add-tls-kind.yaml create mode 100644 integration/testdata/job-master.data create mode 100644 integration/testdata/job-node.data create mode 100644 integration/testdata/job.data diff --git a/.travis.yml b/.travis.yml index 9522ff9..e10585a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ script: - test -d cfg - test -f kube-bench - make tests + - make integration-tests after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/go.mod b/go.mod index 4081298..181954e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aquasecurity/kube-bench -go 1.12 +go 1.13 require ( github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect @@ -8,7 +8,7 @@ require ( github.com/fatih/color v1.5.0 github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/imdario/mergo v0.3.5 // indirect github.com/jinzhu/gorm v0.0.0-20160404144928-5174cc5c242a github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d // indirect github.com/jinzhu/now v1.0.1 // indirect @@ -16,10 +16,17 @@ require ( github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597 // indirect github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c // indirect github.com/mattn/go-sqlite3 v1.10.0 // indirect - github.com/onsi/ginkgo v1.7.0 - github.com/spf13/cobra v0.0.1 + github.com/onsi/ginkgo v1.10.1 + github.com/pkg/errors v0.8.1 + github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.3.0 - gopkg.in/yaml.v2 v2.2.2 - k8s.io/client-go v10.0.0+incompatible + golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect + google.golang.org/appengine v1.5.0 // indirect + gopkg.in/yaml.v2 v2.2.4 + k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b + k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d + k8s.io/client-go v11.0.0+incompatible + k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 // indirect + sigs.k8s.io/kind v0.5.1 ) diff --git a/go.sum b/go.sum index eade2f9..18c8660 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,14 @@ cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU= cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -20,6 +27,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -30,27 +38,47 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= +github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.5.0 h1:vBh+kQp8lg9XPr56u1CPrWjFXtdphMoGWVHr9/1c+A0= github.com/fatih/color v1.5.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -59,9 +87,16 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= +github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -73,6 +108,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/gorm v0.0.0-20160404144928-5174cc5c242a h1:pfPxlCVlKqBRqHpyCxOIKhhB4ERpz02iadDpRVevLm4= @@ -82,21 +119,30 @@ github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLl github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd h1:2RDaVc4/izhWyAvYxNm8c9saSyCDIxefNwOcqaH7pcU= github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481 h1:IaSjLMT6WvkoZZjspGxy3rdaTEmWLoRm49WbtVUi9sA= +github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597 h1:hGizH4aMDFFt1iOA4HNKC13lqIBoCyxIjWcAnWIy7aU= github.com/mattn/go-colorable v0.0.0-20170210172801-5411d3eea597/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.0-20170307163044-57fdcb988a5c h1:AHfQR/s6GNi92TOh+kfGworqDvTxj2rMsS+Hca87nck= @@ -106,18 +152,32 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -136,23 +196,30 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.1 h1:zZh3X5aZbdnoj+4XkaBxKfhO4ot82icYdhhREIAXIj8= -github.com/spf13/cobra v0.0.1/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -167,13 +234,15 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -183,37 +252,54 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190621203818-d432491b9138 h1:t8BZD9RDjkm9/h7yYN6kE8oaeov5r9aztkB7zKA5Tkg= +golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -226,15 +312,40 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34= -k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b h1:aBGgKJUM9Hk/3AE8WaZIApnTxG35kbuQba2w+SXqezo= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d h1:Jmdtdt1ZnoGfWWIIik61Z7nKYgO3J+swQJtPYsP9wHA= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= +k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c= +k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 h1:5sW+fEHvlJI3Ngolx30CmubFulwH28DhKjGf70Xmtco= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= +k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6 h1:p0Ai3qVtkbCG/Af26dBmU0E1W58NID3hSSh7cMyylpM= +k8s.io/utils v0.0.0-20191114200735-6ca3b61696b6/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +sigs.k8s.io/kind v0.5.1 h1:BYnHEJ9DC+0Yjlyyehqd3xnKtEmFdLKU8QxqOqvQzdw= +sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296 h1:iQaIG5Dq+3qSiaFrJ/l/0MjjxKmdwyVNpKRYJwUe/+0= +sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/integration/docker.go b/integration/docker.go new file mode 100644 index 0000000..fa7fddd --- /dev/null +++ b/integration/docker.go @@ -0,0 +1,61 @@ +package integration + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" + + "sigs.k8s.io/kind/pkg/cluster" + clusternodes "sigs.k8s.io/kind/pkg/cluster/nodes" + "sigs.k8s.io/kind/pkg/container/docker" + "sigs.k8s.io/kind/pkg/fs" + "sigs.k8s.io/kind/pkg/util/concurrent" +) + +func loadImageFromDocker(imageName string, kindCtx *cluster.Context) error { + + // Check that the image exists locally and gets its ID, if not return error + _, err := docker.ImageID(imageName) + if err != nil { + return errors.Errorf("Image: %q not present locally", imageName) + } + + selectedNodes, err := kindCtx.ListInternalNodes() + if err != nil { + return err + } + + // Save the image into a tar + dir, err := fs.TempDir("", "image-tar") + if err != nil { + return errors.Wrap(err, "failed to create tempdir") + } + defer os.RemoveAll(dir) + imageTarPath := filepath.Join(dir, "image.tar") + + err = docker.Save(imageName, imageTarPath) + if err != nil { + return err + } + + // Load the image on the selected nodes + fns := []func() error{} + for _, selectedNode := range selectedNodes { + selectedNode := selectedNode // capture loop variable + fns = append(fns, func() error { + return loadImage(imageTarPath, &selectedNode) + }) + } + return concurrent.UntilError(fns) +} + +// loads an image tarball onto a node +func loadImage(imageTarName string, node *clusternodes.Node) error { + f, err := os.Open(imageTarName) + if err != nil { + return errors.Wrap(err, "failed to open image") + } + defer f.Close() + return node.LoadImageArchive(f) +} \ No newline at end of file diff --git a/integration/integration.go b/integration/integration.go new file mode 100644 index 0000000..90dc5ea --- /dev/null +++ b/integration/integration.go @@ -0,0 +1,180 @@ +package integration + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "strings" + "time" + + batchv1 "k8s.io/api/batch/v1" + apiv1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + yaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/kind/pkg/cluster" + "sigs.k8s.io/kind/pkg/cluster/create" +) + +func runWithKind(clusterName, kindCfg, kubebenchYAML, kubebenchImg string, timeout, ticker time.Duration) (string, error) { + options := create.WithConfigFile(kindCfg) + ctx := cluster.NewContext(clusterName) + if err := ctx.Create(options); err != nil { + return "", err + } + defer func() { + ctx.Delete() + }() + + clientset, err := getClientSet(ctx.KubeConfigPath()) + if err != nil { + return "", err + } + + jobYAML, err := ioutil.ReadFile(kubebenchYAML) + if err != nil { + return "", err + } + + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML)) + if err != nil { + return "", err + } + + job := &batchv1.Job{} + if err := decoder.Decode(job); err != nil { + return "", err + } + job.Spec.Template.Spec.Containers[0].Image = kubebenchImg + + if err := loadImageFromDocker(kubebenchImg, ctx); err != nil { + return "", err + } + + _, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job) + if err != nil { + return "", err + } + + clientset, err = getClientSet(ctx.KubeConfigPath()) + if err != nil { + return "", err + } + + p, err := findPodForJob(clientset, "kube-bench", timeout, ticker) + if err != nil { + return "", err + } + + output := getPodLogs(clientset, p) + return output, nil +} + +func getClientSet(configPath string) (*kubernetes.Clientset, error) { + config, err := clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer time.Duration) (*apiv1.Pod, error) { + timeout := time.After(tout) + failedPods := make(map[string]struct{}) + for { + podfailed: + select { + case <-timeout: + return nil, fmt.Errorf("podList - time out: no Pod with %s", name) + default: + pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + fmt.Printf("Found (%d) pods\n", len(pods.Items)) + for _, cp := range pods.Items { + if _, found := failedPods[cp.Name]; found { + continue + } + + if strings.HasPrefix(cp.Name, name) { + fmt.Printf("pod (%s) - %#v\n", cp.Name, cp.Status.Phase) + if cp.Status.Phase == apiv1.PodSucceeded { + return &cp, nil + } + + if cp.Status.Phase == apiv1.PodFailed { + fmt.Printf("pod (%s) - %s - retrying...\n", cp.Name, cp.Status.Phase) + failedPods[cp.Name] = struct{}{} + break podfailed + } + + // Pod still working + // Wait and try again... + ticker := time.NewTicker(timer) + for { + fmt.Println("using ticker and an timer...") + select { + case <-ticker.C: + thePod, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).Get(cp.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + fmt.Printf("thePod (%s) - status:%#v \n", thePod.Name, thePod.Status.Phase) + if thePod.Status.Phase == apiv1.PodSucceeded { + return thePod, nil + } + + if thePod.Status.Phase == apiv1.PodFailed { + fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Phase) + failedPods[thePod.Name] = struct{}{} + ticker.Stop() + break podfailed + } + + if thePod.Status.Phase == apiv1.PodPending && strings.Contains(thePod.Status.Reason, "Failed") { + fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Reason) + failedPods[thePod.Name] = struct{}{} + ticker.Stop() + break podfailed + } + + case <-timeout: + ticker.Stop() + return nil, fmt.Errorf("getPod time out: no Pod with %s", name) + } + } + } + } + } + time.Sleep(1 * time.Second) + } + + return nil, fmt.Errorf("no Pod with %s", name) +} + +func getPodLogs(clientset *kubernetes.Clientset, pod *apiv1.Pod) string { + podLogOpts := corev1.PodLogOptions{} + req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + podLogs, err := req.Stream() + if err != nil { + return "getPodLogs - error in opening stream" + } + defer podLogs.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return "getPodLogs - error in copy information from podLogs to buf" + } + + return buf.String() +} diff --git a/integration/integration_test.go b/integration/integration_test.go new file mode 100644 index 0000000..b5c0352 --- /dev/null +++ b/integration/integration_test.go @@ -0,0 +1,70 @@ +// +build integration + +package integration + +import ( + "flag" + "fmt" + "io/ioutil" + "strings" + "testing" + "time" +) + +var kubebenchImg = flag.String("kubebenchImg", "aquasec/kube-bench:latest", "kube-bench image used as part of this test") + +func TestRunWithKind(t *testing.T) { + flag.Parse() + fmt.Printf("kube-bench Container Image: %s\n", *kubebenchImg) + timeout := time.Duration(10 * time.Minute) + ticker := time.Duration(2 * time.Second) + + mustMatch := func(expFname, data string) { + d, err := ioutil.ReadFile(expFname) + if err != nil { + t.Error(err) + } + expectedData := strings.TrimSpace(string(d)) + data = strings.TrimSpace(data) + if expectedData != data { + t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, data) + } + } + + cases := []struct { + TestName string + KindCfg string + KubebenchYAML string + ExpectedFile string + ExpectError bool + }{ + { + TestName: "job", + KindCfg: "./testdata/add-tls-kind.yaml", + KubebenchYAML: "../job.yaml", + ExpectedFile: "./testdata/job.data", + }, + { + TestName: "job-node", + KindCfg: "./testdata/add-tls-kind.yaml", + KubebenchYAML: "../job-node.yaml", + ExpectedFile: "./testdata/job-node.data", + }, + { + TestName: "job-master", + KindCfg: "./testdata/add-tls-kind.yaml", + KubebenchYAML: "../job-master.yaml", + ExpectedFile: "./testdata/job-master.data", + }, + } + for _, c := range cases { + t.Run(c.TestName, func(t *testing.T) { + data, err := runWithKind(c.TestName, c.KindCfg, c.KubebenchYAML, *kubebenchImg, timeout, ticker) + if err != nil { + t.Fatalf("unexpected error: %v", err) + return + } + mustMatch(c.ExpectedFile, data) + }) + } +} diff --git a/integration/testdata/add-tls-kind.yaml b/integration/testdata/add-tls-kind.yaml new file mode 100644 index 0000000..3e38500 --- /dev/null +++ b/integration/testdata/add-tls-kind.yaml @@ -0,0 +1,13 @@ +apiVersion: kind.sigs.k8s.io/v1alpha3 +kind: Cluster +networking: + apiServerAddress: "0.0.0.0" + +kubeadmConfigPatchesJson6902: +- group: kubelet.config.k8s.io + version: v1beta1 + kind: KubeletConfiguration + patch: | + - op: add + path: /tlsCipherSuites + value: ["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"] \ No newline at end of file diff --git a/integration/testdata/job-master.data b/integration/testdata/job-master.data new file mode 100644 index 0000000..1209fad --- /dev/null +++ b/integration/testdata/job-master.data @@ -0,0 +1,426 @@ +[INFO] 1 Master Node Security Configuration +[INFO] 1.1 API Server +[WARN] 1.1.1 Ensure that the --anonymous-auth argument is set to false (Not Scored) +[PASS] 1.1.2 Ensure that the --basic-auth-file argument is not set (Scored) +[PASS] 1.1.3 Ensure that the --insecure-allow-any-token argument is not set (Not Scored) +[PASS] 1.1.4 Ensure that the --kubelet-https argument is set to true (Scored) +[PASS] 1.1.5 Ensure that the --insecure-bind-address argument is not set (Scored) +[FAIL] 1.1.6 Ensure that the --insecure-port argument is set to 0 (Scored) +[PASS] 1.1.7 Ensure that the --secure-port argument is not set to 0 (Scored) +[FAIL] 1.1.8 Ensure that the --profiling argument is set to false (Scored) +[FAIL] 1.1.9 Ensure that the --repair-malformed-updates argument is set to false (Scored) +[PASS] 1.1.10 Ensure that the admission control plugin AlwaysAdmit is not set (Scored) +[FAIL] 1.1.11 Ensure that the admission control plugin AlwaysPullImages is set (Scored) +[INFO] 1.1.12 [DEPRECATED] Ensure that the admission control plugin DenyEscalatingExec is set (Not Scored) +[WARN] 1.1.13 Ensure that the admission control plugin SecurityContextDeny is set (Not Scored) +[PASS] 1.1.14 Ensure that the admission control plugin NamespaceLifecycle is set (Scored) +[FAIL] 1.1.15 Ensure that the --audit-log-path argument is set as appropriate (Scored) +[FAIL] 1.1.16 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Scored) +[FAIL] 1.1.17 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Scored) +[FAIL] 1.1.18 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Scored) +[FAIL] 1.1.19 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored) +[PASS] 1.1.20 Ensure that the --token-auth-file parameter is not set (Scored) +[FAIL] 1.1.21 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Scored) +[FAIL] 1.1.22 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Scored) +[PASS] 1.1.23 Ensure that the --service-account-lookup argument is set to true (Scored) +[FAIL] 1.1.24 Ensure that the admission control plugin PodSecurityPolicy is set (Scored) +[FAIL] 1.1.25 Ensure that the --service-account-key-file argument is set as appropriate (Scored) +[FAIL] 1.1.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Scored) +[PASS] 1.1.27 Ensure that the admission control plugin ServiceAccount is set(Scored) +[FAIL] 1.1.28 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored) +[FAIL] 1.1.29 Ensure that the --client-ca-file argument is set as appropriate (Scored) +[WARN] 1.1.30 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Not Scored) +[FAIL] 1.1.31 Ensure that the --etcd-cafile argument is set as appropriate (Scored) +[FAIL] 1.1.32 Ensure that the --authorization-mode argument is set to Node (Scored) +[FAIL] 1.1.33 Ensure that the admission control plugin NodeRestriction is set (Scored) +[FAIL] 1.1.34 Ensure that the --encryption-provider-config argument is set as appropriate (Scored) +[WARN] 1.1.35 Ensure that the encryption provider is set to aescbc (Scored) +[FAIL] 1.1.36 Ensure that the admission control plugin EventRateLimit is set (Scored) +[PASS] 1.1.37a Ensure that the AdvancedAuditing argument is not set to false (Scored) +[FAIL] 1.1.37b Ensure that the AdvancedAuditing argument is not set to false (Scored) +[PASS] 1.1.38 Ensure that the --request-timeout argument is set as appropriate (Scored) +[FAIL] 1.1.39 Ensure that the --authorization-mode argument includes RBAC (Scored) +[INFO] 1.2 Scheduler +[FAIL] 1.2.1 Ensure that the --profiling argument is set to false (Scored) +[PASS] 1.2.2 Ensure that the --address argument is set to 127.0.0.1 (Scored) +[INFO] 1.3 Controller Manager +[FAIL] 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Scored) +[FAIL] 1.3.2 Ensure that the --profiling argument is set to false (Scored) +[FAIL] 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Scored) +[FAIL] 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Scored) +[FAIL] 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Scored) +[FAIL] 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Scored) +[PASS] 1.3.7 Ensure that the --address argument is set to 127.0.0.1 (Scored) +[INFO] 1.4 Configuration Files +[PASS] 1.4.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.2 Ensure that the API server pod specification file ownership is set to root:root (Scored) +[PASS] 1.4.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.4 Ensure that the controller manager pod specification file ownership is set to root:root (Scored) +[PASS] 1.4.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.6 Ensure that the scheduler pod specification file ownership is set to root:root (Scored) +[PASS] 1.4.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.8 Ensure that the etcd pod specification file ownership is set to root:root (Scored) +[WARN] 1.4.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Not Scored) +[WARN] 1.4.10 Ensure that the Container Network Interface file ownership is set to root:root (Not Scored) +[FAIL] 1.4.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Scored) +[FAIL] 1.4.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Scored) +[PASS] 1.4.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.14 Ensure that the admin.conf file ownership is set to root:root (Scored) +[PASS] 1.4.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.16 Ensure that the scheduler.conf file ownership is set to root:root (Scored) +[PASS] 1.4.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.18 Ensure that the controller-manager.conf file ownership is set to root:root (Scored) +[WARN] 1.4.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Scored) +[WARN] 1.4.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Scored) +[WARN] 1.4.21 Ensure that the Kubernetes PKI key file permissions are set to 600 or more restrictive (Scored) +[INFO] 1.5 etcd +[FAIL] 1.5.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored) +[FAIL] 1.5.2 Ensure that the --client-cert-auth argument is set to true (Scored) +[PASS] 1.5.3 Ensure that the --auto-tls argument is not set to true (Scored) +[FAIL] 1.5.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Scored) +[FAIL] 1.5.5 Ensure that the --peer-client-cert-auth argument is set to true (Scored) +[PASS] 1.5.6 Ensure that the --peer-auto-tls argument is not set to true (Scored) +[WARN] 1.5.7 Ensure that a unique Certificate Authority is used for etcd (Not Scored) +[INFO] 1.6 General Security Primitives +[WARN] 1.6.1 Ensure that the cluster-admin role is only used where required (Not Scored) +[WARN] 1.6.2 Create administrative boundaries between resources using namespaces (Not Scored) +[WARN] 1.6.3 Create network segmentation using Network Policies (Not Scored) +[WARN] 1.6.4 Ensure that the seccomp profile is set to docker/default in your pod definitions (Not Scored) +[WARN] 1.6.5 Apply Security Context to Your Pods and Containers (Not Scored) +[WARN] 1.6.6 Configure Image Provenance using ImagePolicyWebhook admission controller (Not Scored) +[WARN] 1.6.7 Configure Network policies as appropriate (Not Scored) +[WARN] 1.6.8 Place compensating controls in the form of PSP and RBAC for privileged containers usage (Not Scored) +[INFO] 1.7 PodSecurityPolicies +[WARN] 1.7.1 Do not admit privileged containers (Not Scored) +[WARN] 1.7.2 Do not admit containers wishing to share the host process ID namespace (Not Scored) +[WARN] 1.7.3 Do not admit containers wishing to share the host IPC namespace (Not Scored) +[WARN] 1.7.4 Do not admit containers wishing to share the host network namespace (Not Scored) +[WARN] 1.7.5 Do not admit containers with allowPrivilegeEscalation (Not Scored) +[WARN] 1.7.6 Do not admit root containers (Not Scored) +[WARN] 1.7.7 Do not admit containers with dangerous capabilities (Not Scored) + +== Remediations == +1.1.1 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--anonymous-auth=false + +1.1.6 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +apiserver.yaml on the master node and set the below parameter. +--insecure-port=0 + +1.1.8 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--profiling=false + +1.1.9 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--repair-malformed-updates=false + +1.1.11 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins to +include AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +1.1.13 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins parameter to +include SecurityContextDeny. +--enable-admission-plugins=...,SecurityContextDeny,... + +1.1.15 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-path parameter to a suitable +path and file where you would like audit logs to be written, for example: +--audit-log-path=/var/log/apiserver/audit.log + +1.1.16 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-maxage parameter to 30 or +as an appropriate number of days: --audit-log-maxage=30 + +1.1.17 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-maxbackup parameter to 10 +or to an appropriate value. +--audit-log-maxbackup=10 + +1.1.18 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-maxsize parameter to an +appropriate size in MB. For example, to set it as 100 MB: +--audit-log-maxsize=100 + +1.1.19 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --authorization-mode parameter to +values other than AlwaysAllow. One such example could be as below. +--authorization-mode=RBAC + +1.1.21 Follow the Kubernetes documentation and setup the TLS connection between the +apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the --kubelet-certificate-authority +parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +1.1.22 Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +1.1.24 Follow the documentation and create Pod Security Policy objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins parameter to a +value that includes PodSecurityPolicy : +--enable-admission-plugins=...,PodSecurityPolicy,... +Then restart the API Server. + +1.1.25 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --service-account-key-file parameter +to the public key file for service accounts: +--service-account-key-file= + +1.1.26 Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and etcd. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the etcd +certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +1.1.28 Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the TLS certificate and private key file +parameters. +--tls-cert-file= +--tls-private-key-file= + +1.1.29 Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the client certificate authority file. +--client-ca-file= + +1.1.30 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + +1.1.31 Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and etcd. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the etcd +certificate authority file parameter. +--etcd-cafile= + +1.1.32 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --authorization-mode parameter to a +value that includes Node. +--authorization-mode=Node,RBAC + +1.1.33 Follow the Kubernetes documentation and configure NodeRestriction plug-in on +kubelets. Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +1.1.34 [Manual test] +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml on the +master node and set the --encryption-provider-config parameter +to the path of that file: +--encryption-provider-config= + +1.1.35 [Manual test] +Follow the Kubernetes documentation and configure a EncryptionConfig file. In this file, +choose aescbc as the encryption provider. +For example, +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - aescbc: + keys: + - name: key1 + secret: <32-byte base64-encoded secret> + +1.1.36 Follow the Kubernetes documentation and set the desired limits in a +configuration file. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +1.1.37b Follow the Kubernetes documentation and set the desired audit policy in the +/etc/kubernetes/audit-policy.yaml file. Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--audit-policy-file=/etc/kubernetes/audit-policy.yaml + +1.1.39 Edit the API server pod specification file kube-apiserver on the master node and set the --authorization-mode parameter to a value that includes RBAC, for example: --authorization-mode=Node,RBAC + +1.2.1 Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +file on the master node and set the below parameter. +--profiling=false + +1.3.1 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the --terminated-pod-gc-threshold to an appropriate threshold, for example: +--terminated-pod-gc-threshold=10 + +1.3.2 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the below parameter. +--profiling=false + +1.3.3 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node to set the below parameter. +--use-service-account-credentials=true + +1.3.4 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the --service-account-private- +key-file parameter to the private key file for service accounts. +--service-account-private-key-file= + +1.3.5 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the --root-ca-file parameter to +the certificate bundle file. +--root-ca-file= + +1.3.6 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +controller-manager.yaml on the master node and set the --feature-gates parameter to +include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +1.4.9 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, +chmod 644 + +1.4.10 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, +chown root:root + +1.4.11 On the etcd server node, get the etcd data directory, passed as an argument --data-dir , +from the below command: +ps -ef | grep etcd +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +1.4.12 On the etcd server node, get the etcd data directory, passed as an argument --data-dir , +from the below command: +ps -ef | grep etcd +Run the below command (based on the etcd data directory found above). For example, +chown etcd:etcd /var/lib/etcd + +1.4.19 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, chown -R root:root /etc/kubernetes/pki/ + +1.4.20 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, chmod -R 644 /etc/kubernetes/pki/*.crt + +1.4.21 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, chmod -R 600 /etc/kubernetes/pki/*.key + +1.5.1 Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--ca-file= +--key-file= + +1.5.2 Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +1.5.4 Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +1.5.5 Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +1.5.7 [Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +1.6.1 [Manual test] +Remove any unneeded clusterrolebindings : +kubectl delete clusterrolebinding [name] + +1.6.2 [Manual test] +Follow the documentation and create namespaces for objects in your deployment as you +need them. + +1.6.3 [Manual test] +Follow the documentation and create NetworkPolicy objects as you need them. + +1.6.4 [Manual test] +Seccomp is an alpha feature currently. By default, all alpha features are disabled. So, you +would need to enable alpha features in the apiserver by passing "--feature- +gates=AllAlpha=true" argument. +Edit the /etc/kubernetes/manifests/kube-apiserver.yaml file on the master node and set the KUBE_API_ARGS +parameter to "--feature-gates=AllAlpha=true" +KUBE_API_ARGS="--feature-gates=AllAlpha=true" +Based on your system, restart the kube-apiserver service. For example: +systemctl restart kube-apiserver.service +Use annotations to enable the docker/default seccomp profile in your pod definitions. An +example is as below: +apiVersion: v1 +kind: Pod +metadata: + name: trustworthy-pod + annotations: + seccomp.security.alpha.kubernetes.io/pod: docker/default +spec: + containers: + - name: trustworthy-container + image: sotrustworthy:latest + +1.6.5 [Manual test] +Follow the Kubernetes documentation and apply security contexts to your pods. For a +suggested list of security contexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +1.6.6 [Manual test] +Follow the Kubernetes documentation and setup image provenance. + +1.6.7 [Manual test] +Follow the Kubernetes documentation and setup network policies as appropriate. +For example, you could create a "default" isolation policy for a Namespace by creating a +NetworkPolicy that selects all pods but does not allow any traffic: +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny +spec: + podSelector: + +1.6.8 [Manual test] +Follow Kubernetes documentation and setup PSP and RBAC authorization for your cluster. + +1.7.1 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.privileged field is omitted or set to false. + +1.7.2 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.hostPID field is omitted or set to false. + +1.7.3 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.hostIPC field is omitted or set to false. + +1.7.4 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.hostNetwork field is omitted or set to false. + +1.7.5 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.allowPrivilegeEscalation field is omitted or set to false. + +1.7.6 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.runAsUser.rule is set to either MustRunAsNonRoot or MustRunAs with the range of UIDs not including 0. + +1.7.7 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.requiredDropCapabilities is set to include either NET_RAW or ALL. + + +== Summary == +30 checks PASS +36 checks FAIL +25 checks WARN +1 checks INFO \ No newline at end of file diff --git a/integration/testdata/job-node.data b/integration/testdata/job-node.data new file mode 100644 index 0000000..fb449ac --- /dev/null +++ b/integration/testdata/job-node.data @@ -0,0 +1,92 @@ + [INFO] 2 Worker Node Security Configuration +[INFO] 2.1 Kubelet +[PASS] 2.1.1 Ensure that the --anonymous-auth argument is set to false (Scored) +[PASS] 2.1.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored) +[PASS] 2.1.3 Ensure that the --client-ca-file argument is set as appropriate (Scored) +[FAIL] 2.1.4 Ensure that the --read-only-port argument is set to 0 (Scored) +[PASS] 2.1.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Scored) +[FAIL] 2.1.6 Ensure that the --protect-kernel-defaults argument is set to true (Scored) +[PASS] 2.1.7 Ensure that the --make-iptables-util-chains argument is set to true (Scored) +[PASS] 2.1.8 Ensure that the --hostname-override argument is not set (Scored) +[FAIL] 2.1.9 Ensure that the --event-qps argument is set to 0 (Scored) +[FAIL] 2.1.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored) +[INFO] 2.1.11 [DEPRECATED] Ensure that the --cadvisor-port argument is set to 0 +[PASS] 2.1.12 Ensure that the --rotate-certificates argument is not set to false (Scored) +[FAIL] 2.1.13 Ensure that the RotateKubeletServerCertificate argument is set to true (Scored) +[PASS] 2.1.14 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Not Scored) +[INFO] 2.2 Configuration Files +[PASS] 2.2.1 Ensure that the kubelet.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 2.2.2 Ensure that the kubelet.conf file ownership is set to root:root (Scored) +[PASS] 2.2.3 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Scored) +[PASS] 2.2.4 Ensure that the kubelet service file ownership is set to root:root (Scored) +[FAIL] 2.2.5 Ensure that the proxy kubeconfig file permissions are set to 644 or more restrictive (Scored) +[FAIL] 2.2.6 Ensure that the proxy kubeconfig file ownership is set to root:root (Scored) +[WARN] 2.2.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Scored) +[PASS] 2.2.8 Ensure that the client certificate authorities file ownership is set to root:root (Scored) +[PASS] 2.2.9 Ensure that the kubelet configuration file ownership is set to root:root (Scored) +[PASS] 2.2.10 Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Scored) + +== Remediations == +2.1.4 If using a Kubelet config file, edit the file to set readOnlyPort to 0 . +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.6 If using a Kubelet config file, edit the file to set protectKernelDefaults: true . +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.9 If using a Kubelet config file, edit the file to set eventRecordQPS: 0 . +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--event-qps=0 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.10 If using a Kubelet config file, edit the file to set tlsCertFile to the location of the certificate +file to use to identify this Kubelet, and tlsPrivateKeyFile to the location of the +corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +file= +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.13 Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.2.5 Run the below command (based on the file location on your system) on the each worker +node. For example, +chmod 644 /etc/kubernetes/proxy.conf + +2.2.6 Run the below command (based on the file location on your system) on the each worker +node. For example, +chown root:root /etc/kubernetes/proxy.conf + +2.2.7 Run the following command to modify the file permissions of the --client-ca-file +chmod 644 + + +== Summary == +15 checks PASS +7 checks FAIL +1 checks WARN +1 checks INFO \ No newline at end of file diff --git a/integration/testdata/job.data b/integration/testdata/job.data new file mode 100644 index 0000000..a1bb8cc --- /dev/null +++ b/integration/testdata/job.data @@ -0,0 +1,518 @@ +[INFO] 1 Master Node Security Configuration +[INFO] 1.1 API Server +[WARN] 1.1.1 Ensure that the --anonymous-auth argument is set to false (Not Scored) +[PASS] 1.1.2 Ensure that the --basic-auth-file argument is not set (Scored) +[PASS] 1.1.3 Ensure that the --insecure-allow-any-token argument is not set (Not Scored) +[PASS] 1.1.4 Ensure that the --kubelet-https argument is set to true (Scored) +[PASS] 1.1.5 Ensure that the --insecure-bind-address argument is not set (Scored) +[FAIL] 1.1.6 Ensure that the --insecure-port argument is set to 0 (Scored) +[PASS] 1.1.7 Ensure that the --secure-port argument is not set to 0 (Scored) +[FAIL] 1.1.8 Ensure that the --profiling argument is set to false (Scored) +[FAIL] 1.1.9 Ensure that the --repair-malformed-updates argument is set to false (Scored) +[PASS] 1.1.10 Ensure that the admission control plugin AlwaysAdmit is not set (Scored) +[FAIL] 1.1.11 Ensure that the admission control plugin AlwaysPullImages is set (Scored) +[INFO] 1.1.12 [DEPRECATED] Ensure that the admission control plugin DenyEscalatingExec is set (Not Scored) +[WARN] 1.1.13 Ensure that the admission control plugin SecurityContextDeny is set (Not Scored) +[PASS] 1.1.14 Ensure that the admission control plugin NamespaceLifecycle is set (Scored) +[FAIL] 1.1.15 Ensure that the --audit-log-path argument is set as appropriate (Scored) +[FAIL] 1.1.16 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Scored) +[FAIL] 1.1.17 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Scored) +[FAIL] 1.1.18 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Scored) +[FAIL] 1.1.19 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored) +[PASS] 1.1.20 Ensure that the --token-auth-file parameter is not set (Scored) +[FAIL] 1.1.21 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Scored) +[FAIL] 1.1.22 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Scored) +[PASS] 1.1.23 Ensure that the --service-account-lookup argument is set to true (Scored) +[FAIL] 1.1.24 Ensure that the admission control plugin PodSecurityPolicy is set (Scored) +[FAIL] 1.1.25 Ensure that the --service-account-key-file argument is set as appropriate (Scored) +[FAIL] 1.1.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Scored) +[PASS] 1.1.27 Ensure that the admission control plugin ServiceAccount is set(Scored) +[FAIL] 1.1.28 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored) +[FAIL] 1.1.29 Ensure that the --client-ca-file argument is set as appropriate (Scored) +[WARN] 1.1.30 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Not Scored) +[FAIL] 1.1.31 Ensure that the --etcd-cafile argument is set as appropriate (Scored) +[FAIL] 1.1.32 Ensure that the --authorization-mode argument is set to Node (Scored) +[FAIL] 1.1.33 Ensure that the admission control plugin NodeRestriction is set (Scored) +[FAIL] 1.1.34 Ensure that the --encryption-provider-config argument is set as appropriate (Scored) +[WARN] 1.1.35 Ensure that the encryption provider is set to aescbc (Scored) +[FAIL] 1.1.36 Ensure that the admission control plugin EventRateLimit is set (Scored) +[PASS] 1.1.37a Ensure that the AdvancedAuditing argument is not set to false (Scored) +[FAIL] 1.1.37b Ensure that the AdvancedAuditing argument is not set to false (Scored) +[PASS] 1.1.38 Ensure that the --request-timeout argument is set as appropriate (Scored) +[FAIL] 1.1.39 Ensure that the --authorization-mode argument includes RBAC (Scored) +[INFO] 1.2 Scheduler +[FAIL] 1.2.1 Ensure that the --profiling argument is set to false (Scored) +[PASS] 1.2.2 Ensure that the --address argument is set to 127.0.0.1 (Scored) +[INFO] 1.3 Controller Manager +[FAIL] 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Scored) +[FAIL] 1.3.2 Ensure that the --profiling argument is set to false (Scored) +[FAIL] 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Scored) +[FAIL] 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Scored) +[FAIL] 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Scored) +[FAIL] 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Scored) +[PASS] 1.3.7 Ensure that the --address argument is set to 127.0.0.1 (Scored) +[INFO] 1.4 Configuration Files +[PASS] 1.4.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.2 Ensure that the API server pod specification file ownership is set to root:root (Scored) +[PASS] 1.4.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.4 Ensure that the controller manager pod specification file ownership is set to root:root (Scored) +[PASS] 1.4.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.6 Ensure that the scheduler pod specification file ownership is set to root:root (Scored) +[PASS] 1.4.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.8 Ensure that the etcd pod specification file ownership is set to root:root (Scored) +[WARN] 1.4.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Not Scored) +[WARN] 1.4.10 Ensure that the Container Network Interface file ownership is set to root:root (Not Scored) +[FAIL] 1.4.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Scored) +[FAIL] 1.4.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Scored) +[PASS] 1.4.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.14 Ensure that the admin.conf file ownership is set to root:root (Scored) +[PASS] 1.4.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.16 Ensure that the scheduler.conf file ownership is set to root:root (Scored) +[PASS] 1.4.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 1.4.18 Ensure that the controller-manager.conf file ownership is set to root:root (Scored) +[WARN] 1.4.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Scored) +[WARN] 1.4.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Scored) +[WARN] 1.4.21 Ensure that the Kubernetes PKI key file permissions are set to 600 or more restrictive (Scored) +[INFO] 1.5 etcd +[FAIL] 1.5.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored) +[FAIL] 1.5.2 Ensure that the --client-cert-auth argument is set to true (Scored) +[PASS] 1.5.3 Ensure that the --auto-tls argument is not set to true (Scored) +[FAIL] 1.5.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Scored) +[FAIL] 1.5.5 Ensure that the --peer-client-cert-auth argument is set to true (Scored) +[PASS] 1.5.6 Ensure that the --peer-auto-tls argument is not set to true (Scored) +[WARN] 1.5.7 Ensure that a unique Certificate Authority is used for etcd (Not Scored) +[INFO] 1.6 General Security Primitives +[WARN] 1.6.1 Ensure that the cluster-admin role is only used where required (Not Scored) +[WARN] 1.6.2 Create administrative boundaries between resources using namespaces (Not Scored) +[WARN] 1.6.3 Create network segmentation using Network Policies (Not Scored) +[WARN] 1.6.4 Ensure that the seccomp profile is set to docker/default in your pod definitions (Not Scored) +[WARN] 1.6.5 Apply Security Context to Your Pods and Containers (Not Scored) +[WARN] 1.6.6 Configure Image Provenance using ImagePolicyWebhook admission controller (Not Scored) +[WARN] 1.6.7 Configure Network policies as appropriate (Not Scored) +[WARN] 1.6.8 Place compensating controls in the form of PSP and RBAC for privileged containers usage (Not Scored) +[INFO] 1.7 PodSecurityPolicies +[WARN] 1.7.1 Do not admit privileged containers (Not Scored) +[WARN] 1.7.2 Do not admit containers wishing to share the host process ID namespace (Not Scored) +[WARN] 1.7.3 Do not admit containers wishing to share the host IPC namespace (Not Scored) +[WARN] 1.7.4 Do not admit containers wishing to share the host network namespace (Not Scored) +[WARN] 1.7.5 Do not admit containers with allowPrivilegeEscalation (Not Scored) +[WARN] 1.7.6 Do not admit root containers (Not Scored) +[WARN] 1.7.7 Do not admit containers with dangerous capabilities (Not Scored) + +== Remediations == +1.1.1 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--anonymous-auth=false + +1.1.6 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +apiserver.yaml on the master node and set the below parameter. +--insecure-port=0 + +1.1.8 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--profiling=false + +1.1.9 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--repair-malformed-updates=false + +1.1.11 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins to +include AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +1.1.13 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins parameter to +include SecurityContextDeny. +--enable-admission-plugins=...,SecurityContextDeny,... + +1.1.15 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-path parameter to a suitable +path and file where you would like audit logs to be written, for example: +--audit-log-path=/var/log/apiserver/audit.log + +1.1.16 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-maxage parameter to 30 or +as an appropriate number of days: --audit-log-maxage=30 + +1.1.17 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-maxbackup parameter to 10 +or to an appropriate value. +--audit-log-maxbackup=10 + +1.1.18 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --audit-log-maxsize parameter to an +appropriate size in MB. For example, to set it as 100 MB: +--audit-log-maxsize=100 + +1.1.19 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --authorization-mode parameter to +values other than AlwaysAllow. One such example could be as below. +--authorization-mode=RBAC + +1.1.21 Follow the Kubernetes documentation and setup the TLS connection between the +apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the --kubelet-certificate-authority +parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +1.1.22 Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +1.1.24 Follow the documentation and create Pod Security Policy objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins parameter to a +value that includes PodSecurityPolicy : +--enable-admission-plugins=...,PodSecurityPolicy,... +Then restart the API Server. + +1.1.25 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --service-account-key-file parameter +to the public key file for service accounts: +--service-account-key-file= + +1.1.26 Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and etcd. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the etcd +certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +1.1.28 Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the TLS certificate and private key file +parameters. +--tls-cert-file= +--tls-private-key-file= + +1.1.29 Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the client certificate authority file. +--client-ca-file= + +1.1.30 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the below parameter. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + +1.1.31 Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and etcd. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the master node and set the etcd +certificate authority file parameter. +--etcd-cafile= + +1.1.32 Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --authorization-mode parameter to a +value that includes Node. +--authorization-mode=Node,RBAC + +1.1.33 Follow the Kubernetes documentation and configure NodeRestriction plug-in on +kubelets. Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the master node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +1.1.34 [Manual test] +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml on the +master node and set the --encryption-provider-config parameter +to the path of that file: +--encryption-provider-config= + +1.1.35 [Manual test] +Follow the Kubernetes documentation and configure a EncryptionConfig file. In this file, +choose aescbc as the encryption provider. +For example, +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - aescbc: + keys: + - name: key1 + secret: <32-byte base64-encoded secret> + +1.1.36 Follow the Kubernetes documentation and set the desired limits in a +configuration file. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +1.1.37b Follow the Kubernetes documentation and set the desired audit policy in the +/etc/kubernetes/audit-policy.yaml file. Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--audit-policy-file=/etc/kubernetes/audit-policy.yaml + +1.1.39 Edit the API server pod specification file kube-apiserver on the master node and set the --authorization-mode parameter to a value that includes RBAC, for example: --authorization-mode=Node,RBAC + +1.2.1 Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +file on the master node and set the below parameter. +--profiling=false + +1.3.1 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the --terminated-pod-gc-threshold to an appropriate threshold, for example: +--terminated-pod-gc-threshold=10 + +1.3.2 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the below parameter. +--profiling=false + +1.3.3 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node to set the below parameter. +--use-service-account-credentials=true + +1.3.4 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the --service-account-private- +key-file parameter to the private key file for service accounts. +--service-account-private-key-file= + +1.3.5 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the master node and set the --root-ca-file parameter to +the certificate bundle file. +--root-ca-file= + +1.3.6 Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +controller-manager.yaml on the master node and set the --feature-gates parameter to +include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +1.4.9 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, +chmod 644 + +1.4.10 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, +chown root:root + +1.4.11 On the etcd server node, get the etcd data directory, passed as an argument --data-dir , +from the below command: +ps -ef | grep etcd +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +1.4.12 On the etcd server node, get the etcd data directory, passed as an argument --data-dir , +from the below command: +ps -ef | grep etcd +Run the below command (based on the etcd data directory found above). For example, +chown etcd:etcd /var/lib/etcd + +1.4.19 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, chown -R root:root /etc/kubernetes/pki/ + +1.4.20 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, chmod -R 644 /etc/kubernetes/pki/*.crt + +1.4.21 [Manual test] +Run the below command (based on the file location on your system) on the master node. +For example, chmod -R 600 /etc/kubernetes/pki/*.key + +1.5.1 Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--ca-file= +--key-file= + +1.5.2 Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +1.5.4 Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +1.5.5 Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +1.5.7 [Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +1.6.1 [Manual test] +Remove any unneeded clusterrolebindings : +kubectl delete clusterrolebinding [name] + +1.6.2 [Manual test] +Follow the documentation and create namespaces for objects in your deployment as you +need them. + +1.6.3 [Manual test] +Follow the documentation and create NetworkPolicy objects as you need them. + +1.6.4 [Manual test] +Seccomp is an alpha feature currently. By default, all alpha features are disabled. So, you +would need to enable alpha features in the apiserver by passing "--feature- +gates=AllAlpha=true" argument. +Edit the /etc/kubernetes/manifests/kube-apiserver.yaml file on the master node and set the KUBE_API_ARGS +parameter to "--feature-gates=AllAlpha=true" +KUBE_API_ARGS="--feature-gates=AllAlpha=true" +Based on your system, restart the kube-apiserver service. For example: +systemctl restart kube-apiserver.service +Use annotations to enable the docker/default seccomp profile in your pod definitions. An +example is as below: +apiVersion: v1 +kind: Pod +metadata: + name: trustworthy-pod + annotations: + seccomp.security.alpha.kubernetes.io/pod: docker/default +spec: + containers: + - name: trustworthy-container + image: sotrustworthy:latest + +1.6.5 [Manual test] +Follow the Kubernetes documentation and apply security contexts to your pods. For a +suggested list of security contexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +1.6.6 [Manual test] +Follow the Kubernetes documentation and setup image provenance. + +1.6.7 [Manual test] +Follow the Kubernetes documentation and setup network policies as appropriate. +For example, you could create a "default" isolation policy for a Namespace by creating a +NetworkPolicy that selects all pods but does not allow any traffic: +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny +spec: + podSelector: + +1.6.8 [Manual test] +Follow Kubernetes documentation and setup PSP and RBAC authorization for your cluster. + +1.7.1 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.privileged field is omitted or set to false. + +1.7.2 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.hostPID field is omitted or set to false. + +1.7.3 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.hostIPC field is omitted or set to false. + +1.7.4 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.hostNetwork field is omitted or set to false. + +1.7.5 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.allowPrivilegeEscalation field is omitted or set to false. + +1.7.6 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.runAsUser.rule is set to either MustRunAsNonRoot or MustRunAs with the range of UIDs not including 0. + +1.7.7 [Manual test] +Create a PSP as described in the Kubernetes documentation, ensuring that the .spec.requiredDropCapabilities is set to include either NET_RAW or ALL. + + +== Summary == +30 checks PASS +36 checks FAIL +25 checks WARN +1 checks INFO +[INFO] 2 Worker Node Security Configuration +[INFO] 2.1 Kubelet +[PASS] 2.1.1 Ensure that the --anonymous-auth argument is set to false (Scored) +[PASS] 2.1.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored) +[PASS] 2.1.3 Ensure that the --client-ca-file argument is set as appropriate (Scored) +[FAIL] 2.1.4 Ensure that the --read-only-port argument is set to 0 (Scored) +[PASS] 2.1.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Scored) +[FAIL] 2.1.6 Ensure that the --protect-kernel-defaults argument is set to true (Scored) +[PASS] 2.1.7 Ensure that the --make-iptables-util-chains argument is set to true (Scored) +[PASS] 2.1.8 Ensure that the --hostname-override argument is not set (Scored) +[FAIL] 2.1.9 Ensure that the --event-qps argument is set to 0 (Scored) +[FAIL] 2.1.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored) +[INFO] 2.1.11 [DEPRECATED] Ensure that the --cadvisor-port argument is set to 0 +[PASS] 2.1.12 Ensure that the --rotate-certificates argument is not set to false (Scored) +[FAIL] 2.1.13 Ensure that the RotateKubeletServerCertificate argument is set to true (Scored) +[PASS] 2.1.14 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Not Scored) +[INFO] 2.2 Configuration Files +[PASS] 2.2.1 Ensure that the kubelet.conf file permissions are set to 644 or more restrictive (Scored) +[PASS] 2.2.2 Ensure that the kubelet.conf file ownership is set to root:root (Scored) +[PASS] 2.2.3 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Scored) +[PASS] 2.2.4 Ensure that the kubelet service file ownership is set to root:root (Scored) +[FAIL] 2.2.5 Ensure that the proxy kubeconfig file permissions are set to 644 or more restrictive (Scored) +[FAIL] 2.2.6 Ensure that the proxy kubeconfig file ownership is set to root:root (Scored) +[WARN] 2.2.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Scored) +[PASS] 2.2.8 Ensure that the client certificate authorities file ownership is set to root:root (Scored) +[PASS] 2.2.9 Ensure that the kubelet configuration file ownership is set to root:root (Scored) +[PASS] 2.2.10 Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Scored) + +== Remediations == +2.1.4 If using a Kubelet config file, edit the file to set readOnlyPort to 0 . +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.6 If using a Kubelet config file, edit the file to set protectKernelDefaults: true . +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.9 If using a Kubelet config file, edit the file to set eventRecordQPS: 0 . +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--event-qps=0 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.10 If using a Kubelet config file, edit the file to set tlsCertFile to the location of the certificate +file to use to identify this Kubelet, and tlsPrivateKeyFile to the location of the +corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +file= +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.1.13 Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +2.2.5 Run the below command (based on the file location on your system) on the each worker +node. For example, +chmod 644 /etc/kubernetes/proxy.conf + +2.2.6 Run the below command (based on the file location on your system) on the each worker +node. For example, +chown root:root /etc/kubernetes/proxy.conf + +2.2.7 Run the following command to modify the file permissions of the --client-ca-file +chmod 644 + + +== Summary == +15 checks PASS +7 checks FAIL +1 checks WARN +1 checks INFO \ No newline at end of file diff --git a/makefile b/makefile index 33ee843..2b4c319 100644 --- a/makefile +++ b/makefile @@ -35,6 +35,9 @@ build-docker: tests: GO111MODULE=on go test -v -short -race -timeout 30s -coverprofile=coverage.txt -covermode=atomic ./... +integration-tests: build-docker + GO111MODULE=on go test ./integration/... -v -tags integration -timeout 600s -args -kubebenchImg=$(IMAGE_NAME) + # creates a kind cluster to be used for development. HAS_KIND := $(shell command -v kind;) kind-test-cluster: