arno/kube-bench
1
0
Fork 0
mirror of https://github.com/aquasecurity/kube-bench.git synced 2024-11-22 08:08:07 +00:00
Code Issues Releases Wiki Activity

Integration Test: Improves performance and Reliability (#555)

Browse Source 
* Fixes #552: Improves performance and reliability.

Co-Authored-By: Liz Rice <liz@lizrice.com>
This commit is contained in:
Roberto Rojas 2020-01-09 09:57:40 -05:00 committed by GitHub
parent b677c86868
commit efcd63aa38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 82 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
integration/integration.go
View File

@ -19,60 +19,38 @@ import (
"sigs.k8s.io/kind/pkg/cluster/create"
)
func runWithKind(clusterName, kindCfg, kubebenchYAML, kubebenchImg string, timeout, ticker time.Duration) (string, error) {
options := create.WithConfigFile(kindCfg)
ctx := cluster.NewContext(clusterName)
if err := ctx.Create(options); err != nil {
return "", err
}
defer func() {
ctx.Delete()
}()
clientset, err := getClientSet(ctx.KubeConfigPath())
func runWithKind(ctx *cluster.Context, clientset *kubernetes.Clientset, jobName, kubebenchYAML, kubebenchImg string, timeout time.Duration) (string, error) {
err := deployJob(clientset, kubebenchYAML, kubebenchImg)
if err != nil {
return "", err
}
jobYAML, err := ioutil.ReadFile(kubebenchYAML)
if err != nil {
return "", err
}
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML))
if err != nil {
return "", err
}
job := &batchv1.Job{}
if err := decoder.Decode(job); err != nil {
return "", err
}
job.Spec.Template.Spec.Containers[0].Image = kubebenchImg
if err := loadImageFromDocker(kubebenchImg, ctx); err != nil {
return "", err
}
_, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job)
if err != nil {
return "", err
}
clientset, err = getClientSet(ctx.KubeConfigPath())
if err != nil {
return "", err
}
p, err := findPodForJob(clientset, "kube-bench", timeout, ticker)
p, err := findPodForJob(clientset, jobName, timeout)
if err != nil {
return "", err
}
output := getPodLogs(clientset, p)
err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Delete(jobName, nil)
if err != nil {
return "", err
}
return output, nil
}
func setupCluster(clusterName, kindCfg string, duration time.Duration) (*cluster.Context, error) {
options := create.WithConfigFile(kindCfg)
toptions := create.WaitForReady(duration)
ctx := cluster.NewContext(clusterName)
if err := ctx.Create(options, toptions); err != nil {
return nil, err
}
return ctx, nil
}
func getClientSet(configPath string) (*kubernetes.Clientset, error) {
config, err := clientcmd.BuildConfigFromFlags("", configPath)
if err != nil {
@ -86,16 +64,38 @@ func getClientSet(configPath string) (*kubernetes.Clientset, error) {
return clientset, nil
}
func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer time.Duration) (*apiv1.Pod, error) {
timeout := time.After(tout)
func deployJob(clientset *kubernetes.Clientset, kubebenchYAML, kubebenchImg string) error {
jobYAML, err := ioutil.ReadFile(kubebenchYAML)
if err != nil {
return err
}
decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML))
job := &batchv1.Job{}
if err := decoder.Decode(job); err != nil {
return err
}
job.Spec.Template.Spec.Containers[0].Image = kubebenchImg
_, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job)
return err
}
func findPodForJob(clientset *kubernetes.Clientset, jobName string, duration time.Duration) (*apiv1.Pod, error) {
failedPods := make(map[string]struct{})
selector := fmt.Sprintf("job-name=%s", jobName)
timeout := time.After(duration)
for {
time.Sleep(3 * time.Second)
podfailed:
select {
case <-timeout:
return nil, fmt.Errorf("podList - time out: no Pod with %s", name)
return nil, fmt.Errorf("podList - timed out: no Pod found for Job %s", jobName)
default:
pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{})
pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{
LabelSelector: selector,
})
if err != nil {
return nil, err
}
@ -105,7 +105,7 @@ func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer tim
continue
}
if strings.HasPrefix(cp.Name, name) {
if strings.HasPrefix(cp.Name, jobName) {
fmt.Printf("pod (%s) - %#v\n", cp.Name, cp.Status.Phase)
if cp.Status.Phase == apiv1.PodSucceeded {
return &cp, nil
@ -117,48 +117,12 @@ func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer tim
break podfailed
}
// Pod still working
// Wait and try again...
ticker := time.NewTicker(timer)
for {
fmt.Println("using ticker and an timer...")
select {
case <-ticker.C:
thePod, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).Get(cp.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
fmt.Printf("thePod (%s) - status:%#v \n", thePod.Name, thePod.Status.Phase)
if thePod.Status.Phase == apiv1.PodSucceeded {
return thePod, nil
}
}
}
if thePod.Status.Phase == apiv1.PodFailed {
fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Phase)
failedPods[thePod.Name] = struct{}{}
ticker.Stop()
break podfailed
}
if thePod.Status.Phase == apiv1.PodPending && strings.Contains(thePod.Status.Reason, "Failed") {
fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Reason)
failedPods[thePod.Name] = struct{}{}
ticker.Stop()
break podfailed
}
case <-timeout:
ticker.Stop()
return nil, fmt.Errorf("getPod time out: no Pod with %s", name)
}
}
}
}
}
time.Sleep(1 * time.Second)
}
return nil, fmt.Errorf("no Pod with %s", name)
return nil, fmt.Errorf("no Pod found for Job %q", jobName)
}
func getPodLogs(clientset *kubernetes.Clientset, pod *apiv1.Pod) string {

59
integration/integration_test.go
View File

@ -12,59 +12,68 @@ import (
)
var kubebenchImg = flag.String("kubebenchImg", "aquasec/kube-bench:latest", "kube-bench image used as part of this test")
var timeout = flag.Duration("timeout", 10*time.Minute, "Test Timeout")
func TestRunWithKind(t *testing.T) {
flag.Parse()
fmt.Printf("kube-bench Container Image: %s\n", *kubebenchImg)
timeout := time.Duration(10 * time.Minute)
ticker := time.Duration(2 * time.Second)
mustMatch := func(expFname, data string) {
d, err := ioutil.ReadFile(expFname)
if err != nil {
t.Error(err)
}
expectedData := strings.TrimSpace(string(d))
data = strings.TrimSpace(data)
if expectedData != data {
t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, data)
}
}
cases := []struct {
TestName string
KindCfg string
KubebenchYAML string
ExpectedFile string
ExpectError bool
}{
{
TestName: "job",
KindCfg: "./testdata/add-tls-kind-k8s114.yaml",
TestName: "kube-bench",
KubebenchYAML: "../job.yaml",
ExpectedFile: "./testdata/job.data",
},
{
TestName: "job-node",
KindCfg: "./testdata/add-tls-kind-k8s114.yaml",
TestName: "kube-bench-node",
KubebenchYAML: "../job-node.yaml",
ExpectedFile: "./testdata/job-node.data",
},
{
TestName: "job-master",
KindCfg: "./testdata/add-tls-kind-k8s114.yaml",
TestName: "kube-bench-master",
KubebenchYAML: "../job-master.yaml",
ExpectedFile: "./testdata/job-master.data",
},
}
ctx, err := setupCluster("kube-bench", "./testdata/add-tls-kind-k8s114.yaml", *timeout)
if err != nil {
t.Fatalf("failed to setup KIND cluster error: %v", err)
}
defer func() {
ctx.Delete()
}()
if err := loadImageFromDocker(*kubebenchImg, ctx); err != nil {
t.Fatalf("failed to load kube-bench image from Docker to KIND error: %v", err)
}
clientset, err := getClientSet(ctx.KubeConfigPath())
if err != nil {
t.Fatalf("failed to connect to Kubernetes cluster error: %v", err)
}
for _, c := range cases {
t.Run(c.TestName, func(t *testing.T) {
data, err := runWithKind(c.TestName, c.KindCfg, c.KubebenchYAML, *kubebenchImg, timeout, ticker)
resultData, err := runWithKind(ctx, clientset, c.TestName, c.KubebenchYAML, *kubebenchImg, *timeout)
if err != nil {
t.Fatalf("unexpected error: %v", err)
return
t.Errorf("unexpected error: %v", err)
}
c, err := ioutil.ReadFile(c.ExpectedFile)
if err != nil {
t.Error(err)
}
expectedData := strings.TrimSpace(string(c))
resultData = strings.TrimSpace(resultData)
if expectedData != resultData {
t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, resultData)
}
mustMatch(c.ExpectedFile, data)
})
}
}