Integration Test: Improves performance and Reliability (#555)

* Fixes #552: Improves performance and reliability.

Co-Authored-By: Liz Rice <liz@lizrice.com>

integration/integration.go

@@ -19,58 +19,36 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster/create"
 )
 
-func runWithKind(clusterName, kindCfg, kubebenchYAML, kubebenchImg string, timeout, ticker time.Duration) (string, error) {
+func runWithKind(ctx *cluster.Context, clientset *kubernetes.Clientset, jobName, kubebenchYAML, kubebenchImg string, timeout time.Duration) (string, error) {
-	options := create.WithConfigFile(kindCfg)
+	err := deployJob(clientset, kubebenchYAML, kubebenchImg)
-	ctx := cluster.NewContext(clusterName)
-	if err := ctx.Create(options); err != nil {
-		return "", err
-	}
-	defer func() {
-		ctx.Delete()
-	}()
-
-	clientset, err := getClientSet(ctx.KubeConfigPath())
-	if err != nil {
-		return "", err
-	}
-
-	jobYAML, err := ioutil.ReadFile(kubebenchYAML)
 	if err != nil {
 		return "", err
 	}
 
-	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML))
+	p, err := findPodForJob(clientset, jobName, timeout)
 	if err != nil {
 		return "", err
 	}
 
-	job := &batchv1.Job{}
+	output := getPodLogs(clientset, p)
-	if err := decoder.Decode(job); err != nil {
-		return "", err
-	}
-	job.Spec.Template.Spec.Containers[0].Image = kubebenchImg
-
-	if err := loadImageFromDocker(kubebenchImg, ctx); err != nil {
-		return "", err
-	}
 
-	_, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job)
+	err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Delete(jobName, nil)
 	if err != nil {
 		return "", err
 	}
 
-	clientset, err = getClientSet(ctx.KubeConfigPath())
+	return output, nil
-	if err != nil {
+}
-		return "", err
-	}
 
-	p, err := findPodForJob(clientset, "kube-bench", timeout, ticker)
+func setupCluster(clusterName, kindCfg string, duration time.Duration) (*cluster.Context, error) {
-	if err != nil {
+	options := create.WithConfigFile(kindCfg)
-		return "", err
+	toptions := create.WaitForReady(duration)
+	ctx := cluster.NewContext(clusterName)
+	if err := ctx.Create(options, toptions); err != nil {
+		return nil, err
 	}
 
-	output := getPodLogs(clientset, p)
+	return ctx, nil
-	return output, nil
 }
 
 func getClientSet(configPath string) (*kubernetes.Clientset, error) {
@@ -86,16 +64,38 @@ func getClientSet(configPath string) (*kubernetes.Clientset, error) {
 	return clientset, nil
 }
 
-func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer time.Duration) (*apiv1.Pod, error) {
+func deployJob(clientset *kubernetes.Clientset, kubebenchYAML, kubebenchImg string) error {
-	timeout := time.After(tout)
+	jobYAML, err := ioutil.ReadFile(kubebenchYAML)
+	if err != nil {
+		return err
+	}
+
+	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML))
+	job := &batchv1.Job{}
+	if err := decoder.Decode(job); err != nil {
+		return err
+	}
+	job.Spec.Template.Spec.Containers[0].Image = kubebenchImg
+
+	_, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job)
+
+	return err
+}
+
+func findPodForJob(clientset *kubernetes.Clientset, jobName string, duration time.Duration) (*apiv1.Pod, error) {
 	failedPods := make(map[string]struct{})
+	selector := fmt.Sprintf("job-name=%s", jobName)
+	timeout := time.After(duration)
 	for {
+		time.Sleep(3 * time.Second)
 	podfailed:
 		select {
 		case <-timeout:
-			return nil, fmt.Errorf("podList - time out: no Pod with %s", name)
+			return nil, fmt.Errorf("podList - timed out: no Pod found for Job %s", jobName)
 		default:
-			pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{})
+			pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{
+				LabelSelector: selector,
+			})
 			if err != nil {
 				return nil, err
 			}
@@ -105,7 +105,7 @@ func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer tim
 					continue
 				}
 
-				if strings.HasPrefix(cp.Name, name) {
+				if strings.HasPrefix(cp.Name, jobName) {
 					fmt.Printf("pod (%s) - %#v\n", cp.Name, cp.Status.Phase)
 					if cp.Status.Phase == apiv1.PodSucceeded {
 						return &cp, nil
@@ -117,48 +117,12 @@ func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer tim
 						break podfailed
 					}
 
-					// Pod still working
-					// Wait and try again...
-					ticker := time.NewTicker(timer)
-					for {
-						fmt.Println("using ticker and an timer...")
-						select {
-						case <-ticker.C:
-							thePod, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).Get(cp.Name, metav1.GetOptions{})
-							if err != nil {
-								return nil, err
-							}
-							fmt.Printf("thePod (%s) - status:%#v \n", thePod.Name, thePod.Status.Phase)
-							if thePod.Status.Phase == apiv1.PodSucceeded {
-								return thePod, nil
-							}
-
-							if thePod.Status.Phase == apiv1.PodFailed {
-								fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Phase)
-								failedPods[thePod.Name] = struct{}{}
-								ticker.Stop()
-								break podfailed
-							}
-
-							if thePod.Status.Phase == apiv1.PodPending && strings.Contains(thePod.Status.Reason, "Failed") {
-								fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Reason)
-								failedPods[thePod.Name] = struct{}{}
-								ticker.Stop()
-								break podfailed
-							}
-
-						case <-timeout:
-							ticker.Stop()
-							return nil, fmt.Errorf("getPod time out: no Pod with %s", name)
-						}
-					}
 				}
 			}
 		}
-		time.Sleep(1 * time.Second)
 	}
 
-	return nil, fmt.Errorf("no Pod with %s", name)
+	return nil, fmt.Errorf("no Pod found for Job %q", jobName)
 }
 
 func getPodLogs(clientset *kubernetes.Clientset, pod *apiv1.Pod) string {

integration/integration_test.go

@@ -12,59 +12,68 @@ import (
 )
 
 var kubebenchImg = flag.String("kubebenchImg", "aquasec/kube-bench:latest", "kube-bench image used as part of this test")
+var timeout = flag.Duration("timeout", 10*time.Minute, "Test Timeout")
 
 func TestRunWithKind(t *testing.T) {
 	flag.Parse()
 	fmt.Printf("kube-bench Container Image: %s\n", *kubebenchImg)
-	timeout := time.Duration(10 * time.Minute)
-	ticker := time.Duration(2 * time.Second)
-
-	mustMatch := func(expFname, data string) {
-		d, err := ioutil.ReadFile(expFname)
-		if err != nil {
-			t.Error(err)
-		}
-		expectedData := strings.TrimSpace(string(d))
-		data = strings.TrimSpace(data)
-		if expectedData != data {
-			t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, data)
-		}
-	}
 
 	cases := []struct {
 		TestName      string
-		KindCfg       string
 		KubebenchYAML string
 		ExpectedFile  string
 		ExpectError   bool
 	}{
 		{
-			TestName:      "job",
+			TestName:      "kube-bench",
-			KindCfg:       "./testdata/add-tls-kind-k8s114.yaml",
 			KubebenchYAML: "../job.yaml",
 			ExpectedFile:  "./testdata/job.data",
 		},
 		{
-			TestName:      "job-node",
+			TestName:      "kube-bench-node",
-			KindCfg:       "./testdata/add-tls-kind-k8s114.yaml",
 			KubebenchYAML: "../job-node.yaml",
 			ExpectedFile:  "./testdata/job-node.data",
 		},
 		{
-			TestName:      "job-master",
+			TestName:      "kube-bench-master",
-			KindCfg:       "./testdata/add-tls-kind-k8s114.yaml",
 			KubebenchYAML: "../job-master.yaml",
 			ExpectedFile:  "./testdata/job-master.data",
 		},
 	}
+	ctx, err := setupCluster("kube-bench", "./testdata/add-tls-kind-k8s114.yaml", *timeout)
+	if err != nil {
+		t.Fatalf("failed to setup KIND cluster error: %v", err)
+	}
+	defer func() {
+		ctx.Delete()
+	}()
+
+	if err := loadImageFromDocker(*kubebenchImg, ctx); err != nil {
+		t.Fatalf("failed to load kube-bench image from Docker to KIND error: %v", err)
+	}
+
+	clientset, err := getClientSet(ctx.KubeConfigPath())
+	if err != nil {
+		t.Fatalf("failed to connect to Kubernetes cluster error: %v", err)
+	}
+
 	for _, c := range cases {
 		t.Run(c.TestName, func(t *testing.T) {
-			data, err := runWithKind(c.TestName, c.KindCfg, c.KubebenchYAML, *kubebenchImg, timeout, ticker)
+			resultData, err := runWithKind(ctx, clientset, c.TestName, c.KubebenchYAML, *kubebenchImg, *timeout)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			c, err := ioutil.ReadFile(c.ExpectedFile)
 			if err != nil {
-				t.Fatalf("unexpected error: %v", err)
+				t.Error(err)
-				return
+			}
+
+			expectedData := strings.TrimSpace(string(c))
+			resultData = strings.TrimSpace(resultData)
+			if expectedData != resultData {
+				t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, resultData)
 			}
-			mustMatch(c.ExpectedFile, data)
 		})
 	}
 }