From efcd63aa387dc9aa9de6606d988d07b6717cfd8e Mon Sep 17 00:00:00 2001 From: Roberto Rojas Date: Thu, 9 Jan 2020 09:57:40 -0500 Subject: [PATCH] Integration Test: Improves performance and Reliability (#555) * Fixes #552: Improves performance and reliability. Co-Authored-By: Liz Rice --- integration/integration.go | 120 +++++++++++--------------------- integration/integration_test.go | 59 +++++++++------- 2 files changed, 76 insertions(+), 103 deletions(-) diff --git a/integration/integration.go b/integration/integration.go index 90dc5ea..914a8d8 100644 --- a/integration/integration.go +++ b/integration/integration.go @@ -19,58 +19,36 @@ import ( "sigs.k8s.io/kind/pkg/cluster/create" ) -func runWithKind(clusterName, kindCfg, kubebenchYAML, kubebenchImg string, timeout, ticker time.Duration) (string, error) { - options := create.WithConfigFile(kindCfg) - ctx := cluster.NewContext(clusterName) - if err := ctx.Create(options); err != nil { - return "", err - } - defer func() { - ctx.Delete() - }() - - clientset, err := getClientSet(ctx.KubeConfigPath()) - if err != nil { - return "", err - } - - jobYAML, err := ioutil.ReadFile(kubebenchYAML) +func runWithKind(ctx *cluster.Context, clientset *kubernetes.Clientset, jobName, kubebenchYAML, kubebenchImg string, timeout time.Duration) (string, error) { + err := deployJob(clientset, kubebenchYAML, kubebenchImg) if err != nil { return "", err } - decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML)) + p, err := findPodForJob(clientset, jobName, timeout) if err != nil { return "", err } - job := &batchv1.Job{} - if err := decoder.Decode(job); err != nil { - return "", err - } - job.Spec.Template.Spec.Containers[0].Image = kubebenchImg - - if err := loadImageFromDocker(kubebenchImg, ctx); err != nil { - return "", err - } + output := getPodLogs(clientset, p) - _, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job) + err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Delete(jobName, nil) if err != nil { return "", err } - clientset, err = getClientSet(ctx.KubeConfigPath()) - if err != nil { - return "", err - } + return output, nil +} - p, err := findPodForJob(clientset, "kube-bench", timeout, ticker) - if err != nil { - return "", err +func setupCluster(clusterName, kindCfg string, duration time.Duration) (*cluster.Context, error) { + options := create.WithConfigFile(kindCfg) + toptions := create.WaitForReady(duration) + ctx := cluster.NewContext(clusterName) + if err := ctx.Create(options, toptions); err != nil { + return nil, err } - output := getPodLogs(clientset, p) - return output, nil + return ctx, nil } func getClientSet(configPath string) (*kubernetes.Clientset, error) { @@ -86,16 +64,38 @@ func getClientSet(configPath string) (*kubernetes.Clientset, error) { return clientset, nil } -func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer time.Duration) (*apiv1.Pod, error) { - timeout := time.After(tout) +func deployJob(clientset *kubernetes.Clientset, kubebenchYAML, kubebenchImg string) error { + jobYAML, err := ioutil.ReadFile(kubebenchYAML) + if err != nil { + return err + } + + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(jobYAML), len(jobYAML)) + job := &batchv1.Job{} + if err := decoder.Decode(job); err != nil { + return err + } + job.Spec.Template.Spec.Containers[0].Image = kubebenchImg + + _, err = clientset.BatchV1().Jobs(apiv1.NamespaceDefault).Create(job) + + return err +} + +func findPodForJob(clientset *kubernetes.Clientset, jobName string, duration time.Duration) (*apiv1.Pod, error) { failedPods := make(map[string]struct{}) + selector := fmt.Sprintf("job-name=%s", jobName) + timeout := time.After(duration) for { + time.Sleep(3 * time.Second) podfailed: select { case <-timeout: - return nil, fmt.Errorf("podList - time out: no Pod with %s", name) + return nil, fmt.Errorf("podList - timed out: no Pod found for Job %s", jobName) default: - pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{}) + pods, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).List(metav1.ListOptions{ + LabelSelector: selector, + }) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer tim continue } - if strings.HasPrefix(cp.Name, name) { + if strings.HasPrefix(cp.Name, jobName) { fmt.Printf("pod (%s) - %#v\n", cp.Name, cp.Status.Phase) if cp.Status.Phase == apiv1.PodSucceeded { return &cp, nil @@ -117,48 +117,12 @@ func findPodForJob(clientset *kubernetes.Clientset, name string, tout, timer tim break podfailed } - // Pod still working - // Wait and try again... - ticker := time.NewTicker(timer) - for { - fmt.Println("using ticker and an timer...") - select { - case <-ticker.C: - thePod, err := clientset.CoreV1().Pods(apiv1.NamespaceDefault).Get(cp.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - fmt.Printf("thePod (%s) - status:%#v \n", thePod.Name, thePod.Status.Phase) - if thePod.Status.Phase == apiv1.PodSucceeded { - return thePod, nil - } - - if thePod.Status.Phase == apiv1.PodFailed { - fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Phase) - failedPods[thePod.Name] = struct{}{} - ticker.Stop() - break podfailed - } - - if thePod.Status.Phase == apiv1.PodPending && strings.Contains(thePod.Status.Reason, "Failed") { - fmt.Printf("thePod (%s) - %s - retrying...\n", thePod.Name, thePod.Status.Reason) - failedPods[thePod.Name] = struct{}{} - ticker.Stop() - break podfailed - } - - case <-timeout: - ticker.Stop() - return nil, fmt.Errorf("getPod time out: no Pod with %s", name) - } - } } } } - time.Sleep(1 * time.Second) } - return nil, fmt.Errorf("no Pod with %s", name) + return nil, fmt.Errorf("no Pod found for Job %q", jobName) } func getPodLogs(clientset *kubernetes.Clientset, pod *apiv1.Pod) string { diff --git a/integration/integration_test.go b/integration/integration_test.go index ee9b76e..eda7dd2 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -12,59 +12,68 @@ import ( ) var kubebenchImg = flag.String("kubebenchImg", "aquasec/kube-bench:latest", "kube-bench image used as part of this test") +var timeout = flag.Duration("timeout", 10*time.Minute, "Test Timeout") func TestRunWithKind(t *testing.T) { flag.Parse() fmt.Printf("kube-bench Container Image: %s\n", *kubebenchImg) - timeout := time.Duration(10 * time.Minute) - ticker := time.Duration(2 * time.Second) - - mustMatch := func(expFname, data string) { - d, err := ioutil.ReadFile(expFname) - if err != nil { - t.Error(err) - } - expectedData := strings.TrimSpace(string(d)) - data = strings.TrimSpace(data) - if expectedData != data { - t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, data) - } - } cases := []struct { TestName string - KindCfg string KubebenchYAML string ExpectedFile string ExpectError bool }{ { - TestName: "job", - KindCfg: "./testdata/add-tls-kind-k8s114.yaml", + TestName: "kube-bench", KubebenchYAML: "../job.yaml", ExpectedFile: "./testdata/job.data", }, { - TestName: "job-node", - KindCfg: "./testdata/add-tls-kind-k8s114.yaml", + TestName: "kube-bench-node", KubebenchYAML: "../job-node.yaml", ExpectedFile: "./testdata/job-node.data", }, { - TestName: "job-master", - KindCfg: "./testdata/add-tls-kind-k8s114.yaml", + TestName: "kube-bench-master", KubebenchYAML: "../job-master.yaml", ExpectedFile: "./testdata/job-master.data", }, } + ctx, err := setupCluster("kube-bench", "./testdata/add-tls-kind-k8s114.yaml", *timeout) + if err != nil { + t.Fatalf("failed to setup KIND cluster error: %v", err) + } + defer func() { + ctx.Delete() + }() + + if err := loadImageFromDocker(*kubebenchImg, ctx); err != nil { + t.Fatalf("failed to load kube-bench image from Docker to KIND error: %v", err) + } + + clientset, err := getClientSet(ctx.KubeConfigPath()) + if err != nil { + t.Fatalf("failed to connect to Kubernetes cluster error: %v", err) + } + for _, c := range cases { t.Run(c.TestName, func(t *testing.T) { - data, err := runWithKind(c.TestName, c.KindCfg, c.KubebenchYAML, *kubebenchImg, timeout, ticker) + resultData, err := runWithKind(ctx, clientset, c.TestName, c.KubebenchYAML, *kubebenchImg, *timeout) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + c, err := ioutil.ReadFile(c.ExpectedFile) if err != nil { - t.Fatalf("unexpected error: %v", err) - return + t.Error(err) + } + + expectedData := strings.TrimSpace(string(c)) + resultData = strings.TrimSpace(resultData) + if expectedData != resultData { + t.Errorf("expected: %q\n\n Got %q\n\n", expectedData, resultData) } - mustMatch(c.ExpectedFile, data) }) } }