Skip to content

Commit

Permalink
e2e: update e2e tests to use -label-filter
Browse files Browse the repository at this point in the history 
Current system with FOCUS and SKIP is complicated. Introduce label
filter, update plain text labels of e2e tests to ginkgo>Label(),
and update Makefile.

In addition, put labels to admission webhooks for fpga and sgx.

Signed-off-by: Hyeongju Johannes Lee <[email protected]>
  • Loading branch information
@hj-johannes-lee
hj-johannes-lee committed Oct 5, 2024
1 parent 53efaae commit c07fb82
Show file tree
Hide file tree
Showing 12 changed files with 70 additions and 70 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/lib-e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
- accel-config-demo
- intel-deviceplugin-operator
- name: e2e-qat-gnrd
targetjob: e2e-qat FOCUS="Mode:dpdk" SKIP="(App:(crypto-perf|compress-perf)|Functionality)"
targetjob: e2e-qat LABEL_FILTER="Mode:dpdk && !/App:(crypto-perf|compress-perf|noapp)/ && !/Functionality:.+/"
runner: simics-gnrd
images:
- intel-qat-plugin
Expand All @@ -42,7 +42,7 @@ jobs:
- intel-fpga-admissionwebhook
- opae-nlb-demo
- name: e2e-spr
targetjob: e2e-spr SKIP="App:compress-perf"
targetjob: e2e-spr LABEL_FILTER="!(App:compress-perf)"
runner: spr
images:
- intel-qat-plugin
Expand Down
23 changes: 11 additions & 12 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,38 +141,37 @@ TAG?=devel
export TAG

ifeq ($(E2E_LEVEL), $(filter $(E2E_LEVEL), full))
GENERATED_SKIP_OPT=-ginkgo.skip "App:noapp"
DEFAULT_LABEL_FILTER="!App:noapp"
else ifeq ($(E2E_LEVEL),basic)
ADDITIONAL_FOCUS_REGEX=App:noapp
DEFAULT_LABEL_FILTER="App:noapp"
else
$(error Unsupported E2E_LEVEL value: $(E2E_LEVEL). Possible options: full, basic)
endif
GENERATED_SKIP_OPT += $(if $(SKIP),-ginkgo.skip "$(SKIP)")
ADDITIONAL_FOCUS_REGEX := $(if $(FOCUS),$(FOCUS).*$(ADDITIONAL_FOCUS_REGEX),$(ADDITIONAL_FOCUS_REGEX))
LABEL_FILTER := $(if $(LABEL_FILTER),$(LABEL_FILTER),$(DEFAULT_LABEL_FILTER))

e2e-fpga:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:fpga.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:fpga && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-qat:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:qat.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:qat && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-sgx:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:sgx.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:sgx && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-gpu:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:gpu.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:gpu && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-dsa:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:dsa.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:dsa && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-iaa:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:iaa.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:iaa && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-dlb:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:dlb.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "Device:dlb && $(LABEL_FILTER)" -delete-namespace-on-failure=false

e2e-spr:
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:(iaa|dsa)|Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" -ginkgo.focus "Device:gpu.*Resource:i915" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.label-filter "(/Device:(dlb|dsa|gpu|iaa|sgx)/ || (Device:qat && Mode:dpdk && !Functionality)) && $(LABEL_FILTER)" -delete-namespace-on-failure=false

pre-pull:
ifeq ($(TAG),devel)
Expand Down
14 changes: 7 additions & 7 deletions test/e2e/dlb/dlb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const (
)

func init() {
ginkgo.Describe("DLB plugin [Device:dlb]", describe)
ginkgo.Describe("DLB plugin", ginkgo.Label("Device:dlb"), describe)
}

func describe() {
Expand Down Expand Up @@ -81,36 +81,36 @@ func describe() {
}
})

ginkgo.Context("When PF resources are available [Resource:pf]", func() {
ginkgo.Context("When PF resources are available", ginkgo.Label("Resource:pf"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
resource := v1.ResourceName("dlb.intel.com/pf")
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resource, 30*time.Second, utils.WaitForPositiveResource); err != nil {
framework.Failf("unable to wait for nodes to have positive allocatable resource %s: %v", resource, err)
}
})

ginkgo.It("can run demo app [App:libdlb]", func(ctx context.Context) {
ginkgo.It("can run demo app", ginkgo.Label("App:libdlb"), func(ctx context.Context) {
runDemoApp(ctx, "PF", demoPFYaml, f)
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})

ginkgo.Context("When VF resources are available [Resource:vf]", func() {
ginkgo.Context("When VF resources are available", ginkgo.Label("Resource:vf"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
resource := v1.ResourceName("dlb.intel.com/vf")
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resource, 30*time.Second, utils.WaitForPositiveResource); err != nil {
framework.Failf("unable to wait for nodes to have positive allocatable resource %s: %v", resource, err)
}
})

ginkgo.It("can run demo app [App:libdlb]", func(ctx context.Context) {
ginkgo.It("can run demo app", ginkgo.Label("App:libdlb"), func(ctx context.Context) {
runDemoApp(ctx, "VF", demoVFYaml, f)
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/dsa/dsa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const (
)

func init() {
ginkgo.Describe("DSA plugin [Device:dsa]", describe)
ginkgo.Describe("DSA plugin", ginkgo.Label("Device:dsa"), describe)
}

func describe() {
Expand Down Expand Up @@ -94,23 +94,23 @@ func describe() {
}
})

ginkgo.Context("When DSA resources are available [Resource:dedicated]", func() {
ginkgo.Context("When DSA resources are available", ginkgo.Label("Resource:dedicated"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
ginkgo.By("checking if the resource is allocatable")
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second, utils.WaitForPositiveResource); err != nil {
framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
}
})

ginkgo.It("deploys a demo app [App:accel-config]", func(ctx context.Context) {
ginkgo.It("deploys a demo app", ginkgo.Label("App:accel-config"), func(ctx context.Context) {
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoPath)

ginkgo.By("waiting for the DSA demo to succeed")
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, podName, f.Namespace.Name, 200*time.Second)
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, podName, podName))
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})
Expand Down
20 changes: 10 additions & 10 deletions test/e2e/fpga/fpga.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ const (
)

func init() {
ginkgo.Describe("FPGA Plugin [Device:fpga]", describe)
ginkgo.Describe("FPGA Plugin", ginkgo.Label("Device:fpga"), describe)
}

func describe() {
Expand All @@ -64,41 +64,41 @@ func describe() {
fmw := framework.NewDefaultFramework("fpgaplugin-e2e")
fmw.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged

ginkgo.Context("When FPGA plugin is running in region mode [Mode:region]", func() {
ginkgo.Context("When FPGA plugin is running in region mode", ginkgo.Label("Mode:region"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
runDevicePlugin(ctx, fmw, pluginKustomizationPath, mappingsCollectionPath, arria10NodeResource, "region")
})
ginkgo.It("runs an opae-nlb-demo pod two times [App:opae-nlb-demo]", func(ctx context.Context) {
ginkgo.It("runs an opae-nlb-demo pod two times", ginkgo.Label("App:opae-nlb-demo"), func(ctx context.Context) {
runTestCase(ctx, fmw, "region", nlb3PodResource, "nlb3", "nlb0")
runTestCase(ctx, fmw, "region", nlb0PodResource, "nlb0", "nlb3")
})
})

ginkgo.Context("When FPGA plugin is running in af mode [Mode:af]", func() {
ginkgo.Context("When FPGA plugin is running in af mode", ginkgo.Label("Mode:af"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
runDevicePlugin(ctx, fmw, pluginKustomizationPath, mappingsCollectionPath, nlb0NodeResource, "af")
})
ginkgo.It("runs an opae-nlb-demo pod [App:opae-nlb-demo]", func(ctx context.Context) {
ginkgo.It("runs an opae-nlb-demo pod", ginkgo.Label("App:opae-nlb-demo"), func(ctx context.Context) {
runTestCase(ctx, fmw, "af", nlb0PodResourceAF, "nlb0", "nlb3")
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})

ginkgo.Context("When FPGA plugin is running in region mode [Mode:region]", func() {
ginkgo.Context("When FPGA plugin is running in region mode", ginkgo.Label("Mode:region"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
runDevicePlugin(ctx, fmw, pluginKustomizationPath, mappingsCollectionPath, arria10NodeResource, "region")
})
ginkgo.It("runs [App:opae-nlb-demo]", func(ctx context.Context) {
ginkgo.It("runs", ginkgo.Label("App:opae-nlb-demo"), func(ctx context.Context) {
runTestCase(ctx, fmw, "region", nlb3PodResource, "nlb3", "nlb0")
})
ginkgo.It("runs an opae-nlb-demo pod [App:opae-nlb-demo]", func(ctx context.Context) {
ginkgo.It("runs an opae-nlb-demo pod", ginkgo.Label("App:opae-nlb-demo"), func(ctx context.Context) {
runTestCase(ctx, fmw, "region", nlb0PodResource, "nlb0", "nlb3")
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/fpgaadmissionwebhook/fpgaadmissionwebhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ const (
)

func init() {
ginkgo.Describe("FPGA Admission Webhook", describe)
ginkgo.Describe("FPGA Admission Webhook", ginkgo.Label("Device:fpga"), ginkgo.Label("Admission Webhook"), describe)
}

func describe() {
Expand Down
22 changes: 11 additions & 11 deletions test/e2e/gpu/gpu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ const (

func init() {
// This needs to be Ordered because only one GPU plugin can function on the node at once.
ginkgo.Describe("GPU plugin [Device:gpu]", describe, ginkgo.Ordered)
ginkgo.Describe("GPU plugin", ginkgo.Label("Device:gpu"), describe, ginkgo.Ordered)
}

func createPluginAndVerifyExistence(f *framework.Framework, ctx context.Context, kustomizationPath, baseResource string) {
Expand Down Expand Up @@ -104,7 +104,7 @@ func describe() {
framework.Failf("unable to locate %q: %v", healthMgmtYaml, errFailedToLocateRepoFile)
}

ginkgo.Context("When GPU plugin is deployed [Resource:i915]", func() {
ginkgo.Context("When GPU plugin is deployed", ginkgo.Label("Resource:i915"), func() {
ginkgo.AfterEach(func(ctx context.Context) {
framework.Logf("Removing gpu-plugin manually")

Expand All @@ -118,7 +118,7 @@ func describe() {
}
})

ginkgo.It("checks availability of GPU resources [App:busybox]", func(ctx context.Context) {
ginkgo.It("checks availability of GPU resources", ginkgo.Label("App:busybox"), func(ctx context.Context) {
createPluginAndVerifyExistence(f, ctx, vanillaPath, "gpu.intel.com/i915")

podListFunc := framework.ListObjects(f.ClientSet.CoreV1().Pods(f.Namespace.Name).List, metav1.ListOptions{})
Expand Down Expand Up @@ -189,7 +189,7 @@ func describe() {
framework.Logf("found card and renderD from the log")
})

ginkgo.Context("When [Deployment:monitoring] deployment is applied [Resource:i915]", func() {
ginkgo.Context("When [Deployment:monitoring] deployment is applied", ginkgo.Label("Resource:i915"), func() {
ginkgo.It("check if monitoring resource is available", func(ctx context.Context) {
createPluginAndVerifyExistence(f, ctx, monitoringPath, "gpu.intel.com/i915")

Expand All @@ -200,13 +200,13 @@ func describe() {
})
})

ginkgo.Context("When [Deployment:healthManagement] deployment is applied [Resource:i915]", func() {
ginkgo.Context("When [Deployment:healthManagement] deployment is applied", ginkgo.Label("Resource:i915"), func() {
ginkgo.It("check if i915 resources is available", func(ctx context.Context) {
createPluginAndVerifyExistence(f, ctx, healthMgmtPath, "gpu.intel.com/i915")
})
})

ginkgo.Context("When [Deployment:resourceManager] deployment is applied [Resource:i915]", func() {
ginkgo.Context("When [Deployment:resourceManager] deployment is applied", ginkgo.Label("Resource:i915"), func() {
ginkgo.It("check if i915 resources is available", func(ctx context.Context) {
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(nfdRulesPath))

Expand All @@ -227,7 +227,7 @@ func describe() {
})
})

ginkgo.It("run a small workload on the GPU [App:tensorflow]", func(ctx context.Context) {
ginkgo.It("run a small workload on the GPU", ginkgo.Label("App:tensorflow"), func(ctx context.Context) {
createPluginAndVerifyExistence(f, ctx, vanillaPath, "gpu.intel.com/i915")

kustomYaml, err := utils.LocateRepoFile(tfKustomizationYaml)
Expand All @@ -247,13 +247,13 @@ func describe() {
framework.Logf("tensorflow execution succeeded!")
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})

ginkgo.Context("When GPU resources are available [Resource:xe]", func() {
ginkgo.It("checks availability of GPU resources [App:busybox]", func(ctx context.Context) {
ginkgo.Context("When GPU resources are available", ginkgo.Label("Resource:xe"), func() {
ginkgo.It("checks availability of GPU resources", ginkgo.Label("App:busybox"), func(ctx context.Context) {
createPluginAndVerifyExistence(f, ctx, vanillaPath, "gpu.intel.com/xe")

ginkgo.By("submitting a pod requesting GPU resources")
Expand Down Expand Up @@ -296,7 +296,7 @@ func describe() {
framework.Logf("found card and renderD from the log")
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/iaa/iaa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const (
)

func init() {
ginkgo.Describe("IAA plugin [Device:iaa]", describe)
ginkgo.Describe("IAA plugin", ginkgo.Label("Device:iaa"), describe)
}

func describe() {
Expand Down Expand Up @@ -94,23 +94,23 @@ func describe() {
}
})

ginkgo.Context("When IAA resources are available [Resource:dedicated]", func() {
ginkgo.Context("When IAA resources are available", ginkgo.Label("Resource:dedicated"), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
ginkgo.By("checking if the resource is allocatable")
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "iaa.intel.com/wq-user-dedicated", 300*time.Second, utils.WaitForPositiveResource); err != nil {
framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
}
})

ginkgo.It("deploys a demo app [App:accel-config]", func(ctx context.Context) {
ginkgo.It("deploys a demo app", ginkgo.Label("App:accel-config"), func(ctx context.Context) {
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoPath)

ginkgo.By("waiting for the IAA demo to succeed")
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, podName, f.Namespace.Name, 360*time.Second)
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, podName, podName))
})

ginkgo.When("there is no app to run [App:noapp]", func() {
ginkgo.When("there is no app to run", ginkgo.Label("App:noapp"), func() {
ginkgo.It("does nothing", func() {})
})
})
Expand Down
Loading

0 comments on commit c07fb82

Please sign in to comment.