From 564fc0ceedcefced077a30022d0fe72c0d8bd8a4 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Wed, 8 Jan 2025 17:36:03 -0500 Subject: [PATCH] test: run tests in groups to avoid version issues (#95) --- .buildkite/testing.yml | 20 ++++++++++++++---- .github/workflows/CI.yml | 16 ++++++++++++++ test/Project.toml | 12 ----------- test/layer_tests.jl | 4 ++-- test/qa_tests.jl | 2 +- test/runtests.jl | 45 +++++++++++++++++++++++++++++----------- test/shared_testsetup.jl | 1 - test/vision_tests.jl | 28 ++++++++++++++++++------- 8 files changed, 89 insertions(+), 39 deletions(-) diff --git a/.buildkite/testing.yml b/.buildkite/testing.yml index 915337b..7da1b8c 100644 --- a/.buildkite/testing.yml +++ b/.buildkite/testing.yml @@ -1,7 +1,7 @@ steps: - group: ":julia: CUDA GPU" steps: - - label: ":julia: Julia {{matrix.julia}} + CUDA GPU" + - label: ":julia: Julia {{matrix.julia}} + CUDA GPU + {{matrix.group}}" plugins: - JuliaCI/julia#v1: version: "{{matrix.julia}}" @@ -17,17 +17,23 @@ steps: cuda: "*" env: BACKEND_GROUP: "CUDA" + BOLTZ_TEST_GROUP: "{{matrix.group}}" if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip ci\]/ timeout_in_minutes: 60 matrix: setup: julia: - - "1.10" - "1" + group: + - "layers" + - "others" + - "vision" + - "vision_metalhead" + - "integration" - group: ":julia: AMD GPU" steps: - - label: ":julia: Julia: {{matrix.julia}} + AMD GPU" + - label: ":julia: Julia: {{matrix.julia}} + AMD GPU + {{matrix.group}}" plugins: - JuliaCI/julia#v1: version: "{{matrix.julia}}" @@ -43,6 +49,7 @@ steps: JULIA_AMDGPU_HIP_MUST_LOAD: "1" JULIA_AMDGPU_DISABLE_ARTIFACTS: "1" BACKEND_GROUP: "AMDGPU" + BOLTZ_TEST_GROUP: "{{matrix.group}}" agents: queue: "juliagpu" rocm: "*" @@ -52,8 +59,13 @@ steps: matrix: setup: julia: - - "1.10" - "1" + group: + - "layers" + - "others" + - "vision" + - "vision_metalhead" + - "integration" env: SECRET_CODECOV_TOKEN: "gZlC/IAmeUJehhP5mP2QuUV5a1qV61cvo4PUCLkA9vVkt3x6wgD6fTZmCm+f+gHkmkssFxX+q2h1Ud00XXc75H2LrjyR/cDTIthcO46BBOidYocv/U0gfhp6uT2IZ9fi+ryFfTVVpZ0RIUGmDTj0O/b5qt4oaTriAArLAq6mMipbIR9YCz7ZD/hWQXx8oDeAbnDpwQaddwPyhJhz95nayknOpuJj+ClaVOxgsLGZc3ZWiTj1QxkXBNwxLD2ALeG16Qxs9h7eK87sdcbWeTihvJ6OooARgpoVJAa2pJCFYOGy4Bh07c0VTZmicN2M3GIi74Y5T1PWNaz7nGeANO5Pow==;U2FsdGVkX1843DHkbGWCV9PArLBw0rNqmdy56VOTRNTifBSpkC796Oez1lMFU+yDtkElbcrRSIlS5hRFqpsaFA==" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d2f63d1..4456fb2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,6 +31,12 @@ jobs: - "1" os: - ubuntu-latest + group: + - "layers" + - "others" + - "vision" + - "vision_metalhead" + - "integration" steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 @@ -48,6 +54,8 @@ jobs: ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + env: + BOLTZ_TEST_GROUP: ${{ matrix.group }} - uses: julia-actions/julia-processcoverage@v1 with: directories: src,ext @@ -66,6 +74,12 @@ jobs: matrix: version: - "1.10" + group: + - "layers" + - "others" + - "vision" + - "vision_metalhead" + - "integration" steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 @@ -74,6 +88,8 @@ jobs: - uses: julia-actions/julia-downgrade-compat@v1 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + env: + BOLTZ_TEST_GROUP: ${{ matrix.group }} - uses: julia-actions/julia-processcoverage@v1 with: directories: src,ext diff --git a/test/Project.toml b/test/Project.toml index 399a986..66ee7a5 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,23 +1,17 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -Bumper = "8ce10254-0962-460f-a3d8-1f77fea1446e" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" -DataInterpolations = "82cc6244-b520-54b8-b5a6-8a565e85f1d0" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d" -InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" LuxLib = "82251201-b29d-42c6-8e01-566dec8acb11" LuxTestUtils = "ac9de150-d08f-4546-94fb-7472b5760531" MLDataDevices = "7e8f7934-dd98-4c1a-8fe8-92b47a384d40" -Metalhead = "dbeba491-748d-5e0e-a39e-b530a07fa0cc" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -29,24 +23,18 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Aqua = "0.8.7" -Bumper = "0.6, 0.7" ComponentArrays = "0.15.16" -DataInterpolations = "6.4" Downloads = "1.6" -DynamicExpressions = "1" Enzyme = "0.13" ExplicitImports = "1.9.0" ForwardDiff = "0.10.36" GPUArraysCore = "0.1.6, 0.2" Hwloc = "3.2.0" -InteractiveUtils = "<0.0.1, 1" JLD2 = "0.5" -LoopVectorization = "0.12.171" Lux = "1" LuxLib = "1" LuxTestUtils = "1.1.2" MLDataDevices = "1" -Metalhead = "0.9" NNlib = "0.9.21" Pkg = "1.10" Random = "1.10" diff --git a/test/layer_tests.jl b/test/layer_tests.jl index 09437e6..b73d5e4 100644 --- a/test/layer_tests.jl +++ b/test/layer_tests.jl @@ -151,7 +151,7 @@ end end end -@testitem "Spline Layer" setup=[SharedTestSetup] tags=[:layers] begin +@testitem "Spline Layer" setup=[SharedTestSetup] tags=[:integration] begin using ComponentArrays, DataInterpolations, ForwardDiff, Zygote, MLDataDevices @testset "$(mode)" for (mode, aType, dev, ongpu) in MODES @@ -222,7 +222,7 @@ end end end -@testitem "Dynamic Expressions Layer" setup=[SharedTestSetup] tags=[:layers] begin +@testitem "Dynamic Expressions Layer" setup=[SharedTestSetup] tags=[:integration] begin using DynamicExpressions, ForwardDiff, ComponentArrays, Bumper, LoopVectorization operators = OperatorEnum(; binary_operators=[+, -, *], unary_operators=[cos]) diff --git a/test/qa_tests.jl b/test/qa_tests.jl index 0753286..f736955 100644 --- a/test/qa_tests.jl +++ b/test/qa_tests.jl @@ -6,7 +6,7 @@ end @testitem "Explicit Imports: Quality Assurance" tags=[:others] begin - import Lux, Metalhead, Zygote # Load all trigger packages + import Lux, Zygote # Load all trigger packages using ExplicitImports @test check_no_implicit_imports(Boltz; skip=(Base, Core, Lux)) === nothing diff --git a/test/runtests.jl b/test/runtests.jl index 30b27b1..a83b118 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,10 +1,27 @@ -using ReTestItems, Pkg, InteractiveUtils, Hwloc +using ReTestItems, Pkg, Hwloc, Test -@info sprint(versioninfo) +const ALL_BOTLZ_TEST_GROUPS = [ + "layers", "others", "vision", "vision_metalhead", "integration"] + +INPUT_TEST_GROUP = lowercase(get(ENV, "BOLTZ_TEST_GROUP", "all")) +const BOLTZ_TEST_GROUP = if startswith("!", INPUT_TEST_GROUP[1]) + exclude_group = lowercase.(split(INPUT_TEST_GROUP[2:end], ",")) + filter(x -> x ∉ exclude_group, ALL_BOTLZ_TEST_GROUPS) +else + [INPUT_TEST_GROUP] +end const BACKEND_GROUP = lowercase(get(ENV, "BACKEND_GROUP", "all")) const EXTRA_PKGS = String[] +if "all" ∈ BOLTZ_TEST_GROUP || "integration" ∈ BOLTZ_TEST_GROUP + append!(EXTRA_PKGS, + ["DataInterpolations", "DynamicExpressions", "Bumper", "LoopVectorization"]) +end +if "all" ∈ BOLTZ_TEST_GROUP || "vision_metalhead" ∈ BOLTZ_TEST_GROUP + push!(EXTRA_PKGS, "Metalhead") +end + (BACKEND_GROUP == "all" || BACKEND_GROUP == "cuda") && push!(EXTRA_PKGS, "LuxCUDA") (BACKEND_GROUP == "all" || BACKEND_GROUP == "amdgpu") && push!(EXTRA_PKGS, "AMDGPU") @@ -18,16 +35,20 @@ end using Boltz -const BOLTZ_TEST_GROUP = get(ENV, "BOLTZ_TEST_GROUP", "all") const RETESTITEMS_NWORKERS = parse( Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 4)))) -const RETESTITEMS_NWORKER_THREADS = parse(Int, - get(ENV, "RETESTITEMS_NWORKER_THREADS", - string(max(Hwloc.num_virtual_cores() ÷ RETESTITEMS_NWORKERS, 1)))) - -@info "Running tests for group: $BOLTZ_TEST_GROUP with $RETESTITEMS_NWORKERS workers" -ReTestItems.runtests( - Boltz; tags=(BOLTZ_TEST_GROUP == "all" ? nothing : [Symbol(BOLTZ_TEST_GROUP)]), - nworkers=ifelse(BACKEND_GROUP ∈ ("cuda", "amdgpu"), 0, RETESTITEMS_NWORKERS), - nworker_threads=RETESTITEMS_NWORKER_THREADS, testitem_timeout=3600) +@testset "Boltz.jl Tests" begin + @testset "[$(tag)] [$(i)/$(length(BOLTZ_TEST_GROUP))]" for (i, tag) in enumerate(BOLTZ_TEST_GROUP) + nworkers = ifelse( + BACKEND_GROUP ∈ ("cuda", "amdgpu") && + (tag == "vision" || tag == "vision_metalhead"), + 0, RETESTITEMS_NWORKERS) + nworker_threads = parse(Int, + get(ENV, "RETESTITEMS_NWORKER_THREADS", + string(max(Hwloc.num_virtual_cores() ÷ max(nworkers, 1), 1)))) + + ReTestItems.runtests(Boltz; tags=(tag == "all" ? nothing : [Symbol(tag)]), + testitem_timeout=2400, nworkers, nworker_threads) + end +end diff --git a/test/shared_testsetup.jl b/test/shared_testsetup.jl index dc8e429..23ebe80 100644 --- a/test/shared_testsetup.jl +++ b/test/shared_testsetup.jl @@ -3,7 +3,6 @@ import Reexport: @reexport @reexport using Boltz, Lux, GPUArraysCore, LuxLib, LuxTestUtils, Random, StableRNGs using MLDataDevices, JLD2 -import Metalhead LuxTestUtils.jet_target_modules!(["Boltz", "Lux", "LuxLib"]) diff --git a/test/vision_tests.jl b/test/vision_tests.jl index 449e90d..c0093a3 100644 --- a/test/vision_tests.jl +++ b/test/vision_tests.jl @@ -52,7 +52,9 @@ end end end -@testitem "ConvMixer" setup=[SharedTestSetup] tags=[:vision] begin +@testitem "ConvMixer" setup=[SharedTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES, name in [:small, :base, :large] model = Vision.ConvMixer(name; pretrained=false) ps, st = Lux.setup(Random.default_rng(), model) |> dev @@ -66,7 +68,9 @@ end end end -@testitem "GoogLeNet" setup=[SharedTestSetup] tags=[:vision] begin +@testitem "GoogLeNet" setup=[SharedTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES model = Vision.GoogLeNet(; pretrained=false) ps, st = Lux.setup(Random.default_rng(), model) |> dev @@ -80,7 +84,9 @@ end end end -@testitem "MobileNet" setup=[SharedTestSetup] tags=[:vision] begin +@testitem "MobileNet" setup=[SharedTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES, name in [:v1, :v2, :v3_small, :v3_large] model = Vision.MobileNet(name; pretrained=false) ps, st = Lux.setup(Random.default_rng(), model) |> dev @@ -94,7 +100,9 @@ end end end -@testitem "ResNet" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision] begin +@testitem "ResNet" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES, depth in [18, 34, 50, 101, 152] @testset for pretrained in [false, true] model = Vision.ResNet(depth; pretrained) @@ -114,7 +122,9 @@ end end end -@testitem "ResNeXt" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision] begin +@testitem "ResNeXt" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES @testset for (depth, cardinality, base_width) in [ (50, 32, 4), (101, 32, 8), (101, 64, 4), (152, 64, 4)] @@ -139,7 +149,9 @@ end end end -@testitem "WideResNet" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision] begin +@testitem "WideResNet" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES, depth in [50, 101, 152] @testset for pretrained in [false, true] depth == 152 && pretrained && continue @@ -161,7 +173,9 @@ end end end -@testitem "SqueezeNet" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision] begin +@testitem "SqueezeNet" setup=[SharedTestSetup, PretrainedWeightsTestSetup] tags=[:vision_metalhead] begin + import Metalhead + for (mode, aType, dev, ongpu) in MODES @testset for pretrained in [false, true] model = Vision.SqueezeNet(; pretrained)