From 27eb523930a619c407a0d9cc627dbbac505ebf09 Mon Sep 17 00:00:00 2001 From: Lewis Panos Date: Thu, 23 Jan 2025 17:35:27 +0000 Subject: [PATCH] Add nightly explicit parametrization, update nightly flag: its presence means only collect nightly tests, lacktherof means collect everything but nightly tests. --- .github/workflows/run-model-tests.yml | 77 ++++++++++++++++--- .github/workflows/run-tests.yml | 34 ++++---- tests/conftest.py | 39 ++++++++-- tests/models/MobileNetV2/test_MobileNetV2.py | 1 + tests/models/Qwen/test_qwen2_casual_lm.py | 1 + .../Qwen/test_qwen2_token_classification.py | 1 + tests/models/RMBG/test_RMBG.py | 1 + tests/models/albert/test_albert_masked_lm.py | 1 + .../albert/test_albert_question_answering.py | 1 + .../test_albert_sequence_classification.py | 1 + .../test_albert_token_classification.py | 1 + .../autoencoder_conv/test_autoencoder_conv.py | 1 + .../test_autoencoder_conv_v2.py | 1 + .../test_autoencoder_linear.py | 1 + .../beit/test_beit_image_classification.py | 1 + tests/models/bert/test_bert.py | 1 + tests/models/bloom/test_bloom.py | 1 + tests/models/clip/test_clip.py | 1 + tests/models/codegen/test_codegen.py | 1 + tests/models/deit/test_deit.py | 1 + tests/models/detr/test_detr.py | 1 + tests/models/distilbert/test_distilbert.py | 1 + tests/models/dpr/test_dpr.py | 1 + tests/models/falcon/test_falcon.py | 1 + tests/models/flan_t5/test_flan_t5.py | 1 + tests/models/glpn_kitti/test_glpn_kitti.py | 1 + tests/models/gpt2/test_gpt2.py | 1 + tests/models/gpt_neo/test_gpt_neo.py | 1 + .../hand_landmark/test_hand_landmark.py | 1 + tests/models/hardnet/test_hardnet.py | 1 + tests/models/llama/test_llama.py | 1 + .../models/mgp-str-base/test_mgp_str_base.py | 1 + tests/models/mlpmixer/test_mlpmixer.py | 1 + tests/models/mnist/test_mnist.py | 1 + .../mobilenet_ssd/test_mobilenet_ssd.py | 1 + .../musicgen_small/test_musicgen_small.py | 1 + tests/models/openpose/test_openpose.py | 1 + tests/models/openpose/test_openpose_v2.py | 1 + tests/models/opt/test_opt.py | 1 + .../models/perceiver_io/test_perceiver_io.py | 1 + tests/models/resnet/test_resnet.py | 1 + tests/models/resnet50/test_resnet50.py | 1 + tests/models/roberta/test_roberta.py | 1 + tests/models/segformer/test_segformer.py | 1 + .../segment_anything/test_segment_anything.py | 1 + .../models/speecht5_tts/test_speecht5_tts.py | 1 + .../models/squeeze_bert/test_squeeze_bert.py | 1 + .../stable_diffusion/test_stable_diffusion.py | 1 + .../test_stable_diffusion_v2.py | 1 + tests/models/t5/test_t5.py | 1 + .../timm/test_timm_image_classification.py | 3 +- .../test_torchvision_image_classification.py | 14 +++- .../test_torchvision_object_detection.py | 12 ++- tests/models/unet/test_unet.py | 1 + tests/models/unet_brain/test_unet_brain.py | 1 + .../models/unet_carvana/test_unet_carvana.py | 1 + tests/models/vilt/test_vilt.py | 1 + tests/models/whisper/test_whisper.py | 1 + tests/models/xglm/test_xglm.py | 1 + tests/models/yolos/test_yolos.py | 1 + tests/models/yolov3/test_yolov3.py | 1 + tests/models/yolov5/test_yolov5.py | 1 + 62 files changed, 196 insertions(+), 39 deletions(-) diff --git a/.github/workflows/run-model-tests.yml b/.github/workflows/run-model-tests.yml index daf3afbc..a5f7a9ad 100644 --- a/.github/workflows/run-model-tests.yml +++ b/.github/workflows/run-model-tests.yml @@ -14,8 +14,73 @@ jobs: fail-fast: false matrix: build: [ - {runs-on: n150, name: "run1", test_names: "stable_diffusion, Qwen, MobileNetV2, clip, flan_t5, mlpmixer, resnet, vilt, albert, codegen, glpn_kitti, mnist, resnet50, RMBG, unet_carvana, mgp-str-base, musicgen_small, segformer, torchvision, yolos"}, - {runs-on: n150, name: "run2", test_names: "t5, whisper, autoencoder_conv, deit, gpt2, mobilenet_ssd, roberta, timm, xglm, autoencoder_linear, detr, beit, distilbert, hand_landmark, openpose, segment_anything, unet, yolov3, bert, dpr, hardnet, opt, speecht5_tts, unet_brain, yolov5, bloom, falcon, llama, perceiver_io, squeeze_bert, gpt_neo"}, + { + runs-on: n150, name: "run1", tests: "\ + tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2 + tests/models/Qwen/test_qwen2_casual_lm.py::test_qwen2_casual_lm + tests/models/Qwen/test_qwen2_token_classification.py::test_qwen2_token_classification + tests/models/RMBG/test_RMBG.py::test_RMBG + tests/models/albert/test_albert_masked_lm.py::test_albert_masked_lm + tests/models/albert/test_albert_question_answering.py::test_albert_question_answering + tests/models/albert/test_albert_sequence_classification.py::test_albert_sequence_classification + tests/models/albert/test_albert_token_classification.py::test_albert_token_classification + tests/models/clip/test_clip.py::test_clip + tests/models/codegen/test_codegen.py::test_codegen + tests/models/flan_t5/test_flan_t5.py::test_flan_t5 + tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti + tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base + tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer + tests/models/mnist/test_mnist.py::test_mnist_train + tests/models/musicgen_small/test_musicgen_small.py::test_musicgen_small + tests/models/resnet/test_resnet.py::test_resnet + tests/models/resnet50/test_resnet50.py::test_resnet + tests/models/segformer/test_segformer.py::test_segformer + tests/models/stable_diffusion/test_stable_diffusion.py::test_stable_diffusion + tests/models/stable_diffusion/test_stable_diffusion_v2.py::test_stable_diffusion_v2 + tests/models/torchvision/test_torchvision_image_classification.py::test_torchvision_image_classification + tests/models/torchvision/test_torchvision_object_detection.py::test_torchvision_object_detection + tests/models/unet_carvana/test_unet_carvana.py::test_unet_carvana + tests/models/vilt/test_vilt.py::test_vilt + tests/models/yolos/test_yolos.py::test_yolos + " + }, + { + runs-on: n150, name: "run2", tests: "\ + tests/models/autoencoder_conv/test_autoencoder_conv.py::test_autoencoder_conv \ + tests/models/autoencoder_conv/test_autoencoder_conv_v2.py::test_autoencoder_conv_v2 \ + tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear \ + tests/models/beit/test_beit_image_classification.py::test_beit_image_classification \ + tests/models/bert/test_bert.py::test_bert \ + tests/models/bloom/test_bloom.py::test_bloom + tests/models/deit/test_deit.py::test_deit \ + tests/models/detr/test_detr.py::test_detr \ + tests/models/distilbert/test_distilbert.py::test_distilbert \ + tests/models/dpr/test_dpr.py::test_dpr \ + tests/models/falcon/test_falcon.py::test_falcon \ + tests/models/gpt2/test_gpt2.py::test_gpt2 \ + tests/models/gpt_neo/test_gpt_neo.py::test_gpt_neo \ + tests/models/hand_landmark/test_hand_landmark.py::test_hand_landmark \ + tests/models/hardnet/test_hardnet.py::test_hardnet \ + tests/models/llama/test_llama.py::test_llama \ + tests/models/mobilenet_ssd/test_mobilenet_ssd.py::test_mobilenet_ssd \ + tests/models/openpose/test_openpose.py::test_openpose \ + tests/models/openpose/test_openpose_v2.py::test_openpose_v2 \ + tests/models/opt/test_opt.py::test_opt \ + tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io \ + tests/models/roberta/test_roberta.py::test_roberta \ + tests/models/segment_anything/test_segment_anything.py::test_segment_anything \ + tests/models/speecht5_tts/test_speecht5_tts.py::test_speecht5_tts \ + tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert \ + tests/models/t5/test_t5.py::test_t5 \ + tests/models/timm/test_timm_image_classification.py::test_timm_image_classification \ + tests/models/unet/test_unet.py::test_unet \ + tests/models/unet_brain/test_unet_brain.py::test_unet_brain \ + tests/models/whisper/test_whisper.py::test_whisper \ + tests/models/xglm/test_xglm.py::test_xglm \ + tests/models/yolov3/test_yolov3.py::test_yolov3 \ + tests/models/yolov5/test_yolov5.py::test_yolov5 \ + " + }, ] runs-on: - ${{ matrix.build.runs-on }} @@ -91,13 +156,7 @@ jobs: echo ${{ steps.strings.outputs.install-output-dir }} source env/activate export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${{ steps.strings.outputs.build-output-dir }}/lib:./lib/:${LD_LIBRARY_PATH}" - PYTESTS_LIST="" - for testname in ${{ matrix.build.test_names }}; do - testname=${testname//,/} - PYTESTS_LIST="${PYTESTS_LIST} tests/models/$testname --nightly" - done - echo "running tests: ${{ matrix.build.test_names }}" - pytest -svv $PYTESTS_LIST + pytest -svv --nightly ${{ matrix.build.tests }} - name: Tar results if: success() || failure() diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 74757af5..b78a70db 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -108,23 +108,23 @@ jobs: source env/activate export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${{ steps.strings.outputs.build-output-dir }}/lib:./lib/:${LD_LIBRARY_PATH}" pytest -v \ - tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear[False-eval] \ - tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[False-microsoft/beit-base-patch16-224-eval] \ - tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[False-microsoft/beit-large-patch16-224-eval] \ - tests/models/detr/test_detr.py::test_detr[False-eval] \ - tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti[False-eval] \ - tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base[False-eval] \ - tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer[False-eval] \ - tests/models/mnist/test_mnist.py::test_mnist_train[False-eval] \ - tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2[False-eval] \ - tests/models/openpose/test_openpose_v2.py::test_openpose_v2[False-eval] \ - tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io[False-eval] \ - tests/models/resnet/test_resnet.py::test_resnet[False-eval] \ - tests/models/resnet50/test_resnet50.py::test_resnet[False-eval] \ - tests/models/segformer/test_segformer.py::test_segformer[False-eval] \ - tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert[False-eval] \ - tests/models/vilt/test_vilt.py::test_vilt[False-eval] \ - tests/models/yolov3/test_yolov3.py::test_yolov3[False-eval] \ + tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear[push-eval] \ + tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[push-microsoft/beit-base-patch16-224-eval] \ + tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[push-microsoft/beit-large-patch16-224-eval] \ + tests/models/detr/test_detr.py::test_detr[push-eval] \ + tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti[push-eval] \ + tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base[push-eval] \ + tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer[push-eval] \ + tests/models/mnist/test_mnist.py::test_mnist_train[push-eval] \ + tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2[push-eval] \ + tests/models/openpose/test_openpose_v2.py::test_openpose_v2[push-eval] \ + tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io[push-eval] \ + tests/models/resnet/test_resnet.py::test_resnet[push-eval] \ + tests/models/resnet50/test_resnet50.py::test_resnet[push-eval] \ + tests/models/segformer/test_segformer.py::test_segformer[push-eval] \ + tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert[push-eval] \ + tests/models/vilt/test_vilt.py::test_vilt[push-eval] \ + tests/models/yolov3/test_yolov3.py::test_yolov3[push-eval] \ --junit-xml=${{ steps.strings.outputs.test_report_path_models }} - name: Upload Test Report Models diff --git a/tests/conftest.py b/tests/conftest.py index 19c9193a..7ba8ccf6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,14 +15,37 @@ def run_around_tests(): def pytest_addoption(parser): parser.addoption( - "--nightly", - action="store_true", - default=False, + "--nightly", action="store_true", default=False, help="Run nightly tests" ) -def pytest_generate_tests(metafunc): - if "nightly" in metafunc.fixturenames: - # Retrieve value of the custom option - value = metafunc.config.getoption("nightly") - metafunc.parametrize("nightly", [value]) +def pytest_collection_modifyitems(config, items): + # If --nightly flag is set, filter out tests with nightly=False + selected_items = [] + for item in items: + # Check if the test has a parameter called 'nightly' + # and whether it is set to True + + if config.getoption("--nightly"): + for param in item.iter_markers(name="parametrize"): + # Check if the parameter is 'nightly' and its value is True + if "nightly" in param.args[0] and item.callspec.params["nightly"]: + selected_items.append(item) + break + else: + # If the test does not have a 'nightly' parameter, + # add all tests without a nightly parameter, as well + # as all testst with a nightly parameter where nightly=False + has_nightly_param = False + for param in item.iter_markers(name="parametrize"): + if "nightly" in param.args[0]: + has_nightly_param = True + # Only add the test if nightly=False + if not item.callspec.params["nightly"]: + selected_items.append(item) + break + # If theres no nigtly parameter, add the test + if not has_nightly_param: + selected_items.append(item) + # Replace the items with only the nightly tests + items[:] = selected_items diff --git a/tests/models/MobileNetV2/test_MobileNetV2.py b/tests/models/MobileNetV2/test_MobileNetV2.py index 87913112..f32fe022 100644 --- a/tests/models/MobileNetV2/test_MobileNetV2.py +++ b/tests/models/MobileNetV2/test_MobileNetV2.py @@ -35,6 +35,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_MobileNetV2(record_property, mode, nightly): model_name = "MobileNetV2" record_property("model_name", model_name) diff --git a/tests/models/Qwen/test_qwen2_casual_lm.py b/tests/models/Qwen/test_qwen2_casual_lm.py index 0bf904b9..bf8da962 100644 --- a/tests/models/Qwen/test_qwen2_casual_lm.py +++ b/tests/models/Qwen/test_qwen2_casual_lm.py @@ -42,6 +42,7 @@ def set_model_eval(self, model): "Qwen/Qwen2.5-1.5B", ], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_qwen2_casual_lm(record_property, model_name, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/Qwen/test_qwen2_token_classification.py b/tests/models/Qwen/test_qwen2_token_classification.py index 6f41de07..60aba5fc 100644 --- a/tests/models/Qwen/test_qwen2_token_classification.py +++ b/tests/models/Qwen/test_qwen2_token_classification.py @@ -35,6 +35,7 @@ def _load_inputs(self): "Qwen/Qwen2-7B", ], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_qwen2_token_classification(record_property, model_name, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/RMBG/test_RMBG.py b/tests/models/RMBG/test_RMBG.py index 556c2587..e39d02cf 100644 --- a/tests/models/RMBG/test_RMBG.py +++ b/tests/models/RMBG/test_RMBG.py @@ -40,6 +40,7 @@ def _load_inputs(self): ["train", "eval"], ) @pytest.mark.xfail(reason="Fails due pt2 compile issue, graph is traced") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_RMBG(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/albert/test_albert_masked_lm.py b/tests/models/albert/test_albert_masked_lm.py index aadfa53f..a319df35 100644 --- a/tests/models/albert/test_albert_masked_lm.py +++ b/tests/models/albert/test_albert_masked_lm.py @@ -48,6 +48,7 @@ def append_fake_loss_function(self, outputs): "albert/albert-xxlarge-v2", ], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_albert_masked_lm(record_property, model_name, mode, nightly): record_property("model_name", model_name) record_property("mode", mode) diff --git a/tests/models/albert/test_albert_question_answering.py b/tests/models/albert/test_albert_question_answering.py index 0af06718..4955f442 100644 --- a/tests/models/albert/test_albert_question_answering.py +++ b/tests/models/albert/test_albert_question_answering.py @@ -31,6 +31,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.parametrize("model_name", ["twmkn9/albert-base-v2-squad2"]) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_albert_question_answering(record_property, model_name, mode, nightly): record_property("model_name", model_name) record_property("mode", mode) diff --git a/tests/models/albert/test_albert_sequence_classification.py b/tests/models/albert/test_albert_sequence_classification.py index 6ee63e13..fc9f518d 100644 --- a/tests/models/albert/test_albert_sequence_classification.py +++ b/tests/models/albert/test_albert_sequence_classification.py @@ -31,6 +31,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.parametrize("model_name", ["textattack/albert-base-v2-imdb"]) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_albert_sequence_classification(record_property, model_name, mode, nightly): record_property("model_name", model_name) record_property("mode", mode) diff --git a/tests/models/albert/test_albert_token_classification.py b/tests/models/albert/test_albert_token_classification.py index c2de0d32..9259bacb 100644 --- a/tests/models/albert/test_albert_token_classification.py +++ b/tests/models/albert/test_albert_token_classification.py @@ -33,6 +33,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.parametrize("model_name", ["albert/albert-base-v2"]) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_albert_token_classification(record_property, model_name, mode, nightly): record_property("model_name", f"{model_name}-classification") record_property("mode", mode) diff --git a/tests/models/autoencoder_conv/test_autoencoder_conv.py b/tests/models/autoencoder_conv/test_autoencoder_conv.py index 15336adc..83f932d7 100644 --- a/tests/models/autoencoder_conv/test_autoencoder_conv.py +++ b/tests/models/autoencoder_conv/test_autoencoder_conv.py @@ -50,6 +50,7 @@ def _load_inputs(self): ["train", "eval"], ) @pytest.mark.skip(reason="PyTorch compilation flow cannot accept pipeline.") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_autoencoder_conv(record_property, mode, nightly): model_name = "Autoencoder (convolutional)" record_property("model_name", model_name) diff --git a/tests/models/autoencoder_conv/test_autoencoder_conv_v2.py b/tests/models/autoencoder_conv/test_autoencoder_conv_v2.py index fdf1a34f..cea748f2 100644 --- a/tests/models/autoencoder_conv/test_autoencoder_conv_v2.py +++ b/tests/models/autoencoder_conv/test_autoencoder_conv_v2.py @@ -75,6 +75,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_autoencoder_conv_v2(record_property, mode, nightly): model_name = f"Autoencoder (conv)" record_property("model_name", model_name) diff --git a/tests/models/autoencoder_linear/test_autoencoder_linear.py b/tests/models/autoencoder_linear/test_autoencoder_linear.py index a2391324..2479d72c 100644 --- a/tests/models/autoencoder_linear/test_autoencoder_linear.py +++ b/tests/models/autoencoder_linear/test_autoencoder_linear.py @@ -84,6 +84,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_autoencoder_linear(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/beit/test_beit_image_classification.py b/tests/models/beit/test_beit_image_classification.py index 53e72a3a..80805856 100644 --- a/tests/models/beit/test_beit_image_classification.py +++ b/tests/models/beit/test_beit_image_classification.py @@ -43,6 +43,7 @@ def get_results_train(self, model, inputs, outputs): "model_name", ["microsoft/beit-base-patch16-224", "microsoft/beit-large-patch16-224"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_beit_image_classification(record_property, model_name, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/bert/test_bert.py b/tests/models/bert/test_bert.py index 7c5a3b23..966eb0c3 100644 --- a/tests/models/bert/test_bert.py +++ b/tests/models/bert/test_bert.py @@ -42,6 +42,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_bert(record_property, mode, nightly): model_name = "BERT" record_property("model_name", model_name) diff --git a/tests/models/bloom/test_bloom.py b/tests/models/bloom/test_bloom.py index 960f0510..ab835d7b 100644 --- a/tests/models/bloom/test_bloom.py +++ b/tests/models/bloom/test_bloom.py @@ -38,6 +38,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_bloom(record_property, mode, nightly): model_name = "Bloom" record_property("model_name", model_name) diff --git a/tests/models/clip/test_clip.py b/tests/models/clip/test_clip.py index 91339a04..1d01382b 100644 --- a/tests/models/clip/test_clip.py +++ b/tests/models/clip/test_clip.py @@ -59,6 +59,7 @@ def get_results_train(self, model, inputs, outputs): "eval", ], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_clip(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/codegen/test_codegen.py b/tests/models/codegen/test_codegen.py index 2f29fb23..d07a40c0 100644 --- a/tests/models/codegen/test_codegen.py +++ b/tests/models/codegen/test_codegen.py @@ -35,6 +35,7 @@ def set_model_eval(self, model): @pytest.mark.xfail( reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph" ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_codegen(record_property, mode, nightly): model_name = "codegen" record_property("model_name", model_name) diff --git a/tests/models/deit/test_deit.py b/tests/models/deit/test_deit.py index 7a5a2586..3b2f5292 100644 --- a/tests/models/deit/test_deit.py +++ b/tests/models/deit/test_deit.py @@ -47,6 +47,7 @@ def get_results_train(self, model, inputs, outputs): ], ) @pytest.mark.parametrize("model_name", ["facebook/deit-base-patch16-224"]) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_deit(record_property, model_name, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/detr/test_detr.py b/tests/models/detr/test_detr.py index 4b33eae9..7e9a0085 100644 --- a/tests/models/detr/test_detr.py +++ b/tests/models/detr/test_detr.py @@ -44,6 +44,7 @@ def _extract_outputs(self, output_object): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_detr(record_property, mode, nightly): model_name = "DETR" record_property("model_name", model_name) diff --git a/tests/models/distilbert/test_distilbert.py b/tests/models/distilbert/test_distilbert.py index c18272fb..20960ad4 100644 --- a/tests/models/distilbert/test_distilbert.py +++ b/tests/models/distilbert/test_distilbert.py @@ -27,6 +27,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.parametrize("model_name", ["distilbert-base-uncased"]) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_distilbert(record_property, model_name, mode, nightly): record_property("model_name", model_name) record_property("mode", mode) diff --git a/tests/models/dpr/test_dpr.py b/tests/models/dpr/test_dpr.py index 4116ab9d..eeb8d532 100644 --- a/tests/models/dpr/test_dpr.py +++ b/tests/models/dpr/test_dpr.py @@ -34,6 +34,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_dpr(record_property, mode, nightly): model_name = "DPR" record_property("model_name", model_name) diff --git a/tests/models/falcon/test_falcon.py b/tests/models/falcon/test_falcon.py index c56cfce6..d66df8bb 100644 --- a/tests/models/falcon/test_falcon.py +++ b/tests/models/falcon/test_falcon.py @@ -31,6 +31,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_falcon(record_property, mode, nightly): model_name = "Falcon" record_property("model_name", model_name) diff --git a/tests/models/flan_t5/test_flan_t5.py b/tests/models/flan_t5/test_flan_t5.py index bbf2791d..297c7ddc 100644 --- a/tests/models/flan_t5/test_flan_t5.py +++ b/tests/models/flan_t5/test_flan_t5.py @@ -35,6 +35,7 @@ def set_model_eval(self, model): @pytest.mark.xfail( reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph" ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_flan_t5(record_property, mode, nightly): model_name = "FLAN-T5" record_property("model_name", model_name) diff --git a/tests/models/glpn_kitti/test_glpn_kitti.py b/tests/models/glpn_kitti/test_glpn_kitti.py index 3761e7c3..7651c90d 100644 --- a/tests/models/glpn_kitti/test_glpn_kitti.py +++ b/tests/models/glpn_kitti/test_glpn_kitti.py @@ -32,6 +32,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_glpn_kitti(record_property, mode, nightly): model_name = "GLPN-KITTI" record_property("model_name", model_name) diff --git a/tests/models/gpt2/test_gpt2.py b/tests/models/gpt2/test_gpt2.py index e936768b..dd3efc13 100644 --- a/tests/models/gpt2/test_gpt2.py +++ b/tests/models/gpt2/test_gpt2.py @@ -33,6 +33,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_gpt2(record_property, mode, nightly): model_name = "GPT-2" record_property("model_name", model_name) diff --git a/tests/models/gpt_neo/test_gpt_neo.py b/tests/models/gpt_neo/test_gpt_neo.py index e788668e..b8087a81 100644 --- a/tests/models/gpt_neo/test_gpt_neo.py +++ b/tests/models/gpt_neo/test_gpt_neo.py @@ -43,6 +43,7 @@ def set_model_eval(self, model): @pytest.mark.xfail( reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph" ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_gpt_neo(record_property, mode, nightly): model_name = "GPTNeo" record_property("model_name", model_name) diff --git a/tests/models/hand_landmark/test_hand_landmark.py b/tests/models/hand_landmark/test_hand_landmark.py index 459692cc..2c533e7f 100644 --- a/tests/models/hand_landmark/test_hand_landmark.py +++ b/tests/models/hand_landmark/test_hand_landmark.py @@ -43,6 +43,7 @@ def set_model_eval(self, model): ["eval"], ) @pytest.mark.xfail(reason="Need to debud") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_hand_landmark(record_property, mode, nightly): model_name = "Hand Landmark" record_property("model_name", model_name) diff --git a/tests/models/hardnet/test_hardnet.py b/tests/models/hardnet/test_hardnet.py index 6a12af2d..ae8837bf 100644 --- a/tests/models/hardnet/test_hardnet.py +++ b/tests/models/hardnet/test_hardnet.py @@ -50,6 +50,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_hardnet(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/llama/test_llama.py b/tests/models/llama/test_llama.py index 502e895b..2602e9d7 100644 --- a/tests/models/llama/test_llama.py +++ b/tests/models/llama/test_llama.py @@ -41,6 +41,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_llama(record_property, mode, nightly): model_name = "Llama" record_property("model_name", model_name) diff --git a/tests/models/mgp-str-base/test_mgp_str_base.py b/tests/models/mgp-str-base/test_mgp_str_base.py index 2936375c..3e39bef6 100644 --- a/tests/models/mgp-str-base/test_mgp_str_base.py +++ b/tests/models/mgp-str-base/test_mgp_str_base.py @@ -37,6 +37,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_mgp_str_base(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/mlpmixer/test_mlpmixer.py b/tests/models/mlpmixer/test_mlpmixer.py index 8cad3181..6b95802b 100644 --- a/tests/models/mlpmixer/test_mlpmixer.py +++ b/tests/models/mlpmixer/test_mlpmixer.py @@ -34,6 +34,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_mlpmixer(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/mnist/test_mnist.py b/tests/models/mnist/test_mnist.py index d90dd060..b1bf4978 100644 --- a/tests/models/mnist/test_mnist.py +++ b/tests/models/mnist/test_mnist.py @@ -60,6 +60,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_mnist_train(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/mobilenet_ssd/test_mobilenet_ssd.py b/tests/models/mobilenet_ssd/test_mobilenet_ssd.py index 7309a1e9..858cde11 100644 --- a/tests/models/mobilenet_ssd/test_mobilenet_ssd.py +++ b/tests/models/mobilenet_ssd/test_mobilenet_ssd.py @@ -39,6 +39,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.xfail(reason="Need to debug") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_mobilenet_ssd(record_property, mode, nightly): model_name = "MobileNetSSD" record_property("model_name", model_name) diff --git a/tests/models/musicgen_small/test_musicgen_small.py b/tests/models/musicgen_small/test_musicgen_small.py index 65987816..ae72b5c1 100644 --- a/tests/models/musicgen_small/test_musicgen_small.py +++ b/tests/models/musicgen_small/test_musicgen_small.py @@ -39,6 +39,7 @@ def set_model_eval(self, model): ["eval"], ) @pytest.mark.skip("torch run with bypass compilation is stalling") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_musicgen_small(record_property, mode, nightly): model_name = "musicgen_small" record_property("model_name", model_name) diff --git a/tests/models/openpose/test_openpose.py b/tests/models/openpose/test_openpose.py index 4a561f8d..062439d8 100644 --- a/tests/models/openpose/test_openpose.py +++ b/tests/models/openpose/test_openpose.py @@ -35,6 +35,7 @@ def _load_inputs(self): ) @pytest.mark.usefixtures("manage_dependencies") @pytest.mark.skip(reason="failing during torch run with bypass compilation") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_openpose(record_property, mode, nightly): model_name = "OpenPose" record_property("model_name", model_name) diff --git a/tests/models/openpose/test_openpose_v2.py b/tests/models/openpose/test_openpose_v2.py index 7700e9e8..92b40384 100644 --- a/tests/models/openpose/test_openpose_v2.py +++ b/tests/models/openpose/test_openpose_v2.py @@ -50,6 +50,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_openpose_v2(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/opt/test_opt.py b/tests/models/opt/test_opt.py index a9f108d8..f6c8eaad 100644 --- a/tests/models/opt/test_opt.py +++ b/tests/models/opt/test_opt.py @@ -37,6 +37,7 @@ def set_model_eval(self, model): ["eval"], ) @pytest.mark.xfail(reason="Need to debug") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_opt(record_property, mode, nightly): model_name = "OPT" record_property("model_name", model_name) diff --git a/tests/models/perceiver_io/test_perceiver_io.py b/tests/models/perceiver_io/test_perceiver_io.py index 2de9d5f8..c5354ba1 100644 --- a/tests/models/perceiver_io/test_perceiver_io.py +++ b/tests/models/perceiver_io/test_perceiver_io.py @@ -39,6 +39,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_perceiver_io(record_property, mode, nightly): model_name = "Perceiver IO" record_property("model_name", model_name) diff --git a/tests/models/resnet/test_resnet.py b/tests/models/resnet/test_resnet.py index e450669a..f0b652e6 100644 --- a/tests/models/resnet/test_resnet.py +++ b/tests/models/resnet/test_resnet.py @@ -24,6 +24,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_resnet(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/resnet50/test_resnet50.py b/tests/models/resnet50/test_resnet50.py index 1de995ac..d0af010f 100644 --- a/tests/models/resnet50/test_resnet50.py +++ b/tests/models/resnet50/test_resnet50.py @@ -37,6 +37,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_resnet(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/roberta/test_roberta.py b/tests/models/roberta/test_roberta.py index 58fb5db7..7ea1249b 100644 --- a/tests/models/roberta/test_roberta.py +++ b/tests/models/roberta/test_roberta.py @@ -27,6 +27,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_roberta(record_property, mode, nightly): model_name = "RoBERTa" record_property("model_name", model_name) diff --git a/tests/models/segformer/test_segformer.py b/tests/models/segformer/test_segformer.py index 92127058..0d92a6a8 100644 --- a/tests/models/segformer/test_segformer.py +++ b/tests/models/segformer/test_segformer.py @@ -45,6 +45,7 @@ def get_results_train(self, model, inputs, outputs): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_segformer(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/segment_anything/test_segment_anything.py b/tests/models/segment_anything/test_segment_anything.py index d159a0ea..9b55ffef 100644 --- a/tests/models/segment_anything/test_segment_anything.py +++ b/tests/models/segment_anything/test_segment_anything.py @@ -38,6 +38,7 @@ def run_model(self, model, inputs): @pytest.mark.skip( reason="Failed to install sam2. sam2 requires Python >=3.10.0 but the default version on Ubuntu 20.04 is 3.8. We found no other pytorch implementation of segment-anything." ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_segment_anything(record_property, mode, nightly): model_name = "segment-anything" record_property("model_name", model_name) diff --git a/tests/models/speecht5_tts/test_speecht5_tts.py b/tests/models/speecht5_tts/test_speecht5_tts.py index 1bc46e43..1238c84b 100644 --- a/tests/models/speecht5_tts/test_speecht5_tts.py +++ b/tests/models/speecht5_tts/test_speecht5_tts.py @@ -48,6 +48,7 @@ def set_model_eval(self, model): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_speecht5_tts(record_property, mode, nightly): pytest.skip() # crashes in lowering to stable hlo model_name = "speecht5-tts" diff --git a/tests/models/squeeze_bert/test_squeeze_bert.py b/tests/models/squeeze_bert/test_squeeze_bert.py index f0d33bdf..3701767a 100644 --- a/tests/models/squeeze_bert/test_squeeze_bert.py +++ b/tests/models/squeeze_bert/test_squeeze_bert.py @@ -29,6 +29,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_squeeze_bert(record_property, mode, nightly): model_name = "SqueezeBERT" record_property("model_name", model_name) diff --git a/tests/models/stable_diffusion/test_stable_diffusion.py b/tests/models/stable_diffusion/test_stable_diffusion.py index 77b97658..39a6c6c8 100644 --- a/tests/models/stable_diffusion/test_stable_diffusion.py +++ b/tests/models/stable_diffusion/test_stable_diffusion.py @@ -24,6 +24,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.skip(reason="Dynamo cannot support pipeline.") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_stable_diffusion(record_property, mode, nightly): model_name = "Stable Diffusion" record_property("model_name", model_name) diff --git a/tests/models/stable_diffusion/test_stable_diffusion_v2.py b/tests/models/stable_diffusion/test_stable_diffusion_v2.py index 4a8811b0..e2aafad1 100644 --- a/tests/models/stable_diffusion/test_stable_diffusion_v2.py +++ b/tests/models/stable_diffusion/test_stable_diffusion_v2.py @@ -64,6 +64,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_stable_diffusion_v2(record_property, mode, nightly): model_name = "Stable Diffusion V2" record_property("model_name", model_name) diff --git a/tests/models/t5/test_t5.py b/tests/models/t5/test_t5.py index bbe19628..61193074 100644 --- a/tests/models/t5/test_t5.py +++ b/tests/models/t5/test_t5.py @@ -33,6 +33,7 @@ def set_model_eval(self, model): reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph" ) @pytest.mark.parametrize("model_name", ["t5-small", "t5-base", "t5-large"]) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_t5(record_property, model_name, mode, nightly): record_property("model_name", model_name) record_property("mode", mode) diff --git a/tests/models/timm/test_timm_image_classification.py b/tests/models/timm/test_timm_image_classification.py index 04739bb8..c76b5c05 100644 --- a/tests/models/timm/test_timm_image_classification.py +++ b/tests/models/timm/test_timm_image_classification.py @@ -107,7 +107,8 @@ def _load_inputs(self): @pytest.mark.skip # skipped due to missing manage_dependencies package @pytest.mark.usefixtures("manage_dependencies") @pytest.mark.parametrize("model_and_mode", model_and_mode_list) -def test_timm_image_classification(record_property, model_and_mode): +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) +def test_timm_image_classification(record_property, model_and_mode, nightly): model_name = model_and_mode[0] mode = model_and_mode[1] record_property("model_name", model_name) diff --git a/tests/models/torchvision/test_torchvision_image_classification.py b/tests/models/torchvision/test_torchvision_image_classification.py index a22e23f3..41caf26c 100644 --- a/tests/models/torchvision/test_torchvision_image_classification.py +++ b/tests/models/torchvision/test_torchvision_image_classification.py @@ -7,6 +7,7 @@ import requests import pytest from tests.utils import ModelTester +from tt_torch.tools.utils import CompilerConfig, CompileDepth class ThisTester(ModelTester): @@ -123,7 +124,10 @@ def _load_inputs(self): @pytest.mark.parametrize("model_info_and_mode", model_info_and_mode_list) -def test_torchvision_image_classification(record_property, model_info_and_mode): +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) +def test_torchvision_image_classification( + record_property, model_info_and_mode, nightly +): pytest.skip("torchvision modules not supported.") model_info = model_info_and_mode[0] mode = model_info_and_mode[1] @@ -131,7 +135,13 @@ def test_torchvision_image_classification(record_property, model_info_and_mode): record_property("model_name", model_name) record_property("mode", mode) - tester = ThisTester(model_info, mode) + cc = CompilerConfig() + cc.enable_consteval = True + cc.consteval_parameters = True + if nightly: + cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP + + tester = ThisTester(model_info, mode, compiler_config=cc) results = tester.test_model() if mode == "eval": # Print the top 5 predictions diff --git a/tests/models/torchvision/test_torchvision_object_detection.py b/tests/models/torchvision/test_torchvision_object_detection.py index ed4530bf..ec4ae799 100644 --- a/tests/models/torchvision/test_torchvision_object_detection.py +++ b/tests/models/torchvision/test_torchvision_object_detection.py @@ -7,6 +7,7 @@ import requests import pytest from tests.utils import ModelTester +from tt_torch.tools.utils import CompilerConfig, CompileDepth # TODO: RuntimeError: "nms_kernel" not implemented for 'BFloat16' @@ -51,13 +52,20 @@ def _load_inputs(self): ("retinanet_resnet50_fpn_v2", "RetinaNet_ResNet50_FPN_V2_Weights"), ], ) -def test_torchvision_object_detection(record_property, model_info, mode): +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) +def test_torchvision_object_detection(record_property, model_info, mode, nightly): pytest.skip("torchvision modules not supported.") model_name, _ = model_info record_property("model_name", model_name) record_property("mode", mode) - tester = ThisTester(model_info, mode) + cc = CompilerConfig() + cc.enable_consteval = True + cc.consteval_parameters = True + if nightly: + cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP + + tester = ThisTester(model_info, mode, compiler_config=cc) results = tester.test_model() if mode == "eval": print(f"Model: {model_name} | Output: {results}") diff --git a/tests/models/unet/test_unet.py b/tests/models/unet/test_unet.py index c15c6a2c..9cfdac28 100644 --- a/tests/models/unet/test_unet.py +++ b/tests/models/unet/test_unet.py @@ -46,6 +46,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_unet(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/unet_brain/test_unet_brain.py b/tests/models/unet_brain/test_unet_brain.py index 66c39564..fa9e0431 100644 --- a/tests/models/unet_brain/test_unet_brain.py +++ b/tests/models/unet_brain/test_unet_brain.py @@ -56,6 +56,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_unet_brain(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/unet_carvana/test_unet_carvana.py b/tests/models/unet_carvana/test_unet_carvana.py index dcfcfbe3..b371f094 100644 --- a/tests/models/unet_carvana/test_unet_carvana.py +++ b/tests/models/unet_carvana/test_unet_carvana.py @@ -32,6 +32,7 @@ def _load_inputs(self): "mode", ["train", "eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_unet_carvana(record_property, mode, nightly): if mode == "train": pytest.skip() diff --git a/tests/models/vilt/test_vilt.py b/tests/models/vilt/test_vilt.py index 9347969f..94b97b8b 100644 --- a/tests/models/vilt/test_vilt.py +++ b/tests/models/vilt/test_vilt.py @@ -37,6 +37,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_vilt(record_property, mode, nightly): model_name = "ViLT" record_property("model_name", model_name) diff --git a/tests/models/whisper/test_whisper.py b/tests/models/whisper/test_whisper.py index 4a15c16b..f5f85c16 100644 --- a/tests/models/whisper/test_whisper.py +++ b/tests/models/whisper/test_whisper.py @@ -52,6 +52,7 @@ def set_model_eval(self, model): @pytest.mark.xfail( reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph" ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_whisper(record_property, mode, nightly): model_name = "Whisper" record_property("model_name", model_name) diff --git a/tests/models/xglm/test_xglm.py b/tests/models/xglm/test_xglm.py index d821b1e1..70fa1bdf 100644 --- a/tests/models/xglm/test_xglm.py +++ b/tests/models/xglm/test_xglm.py @@ -33,6 +33,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_xglm(record_property, mode, nightly): model_name = "XGLM" record_property("model_name", model_name) diff --git a/tests/models/yolos/test_yolos.py b/tests/models/yolos/test_yolos.py index 1b73d661..526ec802 100644 --- a/tests/models/yolos/test_yolos.py +++ b/tests/models/yolos/test_yolos.py @@ -37,6 +37,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_yolos(record_property, mode, nightly): model_name = "YOLOS" record_property("model_name", model_name) diff --git a/tests/models/yolov3/test_yolov3.py b/tests/models/yolov3/test_yolov3.py index 53df0959..a93f333b 100644 --- a/tests/models/yolov3/test_yolov3.py +++ b/tests/models/yolov3/test_yolov3.py @@ -57,6 +57,7 @@ def _load_inputs(self): "mode", ["eval"], ) +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_yolov3(record_property, mode, nightly): model_name = "YOLOv3" record_property("model_name", model_name) diff --git a/tests/models/yolov5/test_yolov5.py b/tests/models/yolov5/test_yolov5.py index f01ba824..0b6ae4a4 100644 --- a/tests/models/yolov5/test_yolov5.py +++ b/tests/models/yolov5/test_yolov5.py @@ -108,6 +108,7 @@ def _load_inputs(self): ["eval"], ) @pytest.mark.xfail(reason="Fails due to pt2 compile issue") +@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"]) def test_yolov5(record_property, mode, nightly): model_name = "YOLOv5" record_property("model_name", model_name)