Add nightly explicit parametrization, update nightly flag: its presen…

…ce means only collect nightly tests, lacktherof means collect everything but nightly tests.

.github/workflows/run-model-tests.yml

@@ -14,8 +14,73 @@ jobs:
       fail-fast: false
       matrix:
         build: [
-          {runs-on: n150, name: "run1", test_names: "stable_diffusion, Qwen, MobileNetV2, clip, flan_t5, mlpmixer, resnet, vilt, albert, codegen, glpn_kitti, mnist, resnet50, RMBG, unet_carvana, mgp-str-base, musicgen_small, segformer, torchvision, yolos"},
-          {runs-on: n150, name: "run2", test_names: "t5, whisper, autoencoder_conv, deit, gpt2, mobilenet_ssd, roberta, timm, xglm, autoencoder_linear, detr, beit, distilbert, hand_landmark, openpose, segment_anything, unet, yolov3, bert, dpr, hardnet, opt, speecht5_tts, unet_brain, yolov5, bloom, falcon, llama, perceiver_io, squeeze_bert, gpt_neo"},
+          {
+            runs-on: n150, name: "run1", tests: "\
+              tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2
+              tests/models/Qwen/test_qwen2_casual_lm.py::test_qwen2_casual_lm
+              tests/models/Qwen/test_qwen2_token_classification.py::test_qwen2_token_classification
+              tests/models/RMBG/test_RMBG.py::test_RMBG
+              tests/models/albert/test_albert_masked_lm.py::test_albert_masked_lm
+              tests/models/albert/test_albert_question_answering.py::test_albert_question_answering
+              tests/models/albert/test_albert_sequence_classification.py::test_albert_sequence_classification
+              tests/models/albert/test_albert_token_classification.py::test_albert_token_classification
+              tests/models/clip/test_clip.py::test_clip
+              tests/models/codegen/test_codegen.py::test_codegen
+              tests/models/flan_t5/test_flan_t5.py::test_flan_t5
+              tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti
+              tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base
+              tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer
+              tests/models/mnist/test_mnist.py::test_mnist_train
+              tests/models/musicgen_small/test_musicgen_small.py::test_musicgen_small
+              tests/models/resnet/test_resnet.py::test_resnet
+              tests/models/resnet50/test_resnet50.py::test_resnet
+              tests/models/segformer/test_segformer.py::test_segformer
+              tests/models/stable_diffusion/test_stable_diffusion.py::test_stable_diffusion
+              tests/models/stable_diffusion/test_stable_diffusion_v2.py::test_stable_diffusion_v2
+              tests/models/torchvision/test_torchvision_image_classification.py::test_torchvision_image_classification
+              tests/models/torchvision/test_torchvision_object_detection.py::test_torchvision_object_detection
+              tests/models/unet_carvana/test_unet_carvana.py::test_unet_carvana
+              tests/models/vilt/test_vilt.py::test_vilt
+              tests/models/yolos/test_yolos.py::test_yolos
+              "
+          },
+          {
+            runs-on: n150, name: "run2", tests: "\
+              tests/models/autoencoder_conv/test_autoencoder_conv.py::test_autoencoder_conv \
+              tests/models/autoencoder_conv/test_autoencoder_conv_v2.py::test_autoencoder_conv_v2 \
+              tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear \
+              tests/models/beit/test_beit_image_classification.py::test_beit_image_classification \
+              tests/models/bert/test_bert.py::test_bert \
+              tests/models/bloom/test_bloom.py::test_bloom
+              tests/models/deit/test_deit.py::test_deit \
+              tests/models/detr/test_detr.py::test_detr \
+              tests/models/distilbert/test_distilbert.py::test_distilbert \
+              tests/models/dpr/test_dpr.py::test_dpr \
+              tests/models/falcon/test_falcon.py::test_falcon \
+              tests/models/gpt2/test_gpt2.py::test_gpt2 \
+              tests/models/gpt_neo/test_gpt_neo.py::test_gpt_neo \
+              tests/models/hand_landmark/test_hand_landmark.py::test_hand_landmark \
+              tests/models/hardnet/test_hardnet.py::test_hardnet \
+              tests/models/llama/test_llama.py::test_llama \
+              tests/models/mobilenet_ssd/test_mobilenet_ssd.py::test_mobilenet_ssd \
+              tests/models/openpose/test_openpose.py::test_openpose \
+              tests/models/openpose/test_openpose_v2.py::test_openpose_v2 \
+              tests/models/opt/test_opt.py::test_opt \
+              tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io \
+              tests/models/roberta/test_roberta.py::test_roberta \
+              tests/models/segment_anything/test_segment_anything.py::test_segment_anything \
+              tests/models/speecht5_tts/test_speecht5_tts.py::test_speecht5_tts \
+              tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert \
+              tests/models/t5/test_t5.py::test_t5 \
+              tests/models/timm/test_timm_image_classification.py::test_timm_image_classification \
+              tests/models/unet/test_unet.py::test_unet \
+              tests/models/unet_brain/test_unet_brain.py::test_unet_brain \
+              tests/models/whisper/test_whisper.py::test_whisper \
+              tests/models/xglm/test_xglm.py::test_xglm \
+              tests/models/yolov3/test_yolov3.py::test_yolov3 \
+              tests/models/yolov5/test_yolov5.py::test_yolov5 \
+              "
+          },
         ]
     runs-on:
       - ${{ matrix.build.runs-on }}
@@ -91,13 +156,7 @@ jobs:
         echo ${{ steps.strings.outputs.install-output-dir }}
         source env/activate
         export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${{ steps.strings.outputs.build-output-dir }}/lib:./lib/:${LD_LIBRARY_PATH}"
-        PYTESTS_LIST=""
-        for testname in ${{ matrix.build.test_names }}; do
-        testname=${testname//,/}
-        PYTESTS_LIST="${PYTESTS_LIST} tests/models/$testname --nightly"
-        done
-        echo "running tests: ${{ matrix.build.test_names }}"
-        pytest -svv $PYTESTS_LIST
+        pytest -svv --nightly ${{ matrix.build.tests }}
 
     - name: Tar results
       if: success() || failure()

.github/workflows/run-tests.yml

@@ -108,23 +108,23 @@ jobs:
         source env/activate
         export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${{ steps.strings.outputs.build-output-dir }}/lib:./lib/:${LD_LIBRARY_PATH}"
         pytest -v \
-                  tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear[False-eval] \
-                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[False-microsoft/beit-base-patch16-224-eval] \
-                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[False-microsoft/beit-large-patch16-224-eval] \
-                  tests/models/detr/test_detr.py::test_detr[False-eval] \
-                  tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti[False-eval] \
-                  tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base[False-eval] \
-                  tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer[False-eval] \
-                  tests/models/mnist/test_mnist.py::test_mnist_train[False-eval] \
-                  tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2[False-eval] \
-                  tests/models/openpose/test_openpose_v2.py::test_openpose_v2[False-eval] \
-                  tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io[False-eval] \
-                  tests/models/resnet/test_resnet.py::test_resnet[False-eval] \
-                  tests/models/resnet50/test_resnet50.py::test_resnet[False-eval] \
-                  tests/models/segformer/test_segformer.py::test_segformer[False-eval] \
-                  tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert[False-eval] \
-                  tests/models/vilt/test_vilt.py::test_vilt[False-eval] \
-                  tests/models/yolov3/test_yolov3.py::test_yolov3[False-eval] \
+                  tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear[push-eval] \
+                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[push-microsoft/beit-base-patch16-224-eval] \
+                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[push-microsoft/beit-large-patch16-224-eval] \
+                  tests/models/detr/test_detr.py::test_detr[push-eval] \
+                  tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti[push-eval] \
+                  tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base[push-eval] \
+                  tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer[push-eval] \
+                  tests/models/mnist/test_mnist.py::test_mnist_train[push-eval] \
+                  tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2[push-eval] \
+                  tests/models/openpose/test_openpose_v2.py::test_openpose_v2[push-eval] \
+                  tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io[push-eval] \
+                  tests/models/resnet/test_resnet.py::test_resnet[push-eval] \
+                  tests/models/resnet50/test_resnet50.py::test_resnet[push-eval] \
+                  tests/models/segformer/test_segformer.py::test_segformer[push-eval] \
+                  tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert[push-eval] \
+                  tests/models/vilt/test_vilt.py::test_vilt[push-eval] \
+                  tests/models/yolov3/test_yolov3.py::test_yolov3[push-eval] \
            --junit-xml=${{ steps.strings.outputs.test_report_path_models }}
 
     - name: Upload Test Report Models

tests/conftest.py

@@ -15,14 +15,37 @@ def run_around_tests():
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--nightly",
-        action="store_true",
-        default=False,
+        "--nightly", action="store_true", default=False, help="Run nightly tests"
     )
 
 
-def pytest_generate_tests(metafunc):
-    if "nightly" in metafunc.fixturenames:
-        # Retrieve value of the custom option
-        value = metafunc.config.getoption("nightly")
-        metafunc.parametrize("nightly", [value])
+def pytest_collection_modifyitems(config, items):
+    # If --nightly flag is set, filter out tests with nightly=False
+    selected_items = []
+    for item in items:
+        # Check if the test has a parameter called 'nightly'
+        # and whether it is set to True
+
+        if config.getoption("--nightly"):
+            for param in item.iter_markers(name="parametrize"):
+                # Check if the parameter is 'nightly' and its value is True
+                if "nightly" in param.args[0] and item.callspec.params["nightly"]:
+                    selected_items.append(item)
+                    break
+        else:
+            # If the test does not have a 'nightly' parameter,
+            # add all tests without a nightly parameter, as well
+            # as all testst with a nightly parameter where nightly=False
+            has_nightly_param = False
+            for param in item.iter_markers(name="parametrize"):
+                if "nightly" in param.args[0]:
+                    has_nightly_param = True
+                    # Only add the test if nightly=False
+                    if not item.callspec.params["nightly"]:
+                        selected_items.append(item)
+                        break
+            # If theres no nigtly parameter, add the test
+            if not has_nightly_param:
+                selected_items.append(item)
+        # Replace the items with only the nightly tests
+    items[:] = selected_items

tests/models/MobileNetV2/test_MobileNetV2.py

@@ -35,6 +35,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_MobileNetV2(record_property, mode, nightly):
     model_name = "MobileNetV2"
     record_property("model_name", model_name)

tests/models/Qwen/test_qwen2_casual_lm.py

@@ -42,6 +42,7 @@ def set_model_eval(self, model):
         "Qwen/Qwen2.5-1.5B",
     ],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_qwen2_casual_lm(record_property, model_name, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/Qwen/test_qwen2_token_classification.py

@@ -35,6 +35,7 @@ def _load_inputs(self):
         "Qwen/Qwen2-7B",
     ],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_qwen2_token_classification(record_property, model_name, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/RMBG/test_RMBG.py

@@ -40,6 +40,7 @@ def _load_inputs(self):
     ["train", "eval"],
 )
 @pytest.mark.xfail(reason="Fails due pt2 compile issue, graph is traced")
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_RMBG(record_property, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/albert/test_albert_masked_lm.py

@@ -48,6 +48,7 @@ def append_fake_loss_function(self, outputs):
         "albert/albert-xxlarge-v2",
     ],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_albert_masked_lm(record_property, model_name, mode, nightly):
     record_property("model_name", model_name)
     record_property("mode", mode)

tests/models/albert/test_albert_question_answering.py

@@ -31,6 +31,7 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["twmkn9/albert-base-v2-squad2"])
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_albert_question_answering(record_property, model_name, mode, nightly):
     record_property("model_name", model_name)
     record_property("mode", mode)

tests/models/albert/test_albert_sequence_classification.py

@@ -31,6 +31,7 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["textattack/albert-base-v2-imdb"])
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_albert_sequence_classification(record_property, model_name, mode, nightly):
     record_property("model_name", model_name)
     record_property("mode", mode)

tests/models/albert/test_albert_token_classification.py

@@ -33,6 +33,7 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["albert/albert-base-v2"])
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_albert_token_classification(record_property, model_name, mode, nightly):
     record_property("model_name", f"{model_name}-classification")
     record_property("mode", mode)

tests/models/autoencoder_conv/test_autoencoder_conv.py

@@ -50,6 +50,7 @@ def _load_inputs(self):
     ["train", "eval"],
 )
 @pytest.mark.skip(reason="PyTorch compilation flow cannot accept pipeline.")
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_autoencoder_conv(record_property, mode, nightly):
     model_name = "Autoencoder (convolutional)"
     record_property("model_name", model_name)

tests/models/autoencoder_conv/test_autoencoder_conv_v2.py

@@ -75,6 +75,7 @@ def _load_inputs(self):
     "mode",
     ["train", "eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_autoencoder_conv_v2(record_property, mode, nightly):
     model_name = f"Autoencoder (conv)"
     record_property("model_name", model_name)

tests/models/autoencoder_linear/test_autoencoder_linear.py

@@ -84,6 +84,7 @@ def _load_inputs(self):
     "mode",
     ["train", "eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_autoencoder_linear(record_property, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/beit/test_beit_image_classification.py

@@ -43,6 +43,7 @@ def get_results_train(self, model, inputs, outputs):
     "model_name",
     ["microsoft/beit-base-patch16-224", "microsoft/beit-large-patch16-224"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_beit_image_classification(record_property, model_name, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/bert/test_bert.py

@@ -42,6 +42,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_bert(record_property, mode, nightly):
     model_name = "BERT"
     record_property("model_name", model_name)

tests/models/bloom/test_bloom.py

@@ -38,6 +38,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_bloom(record_property, mode, nightly):
     model_name = "Bloom"
     record_property("model_name", model_name)

tests/models/clip/test_clip.py

@@ -59,6 +59,7 @@ def get_results_train(self, model, inputs, outputs):
         "eval",
     ],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_clip(record_property, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/codegen/test_codegen.py

@@ -35,6 +35,7 @@ def set_model_eval(self, model):
 @pytest.mark.xfail(
     reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph"
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_codegen(record_property, mode, nightly):
     model_name = "codegen"
     record_property("model_name", model_name)

tests/models/deit/test_deit.py

@@ -47,6 +47,7 @@ def get_results_train(self, model, inputs, outputs):
     ],
 )
 @pytest.mark.parametrize("model_name", ["facebook/deit-base-patch16-224"])
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_deit(record_property, model_name, mode, nightly):
     if mode == "train":
         pytest.skip()

tests/models/detr/test_detr.py

@@ -44,6 +44,7 @@ def _extract_outputs(self, output_object):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_detr(record_property, mode, nightly):
     model_name = "DETR"
     record_property("model_name", model_name)

tests/models/distilbert/test_distilbert.py

@@ -27,6 +27,7 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["distilbert-base-uncased"])
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_distilbert(record_property, model_name, mode, nightly):
     record_property("model_name", model_name)
     record_property("mode", mode)

tests/models/dpr/test_dpr.py

@@ -34,6 +34,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_dpr(record_property, mode, nightly):
     model_name = "DPR"
     record_property("model_name", model_name)

tests/models/falcon/test_falcon.py

@@ -31,6 +31,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_falcon(record_property, mode, nightly):
     model_name = "Falcon"
     record_property("model_name", model_name)

tests/models/flan_t5/test_flan_t5.py

@@ -35,6 +35,7 @@ def set_model_eval(self, model):
 @pytest.mark.xfail(
     reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph"
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_flan_t5(record_property, mode, nightly):
     model_name = "FLAN-T5"
     record_property("model_name", model_name)

tests/models/glpn_kitti/test_glpn_kitti.py

@@ -32,6 +32,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_glpn_kitti(record_property, mode, nightly):
     model_name = "GLPN-KITTI"
     record_property("model_name", model_name)

tests/models/gpt2/test_gpt2.py

@@ -33,6 +33,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_gpt2(record_property, mode, nightly):
     model_name = "GPT-2"
     record_property("model_name", model_name)

tests/models/gpt_neo/test_gpt_neo.py

@@ -43,6 +43,7 @@ def set_model_eval(self, model):
 @pytest.mark.xfail(
     reason="Fails due to pt2 compile issue when finishing generation, but we can still generate a graph"
 )
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_gpt_neo(record_property, mode, nightly):
     model_name = "GPTNeo"
     record_property("model_name", model_name)

tests/models/hand_landmark/test_hand_landmark.py

@@ -43,6 +43,7 @@ def set_model_eval(self, model):
     ["eval"],
 )
 @pytest.mark.xfail(reason="Need to debud")
+@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
 def test_hand_landmark(record_property, mode, nightly):
     model_name = "Hand Landmark"
     record_property("model_name", model_name)