diff --git a/.github/workflows/test_inc_examples.yml b/.github/workflows/test_inc_examples.yml new file mode 100644 index 0000000000..8e80474d98 --- /dev/null +++ b/.github/workflows/test_inc_examples.yml @@ -0,0 +1,47 @@ +name: INC - Examples Test + +on: + workflow_dispatch: + schedule: + - cron: 0 1 * * 1 # run weekly: every Monday at 1am + push: + paths: + - '.github/workflows/test_inc_examples.yml' + - 'examples/neural_compressor/*' + pull_request: + paths: + - '.github/workflows/test_inc_examples.yml' + - 'examples/neural_compressor/*' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.10"] + + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip install optimum[neural-compressor] pytest==8.0.0 + pip install -r examples/neural_compressor/text-classification/requirements.txt + pip install -r examples/neural_compressor/question-answering/requirements.txt + pip install -r examples/neural_compressor/token-classification/requirements.txt + pip install -r examples/neural_compressor/language-modeling/requirements.txt + pip install -r examples/neural_compressor/multiple-choice/requirements.txt + + - name: Test examples + run: | + python -m pytest examples/neural_compressor/test_examples.py diff --git a/examples/neural_compressor/test_examples.py b/examples/neural_compressor/test_examples.py index 89779ffea3..f7e91054e9 100644 --- a/examples/neural_compressor/test_examples.py +++ b/examples/neural_compressor/test_examples.py @@ -59,7 +59,7 @@ def test_run_glue(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_glue.py - --model_name_or_path distilbert-base-uncased-finetuned-sst-2-english + --model_name_or_path hf-internal-testing/tiny-random-DistilBertForSequenceClassification --task_name sst2 --apply_quantization --apply_pruning @@ -78,14 +78,13 @@ def test_run_glue(self): with patch.object(sys, "argv", test_args): run_glue.main() - results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_accuracy"], 0.70) + get_results(tmp_dir) def test_run_qa(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_qa.py - --model_name_or_path distilbert-base-uncased-distilled-squad + --model_name_or_path hf-internal-testing/tiny-random-DistilBertForQuestionAnswering --dataset_name squad --apply_quantization --apply_pruning @@ -104,15 +103,13 @@ def test_run_qa(self): with patch.object(sys, "argv", test_args): run_qa.main() - results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_f1"], 70) - self.assertGreaterEqual(results["eval_exact_match"], 70) + get_results(tmp_dir) def test_run_ner(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_ner.py - --model_name_or_path elastic/distilbert-base-uncased-finetuned-conll03-english + --model_name_or_path hf-internal-testing/tiny-random-RobertaForTokenClassification --dataset_name conll2003 --apply_quantization --apply_pruning @@ -131,17 +128,13 @@ def test_run_ner(self): with patch.object(sys, "argv", test_args): run_ner.main() - results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_accuracy"], 0.70) - self.assertGreaterEqual(results["eval_f1"], 0.70) - self.assertGreaterEqual(results["eval_precision"], 0.70) - self.assertGreaterEqual(results["eval_recall"], 0.70) + get_results(tmp_dir) def test_run_swag(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_swag.py - --model_name_or_path ehdwns1516/bert-base-uncased_SWAG + --model_name_or_path hf-internal-testing/tiny-random-AlbertForMultipleChoice --apply_quantization --apply_pruning --target_sparsity 0.02 @@ -159,8 +152,7 @@ def test_run_swag(self): with patch.object(sys, "argv", test_args): run_swag.main() - results = get_results(tmp_dir) - self.assertGreaterEqual(results["eval_accuracy"], 0.60) + get_results(tmp_dir) def test_run_clm(self): quantization_approach = "dynamic" @@ -168,7 +160,7 @@ def test_run_clm(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_clm.py - --model_name_or_path EleutherAI/gpt-neo-125M + --model_name_or_path hf-internal-testing/tiny-random-GPT2LMHeadModel --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --apply_quantization @@ -189,8 +181,7 @@ def test_run_clm(self): with patch.object(sys, "argv", test_args): run_clm.main() - results = get_results(tmp_dir) - self.assertLessEqual(results["eval_loss"], 15) + get_results(tmp_dir) def test_run_mlm(self): quantization_approach = "static" @@ -198,7 +189,7 @@ def test_run_mlm(self): with tempfile.TemporaryDirectory() as tmp_dir: test_args = f""" run_mlm.py - --model_name_or_path google/electra-small-discriminator + --model_name_or_path hf-internal-testing/tiny-random-DistilBertForMaskedLM --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --apply_quantization @@ -219,8 +210,7 @@ def test_run_mlm(self): with patch.object(sys, "argv", test_args): run_mlm.main() - results = get_results(tmp_dir) - self.assertLessEqual(results["eval_loss"], 15) + get_results(tmp_dir) if __name__ == "__main__":