ci: update Transformers to v4.49.0

Benchmarking scripts are pruned from Transformers by v4.49.0 due to deprecation. So we don't need to test them anymore. Some cuda specific tests were generalized to cover non-cuda devices which uncovered some issues. Some new tests were added which fail for both cuda and xpu. Signed-off-by: Dmitry Rogozhkin <[email protected]>
intel · Feb 20, 2025 · 8eea727 · 8eea727
1 parent 414281f
commit 8eea727
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 28 deletions.
diff --git a/.github/scripts/check-transformers.py b/.github/scripts/check-transformers.py
@@ -7,13 +7,16 @@
 parser.add_argument('junitxml', nargs='+')
 args = parser.parse_args()
 
-benchmark_failures = {
-    'link': 'https://github.com/huggingface/transformers/pull/35620',
+layernorm_accuracy_failures = {
+    'link': 'https://github.com/pytorch/pytorch/issues/141642',
     'cuda': 'passed',
 }
 
-layernorm_accuracy_failures = {
-    'link': 'https://github.com/pytorch/pytorch/issues/141642',
+# Tests were enabled for non-cuda backends by v4.49.0 (previously were
+# skipped for xpu):
+# https://github.com/huggingface/transformers/commit/2fa876d2d824123b80ced9d689f75a153731769b
+test_cpu_offload_failures = {
+    'link': 'https://github.com/huggingface/accelerate/issues/3402',
     'cuda': 'passed',
 }
 
@@ -30,21 +33,6 @@
 #   }
 # Use None if no metadata is needed.
 failing_cases = {
-    'tests.benchmark.test_benchmark.BenchmarkTest': {
-        'test_inference_encoder_decoder_with_configs': benchmark_failures,
-        'test_inference_fp16': benchmark_failures,
-        'test_inference_no_configs': benchmark_failures,
-        'test_inference_no_configs_only_pretrain': benchmark_failures,
-        'test_inference_no_model_no_architectures': benchmark_failures,
-        'test_inference_torchscript': benchmark_failures,
-        'test_inference_with_configs': benchmark_failures,
-        'test_save_csv_files': benchmark_failures,
-        'test_trace_memory': benchmark_failures,
-        'test_train_encoder_decoder_with_configs': benchmark_failures,
-        'test_train_no_configs': benchmark_failures,
-        'test_train_no_configs_fp16': benchmark_failures,
-        'test_train_with_configs': benchmark_failures,
-    },
     'tests.generation.test_logits_process.LogitsProcessorTest': {
         'test_watermarking_processor': { 'cuda': 'passed', },
     },
@@ -66,6 +54,21 @@
         'test_prepare_inputs_for_generation_decoder_llm': { 'cuda': 'failed' },
         'test_stop_sequence_stopping_criteria': { 'cuda': 'failed' },
     },
+    'tiests.models.blip.test_modeling_blip.BlipTextImageModelTest': {
+        'test_cpu_offload': test_cpu_offload_failures,
+        'test_disk_offload_bin': test_cpu_offload_failures,
+        'test_disk_offload_safetensors': test_cpu_offload_failures,
+    },
+    'tests.models.blip.test_modeling_blip.BlipVQAModelTest': {
+        'test_cpu_offload': test_cpu_offload_failures,
+        'test_disk_offload_bin': test_cpu_offload_failures,
+        'test_disk_offload_safetensors': test_cpu_offload_failures,
+    },
+    'tests.models.dab_detr.test_modeling_dab_detr.DabDetrModelTest': {
+        'test_cpu_offload': test_cpu_offload_failures,
+        'test_disk_offload_bin': test_cpu_offload_failures,
+        'test_disk_offload_safetensors': test_cpu_offload_failures,
+    },
     'tests.models.detr.test_image_processing_detr.DetrImageProcessingTest': {
         'test_fast_is_faster_than_slow': { 'flaky': True },
     },
@@ -86,14 +89,43 @@
     'tests.models.mamba.test_modeling_mamba.MambaIntegrationTests': {
         'test_simple_generate_1_cpu': { 'cuda': 'passed' },
     },
+    # introduced by v4.49.0:
+    # https://github.com/huggingface/transformers/commit/be2ac0916a7902e1683d708805270142257a254a
+    'tests.models.paligemma.test_modeling_paligemma.PaliGemmaForConditionalGenerationModelTest': {
+        'test_generate_compilation_all_outputs': { 'cuda': 'failed' },
+    },
+    # introduced by v4.49.0:
+    # https://github.com/huggingface/transformers/commit/be2ac0916a7902e1683d708805270142257a254a
+    'tests.models.paligemma2.test_modeling_paligemma2.PaliGemma2ForConditionalGenerationModelTest': {
+        'test_generate_compilation_all_outputs': { 'cuda': 'failed' },
+    },
     'tests.models.pix2struct.test_modeling_pix2struct.Pix2StructModelTest': {
         'test_new_cache_format_0': { 'cuda': 'passed' },
         'test_new_cache_format_1': { 'cuda': 'passed' },
         'test_new_cache_format_2': { 'cuda': 'passed' },
     },
+    'tests.models.qwen2_5_vl.test_processor_qwen2_5_vl.Qwen2_5_VLProcessorTest': {
+        'test_chat_template_video_custom_sampling': { 'cuda': 'failed' },
+        'test_chat_template_video_special_processing': { 'cuda': 'failed' },
+    },
+    'tests.models.qwen2_vl.test_processor_qwen2_vl.Qwen2VLProcessorTest': {
+        'test_chat_template_video_custom_sampling': { 'cuda': 'failed' },
+        'test_chat_template_video_special_processing': { 'cuda': 'failed' },
+    },
+    # different failure signature than described in 'test_cpu_offload_failures'
+    'tests.models.roberta.test_modeling_roberta.RobertaModelTest': {
+        'test_cpu_offload': { 'cuda': 'failed' },
+        'test_disk_offload_bin': { 'cuda': 'failed' },
+        'test_disk_offload_safetensors': { 'cuda': 'failed' },
+    },
     'tests.models.speecht5.test_modeling_speecht5.SpeechT5ForTextToSpeechIntegrationTests': {
         'test_batch_generation': { 'cuda': 'passed' },
     },
+    'tests.models.vilt.test_modeling_vilt.ViltModelTest': {
+        'test_cpu_offload': test_cpu_offload_failures,
+        'test_disk_offload_bin': test_cpu_offload_failures,
+        'test_disk_offload_safetensors': test_cpu_offload_failures,
+    },
     'tests.pipelines.test_pipelines_automatic_speech_recognition.AutomaticSpeechRecognitionPipelineTests': {
         'test_small_model_pt_seq2seq': { 'cuda': "failed" },
     },

diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
@@ -41,7 +41,7 @@ on:
       transformers:
         required: false
         type: string
-        default: 'v4.48.3'
+        default: 'v4.49.0'
         description: Transformers version
 
 permissions: read-all
@@ -56,7 +56,7 @@ jobs:
       DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
       python: ${{ inputs.python != '' && inputs.python || '3.10' }}
       pytorch: ${{ inputs.pytorch != '' && inputs.pytorch || 'nightly' }}
-      transformers: ${{ inputs.transformers != '' && inputs.transformers || 'v4.48.3' }}
+      transformers: ${{ inputs.transformers != '' && inputs.transformers || 'v4.49.0' }}
       PYTORCH_DEBUG_XPU_FALLBACK: '1'
       TRANSFORMERS_TEST_DEVICE_SPEC: 'spec.py'
     steps:
@@ -157,13 +157,6 @@ jobs:
           source activate $CONDA_ENV_NAME
           cd transformers
           python3 -m pytest -rsf --make-reports=$TEST_CASE --junit-xml=reports/$TEST_CASE.xml tests/*.py || true
-      - name: Run tests/benchmark
-        env:
-          TEST_CASE: 'tests_benchmark'
-        run: |
-          source activate $CONDA_ENV_NAME
-          cd transformers
-          python3 -m pytest -rsf --make-reports=$TEST_CASE --junit-xml=reports/$TEST_CASE.xml tests/benchmark || true
       - name: Run tests/generation
         env:
           TEST_CASE: 'tests_generation'