clean and clear CI

.github/workflows/build_pr_documentation.yml

@@ -2,7 +2,8 @@ name: Build PR documentation
 
 on:
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -18,20 +19,18 @@ jobs:
       PR_CLONE_URL: ${{ github.event.pull_request.head.repo.clone_url }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
-          repository: 'huggingface/doc-builder'
+          repository: "huggingface/doc-builder"
           path: doc-builder
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
-          repository: 'huggingface/optimum-intel'
+          repository: "huggingface/optimum-intel"
           path: optimum-intel
 
       - name: Setup environment
         run: |
-          python -m venv venv-doc
-          source venv-doc/bin/activate
           pip uninstall -y doc-builder
           cd doc-builder
           git pull origin main

.github/workflows/quality.yml

@@ -0,0 +1,45 @@
+name: Code Quality Checks
+on:
+  push:
+    branches:
+      - main
+
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9"]
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          # TODO: we should remove the the version pinning at some point
+          pip install "black~=23.1" "ruff==0.4.4"
+
+      - name: Check style with black
+        run: |
+          black --check .
+
+      - name: Check style with ruff
+        run: |
+          ruff check .

.github/workflows/security.yml

@@ -1,4 +1,4 @@
-name: Security Checks
+name: Security Check
 
 on:
   push:

.github/workflows/test_generation.yml

@@ -1,12 +1,13 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-name: Intel Generation Utils - Test
+name: Generation Utils - Test
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - main
+      - v*-release
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -17,22 +18,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.9]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
+
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install transformers==4.43.*
-        pip install optimum[exporters]
-        pip install .[tests]
-    - name: Test with Pytest
-      run: |
-        pytest tests/generation/
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install transformers==4.43.*
+          pip install .[tests]
+
+      - name: Test with Pytest
+        run: |
+          pytest tests/generation/

.github/workflows/test_inc.yml

@@ -1,6 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-name: Intel Neural Compressor - Test
+name: INC - Test
 
 on:
   push:
@@ -20,31 +18,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        torch-version: ["2.3.*", "2.4.*"]
+        python-version: [3.9]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
+
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install cmake
-        pip install py-cpuinfo
-        pip install torch==2.3.0 torchaudio==2.3.0 torchvision==0.18 --index-url https://download.pytorch.org/whl/cpu
-        pip install intel-extension-for-pytorch==2.3.0
-        pip install datasets==2.19.0
-        pip install .[neural-compressor,diffusers,tests]
-        pip install peft
-
-    - name: Test with Pytest
-      run: |
-        pytest tests/neural_compressor/ --ignore tests/neural_compressor/test_ipex.py --durations=0
-    - name: Test IPEX
-      run: |
-        pytest tests/neural_compressor/test_ipex.py
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install torch==${{ matrix.torch-version }} torchaudio torchvision --index-url https://download.pytorch.org/whl/cpu
+          pip install intel-extension-for-pytorch==${{ matrix.torch-version }}
+          pip install .[neural-compressor,ipex,diffusers,peft,tests]
 
+      - name: Test with Pytest
+        run: |
+          pytest tests/neural_compressor/ -s -vvvv --durations=0

.github/workflows/test_ipex.yml

@@ -1,6 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-name: Intel IPEX - Test
+name: IPEX - Test
 
 on:
   push:
@@ -17,36 +15,43 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        torch-version: ["2.2.*", "2.3.*", "2.4.*"]
         transformers-version: ["4.39.0", "4.44.*"]
-        ipex-version: ["2.2.0", "2.3.*"]
-        include:
-          - python-version: 3.8
-            transformers-version: 4.39.0
-            ipex-version: 2.2.0
+        python-version: [3.9]
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install torch==${{ matrix.ipex-version }} --extra-index-url https://download.pytorch.org/whl/cpu
-          pip install intel_extension_for_pytorch==${{ matrix.ipex-version }}
-          pip install Pillow parameterized
-          pip install transformers[testing]==${{ matrix.transformers-version }}
+          pip install --upgrade pip
+          pip install torch==${{ matrix.torch-version }} torchaudio torchvision --extra-index-url https://download.pytorch.org/whl/cpu
+          pip install intel_extension_for_pytorch==${{ matrix.torch-version }}
+          pip install transformers==${{ matrix.transformers-version }}
           pip install .[ipex]
 
-      - if: ${{ matrix.ipex-version == '2.2.0' }}
+      - name: Assert versions
+        run: |
+          python -c "import torch; assert torch.__version__.startswith('${{ matrix.torch-version }}'.replace('.*', ''))"
+          python -c "import transformers; assert transformers.__version__.startswith('${{ matrix.transformers-version }}'.replace('.*', ''))"
+          python -c "import intel_extension_for_pytorch; assert intel_extension_for_pytorch.__version__.startswith('${{ matrix.torch-version }}'.replace('.*', ''))"
+
+      - if: ${{ matrix.torch-version == '2.2.*' }}
+        name: Downgrade numpy for torch 2.2.*
         run: pip install numpy==1.*
 
       - name: Test with Pytest
         run: |
-          pytest tests/ipex/
+          pytest tests/ipex/ -s -vvvv --durations=0

.github/workflows/test_offline.yaml

@@ -1,10 +1,13 @@
-name: Offline usage / Python - Test
+name: Offline - Test
 
 on:
   push:
-    branches: [main]
+    branches:
+      - main
+      - v*-release
   pull_request:
-    branches: [main]
+    branches:
+      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -15,19 +18,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: ["3.9"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           pip install .[tests,openvino]
+
       - name: Test
         run: |
           HF_HOME=/tmp/ huggingface-cli download hf-internal-testing/tiny-random-gpt2