Merge branch main into colin/par-eval-expr

.github/workflows/python-package.yml

@@ -695,7 +695,7 @@ jobs:
 
   rust-tests-platform:
     runs-on: ${{ matrix.os }}-latest
-    timeout-minutes: 30
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:

.github/workflows/python-publish.yml

@@ -15,7 +15,6 @@ on:
     - v*
   workflow_dispatch:
 env:
-  PACKAGE_NAME: getdaft
   PYTHON_VERSION: 3.11
   DAFT_ANALYTICS_ENABLED: '0'
   UV_SYSTEM_PYTHON: 1
@@ -24,19 +23,28 @@ env:
   IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
   RUST_DAFT_PKG_BUILD_TYPE: ${{ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0'))) && 'release' || 'nightly' }}
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   build-and-test:
 
-    name: platform wheels for ${{ matrix.os }}-${{ matrix.compile_arch }}
+    name: platform wheels for ${{ matrix.os }}-${{ matrix.compile_arch }}-lts=${{ matrix.lts }}
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu, macos, windows]
         compile_arch: [x86_64, aarch64]
+        lts: [0, 1] # LongTerm Support CPUs
+
         exclude:
         - os: windows
           compile_arch: aarch64
+        - lts: 1
+          compile_arch: aarch64
+
     steps:
     - uses: actions/checkout@v4
       with:
@@ -47,16 +55,29 @@ jobs:
         python-version: ${{ env.PYTHON_VERSION }}
         architecture: x64
     - run: pip install uv
-    - run: uv pip install twine toml
+    - run: uv pip install twine toml yq
     - run: python tools/patch_package_version.py
+    - name: Patch name to daft-lts if LTS
+      if: ${{ matrix.lts }}
+      run: tomlq -i -t ".project.name = \"daft-lts\"" pyproject.toml
+
+    - name: Configure RUSTFLAGS for x86
+      if: ${{ (matrix.compile_arch == 'x86_64') }}
+      run: |
+        if [[ ${{ matrix.lts }} ]]; then
+          echo "RUSTFLAGS=-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b" >> $GITHUB_ENV && \
+          echo "CFLAGS=-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16" >> $GITHUB_ENV
+        else
+          echo "RUSTFLAGS=-C target-feature=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq,+movbe -Z tune-cpu=skylake" >> $GITHUB_ENV && \
+          echo "CFLAGS=-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mpclmul -mmovbe -mtune=skylake" >> $GITHUB_ENV
+        fi
+
     - name: Build wheels - Mac and Windows x86
       if: ${{ ((matrix.os == 'macos') || (matrix.os == 'windows')) && (matrix.compile_arch == 'x86_64')  }}
       uses: messense/maturin-action@v1
       with:
         target: x86_64
         args: --profile release-lto --out dist
-      env:
-        RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2
     - name: Build wheels - Linux x86
       if: ${{ (matrix.os == 'ubuntu') && (matrix.compile_arch == 'x86_64') }}
       uses: messense/maturin-action@v1
@@ -66,8 +87,7 @@ jobs:
         # only produce sdist for linux x86 to avoid multiple copies
         args: --profile release-lto --out dist --sdist
         before-script-linux: yum -y install perl-IPC-Cmd
-      env:
-        RUSTFLAGS: -C target-feature=+fxsr,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma
+
     - name: Build wheels - Linux aarch64
       if: ${{ (matrix.os == 'ubuntu') && (matrix.compile_arch == 'aarch64') }}
       uses: messense/maturin-action@v1
@@ -88,28 +108,21 @@ jobs:
         args: --profile release-lto --out dist
       env:
         RUSTFLAGS: -Ctarget-cpu=apple-m1
+        CFLAGS: -mtune=apple-m1
 
     - name: Install and test built wheel - Linux and Mac x86_64
       if: ${{ (matrix.os == 'ubuntu') && (matrix.compile_arch == 'x86_64')  }}
       run: |
-        uv pip install -r requirements-dev.txt dist/${{ env.PACKAGE_NAME }}-*x86_64*.whl --force-reinstall
+        uv pip install -r requirements-dev.txt dist/*-*x86_64*.whl --force-reinstall
         rm -rf daft
         pytest -v
       env:
-        DAFT_RUNNER: py
-    # Disable until we figure out why are we getting FileNotFoundError: [WinError 3] Failed to open local file
-    # - name: Install and test built wheel - Windows x86_64
-    #   if: ${{ (matrix.os == 'windows') && (matrix.compile_arch == 'x86_64')  }}
-    #   run: |
-    #     $FILES = Get-ChildItem -Path .\dist\${{ env.PACKAGE_NAME }}-*-win_amd64.whl -Force -Recurse
-    #     pip install -r requirements-dev.txt $FILES[0].FullName --force-reinstall
-    #     rd -r daft
-    #     pytest -v
+        DAFT_RUNNER: native
 
     - name: Upload wheels
       uses: actions/upload-artifact@v4
       with:
-        name: wheels-${{ matrix.os }}-${{ matrix.compile_arch }}
+        name: wheels-${{ matrix.os }}-${{ matrix.compile_arch }}-lts-${{ matrix.lts }}
         path: dist
 
     - name: Send Slack notification on failure
@@ -132,10 +145,13 @@ jobs:
         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
-
   publish:
     name: Publish wheels to PYPI and Anaconda
     runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
     needs:
     - build-and-test
     steps:
@@ -144,19 +160,17 @@ jobs:
         python-version: ${{ env.PYTHON_VERSION }}
         architecture: x64
     - run: pip install -U twine
-    - uses: actions/checkout@v4
     - uses: actions/download-artifact@v4
       with:
         pattern: wheels-*
         merge-multiple: true
         path: dist
     - run: ls -R ./dist
-    - name: Publish bdist package to PYPI
+    - name: Publish package distributions to PyPI
       if: ${{ success() && (env.IS_PUSH == 'true') }}
-      run: python -m twine upload --skip-existing --disable-progress-bar ./dist/*
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        skip-existing: true
 
     - uses: conda-incubator/setup-miniconda@v3
       with: