test-language-lang_id-kfp.yml

#
# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
#
name: Test KFP - transforms/language/lang_id

on:
    workflow_dispatch:
    push:
        branches:
            - "dev"
            - "releases/**"
        tags:
            - "*"
        paths:
            - ".make.*"
            - "scripts/k8s-setup/requirements.env"
            - "transforms/.make.workflows"
            - "transforms/language/lang_id/**"
            - "!kfp/**" # This is tested in separate workflow
            - "!data-processing-lib/**" # This is tested in separate workflow
            - "!**.md"
            - "!**/doc/**"
            - "!**/images/**"
            - "!**.gitignore"
    pull_request:
        branches:
            - "dev"
            - "releases/**"
        paths:
            - ".github/workflows/test-language-lang_id-kfp.yml"
            - ".make.*"
            - "scripts/k8s-setup/requirements.env"
            - "transforms/.make.workflows"
            - "transforms/language/lang_id/**"
            - "!data-processing-lib/**" # This is tested in separate workflow
            - "!kfp/**" # This is tested in separate workflow
            - "!**.md"
            - "!**/doc/**"
            - "!**/images/**"
            - "!**.gitignore"

# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
concurrency:
    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
    cancel-in-progress: true

jobs:
    test-kfp-v1:
        runs-on: ubuntu-22.04
        env:
            HF_READ_ACCESS_TOKEN: ${{ secrets.HF_READ_ACCESS_TOKEN }}
        steps:
            - name: Expose secret to trusted users working in a fork
              run: |
                echo "Checking access- PR head:  ${{ github.event.pull_request.head.sha }}"
                echo "Checking access- github ref:  ${{ github.ref }}"
                ./scripts/check-restricted_access.sh ${{ github.triggering_actor }} ${{ github.event.pull_request.head.repo.fork }}

            - name: Checkout
              uses: actions/checkout@v4
              with:
                # Requires careful code review prior to triggering workflow
                ref: ${{ github.event.pull_request.head.sha }}

            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  echo "pr head:  ${{ github.event.pull_request.head.sha }}"
                  echo "github ref:  ${{ github.ref }}"
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Import environment variables
              run: |
                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
            - name: Test V1 KFP workflow for transforms/language/lang_id
              timeout-minutes: 120
              run: |
                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                  if [ -e "transforms/language/lang_id/Makefile" -a -e "transforms/language/lang_id/kfp_ray/Makefile" ]; then
                     transform=$(basename "transforms/language/lang_id")
                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
                        $PWD/scripts/workflow_helper.sh install-tools
                        $PWD/scripts/workflow_helper.sh test-workflow transforms/language/lang_id   
                     else
                        $PWD/scripts/workflow_helper.sh build-workflow transforms/language/lang_id
                     fi              
                  else
                     echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi

    test-kfp-v2:
        runs-on: ubuntu-22.04
        env:
            HF_READ_ACCESS_TOKEN: ${{ secrets.HF_READ_ACCESS_TOKEN }}
        steps:
            - name: Expose secret to trusted users working in a fork
              run: |
                allowed=false
                allowed_list=("touma-I" "revit13" "roytman")
                for user in "${allowed_list[@]}"; do
                    if [ "$user" ==  "${{ github.triggering_actor }}" ]; then
                       allowed=true
                       break
                    fi
                done
                if $allowed; then
                   echo "Checking ${{ github.triggering_actor }} permissions."
                   echo "Only select users will be able to trigger this workflow from a fork"
                   echo "This prevents the secret from being exposed to all public users"
                   echo "pr head:  ${{ github.event.pull_request.head.sha }}"
                   echo "github ref:  ${{ github.ref }}"
                else
                   echo "User ${{ github.triggering_actor }} is not allowed to trigger this workflow."
                   echo "${{ github.triggering_actor }} is not in list: $allowed_list "
                   echo "pr head:  ${{ github.event.pull_request.head.sha }}"
                   echo "github ref:  ${{ github.ref }}"
                   exit 1
                fi

            - name: Checkout
              uses: actions/checkout@v4
              with:
                # Requires careful code review prior to triggering workflow from a fork
                ref: ${{  github.event.pull_request.head.sha }}
            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Import environment variables
              run: |
                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
                  echo "KFPv2=1" >> $GITHUB_ENV
            - name: Test V2 KFP workflow for transforms/language/lang_id
              timeout-minutes: 120
              run: |
                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                  if [ -e "transforms/language/lang_id/Makefile" -a -e "transforms/language/lang_id/kfp_ray/Makefile" ]; then
                     transform=$(basename "transforms/language/lang_id")
                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
                        $PWD/scripts/workflow_helper.sh install-tools
                        $PWD/scripts/workflow_helper.sh test-workflow transforms/language/lang_id   
                     else
                        $PWD/scripts/workflow_helper.sh build-workflow transforms/language/lang_id
                     fi              
                  else
                     echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray/Makefile"
                   fi