python.yaml

name: Python CI

on:
  push:
    branches:
      - 'master'
      - 'release/*'
    tags:
      - 'v*'
  pull_request:

# Cancel any previous runs with the same key.
concurrency:
  # Example key: Repository Structure CI-refs/pull/1381/merge
  #   or Repository Structure CI-<owner>-<sha> for pushes
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || format('{0}-{1}', github.repository_owner, github.sha) }}
  cancel-in-progress: true

jobs:
  do_python:
    if: |
      github.event_name != 'pull_request' ||
      (
        !contains(github.event.pull_request.body, '[no Python]')
      )
    runs-on: ubuntu-latest
    steps:
      - name: Info
        run: echo "will build"

  do_python_package:
    needs:
      - do_python
    if: |
      github.event_name != 'pull_request' ||
      (
        !contains(github.event.pull_request.body, '[no package]') &&
        !contains(github.event.pull_request.body, '[no Python package]')
      )
    runs-on: ubuntu-latest
    steps:
      - name: Info
        run: echo "will build"

  do_python_test:
    needs:
      - do_python
    # Don't run this in tag CI runs. It's useless since it doesn't produce meaningful artifacts.
    if: |
      github.event_name != 'pull_request' && !startsWith(github.ref, 'refs/tags/') ||
      (
        !contains(github.event.pull_request.body, '[no test]') &&
        !contains(github.event.pull_request.body, '[no Python test]')
      )
    runs-on: ubuntu-latest
    steps:
      - name: Info
        run: echo "will build"

  build_and_test_conda_package:
    needs: do_python_package
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        os:
        - ubuntu-20.04
    steps:
    - name: Checkout repo
      uses: actions/checkout@v2
      with:
        submodules: 'recursive'
        # We need history so that we have tags to provide version information.
        fetch-depth: 0
        # 0 fetches all history. See https://github.com/actions/checkout#checkout-v2

    ###
    # Caching between builds
    ###
    - name: Cache miniconda
      id: cache-miniconda
      uses: actions/cache@v2
      with:
        key: ${{matrix.os}}-miniconda-${{hashFiles('.github/workflows/download_miniconda.sh')}}
        path: ~/.cache/miniconda
    - name: Cache ccache objects
      uses: actions/cache@v2
      with:
        key: ${{matrix.os}}-ccache-arrow-6-v1-${{github.sha}}
        restore-keys: |
          ${{matrix.os}}-ccache-arrow-6-v1-
        path: ~/.ccache

    ###
    # Install platform-level dependencies (OS-specific)
    ###
    - name: Install dependencies
      if: startsWith(matrix.os, 'ubuntu-')
      run: |
        sudo apt install -yq ccache curl libz-dev bzip2 make libnuma1

    - name: Setup ccache
      run: |
        .github/workflows/configure_ccache.sh
        ccache --max-files 350 --max-size=1.5G # Roughly two builds worth of objects, and limited in size as a safety net.
        ccache --zero-stats

    ###
    # Install Conda environment
    ###
    - name: Download miniconda
      run: .github/workflows/download_miniconda.sh ${{matrix.os}}
      if: steps.cache-miniconda.outputs.cache-hit != 'true'
    - name: Create directories
      run: mkdir -p $HOME/build
    - name: Install Conda dependencies
      run: |
        . .github/workflows/activate_miniconda.sh
        mamba install -y -q -c conda-forge conda-build boa conda-verify pygithub packaging

    ###
    # Conda build process
    ###
    - name: Build conda Packages
      run: |
        . .github/workflows/activate_miniconda.sh
        export CPU_COUNT=1
        scripts/build_conda.py --output-folder $HOME/build/

    - name: Build conda Packages for Katana Metagraph Plugin
      run: |
        . .github/workflows/activate_miniconda.sh
        export KATANA_VERSION=$(scripts/version show --pep440)
        export KATANA_TEST_DATASETS=$(realpath external/test-datasets)
        cd metagraph-plugin/conda_recipe
        conda mambabuild . -c conda-forge -c metagraph -c file://$HOME/build/ --output-folder $HOME/build/

    - name: Clean ccache
      run: |
        ccache --show-stats
        ccache --cleanup

    # Upload path is relative to work directory and "~" and "${{env.HOME}}" do
    # not seem to be expanded, so hard code HOME for now.
    - name: Upload built packages
      uses: actions/upload-artifact@v1
      if: startsWith(matrix.os, 'ubuntu-')
      with:
        name: conda-pkgs-${{matrix.os}}
        path: /home/runner/build

  test_conda_package_install:
    needs: build_and_test_conda_package
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        image:
          - ubuntu:18.04
          - ubuntu:latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Download conda packages
        uses: actions/download-artifact@v2
        with:
          name: conda-pkgs-ubuntu-20.04
          path: pkgs
      - name: Install in docker container
        run: python3 test/install_conda_packages_test.py pkgs ${{matrix.image}}

  upload_conda_packages:
    if: |
      (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/v')
        || startsWith(github.ref, 'refs/tags/v'))
      && github.event_name == 'push' && github.repository == 'KatanaGraph/katana'
    environment: AnacondaUpload
    needs: test_conda_package_install
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'

      ###
      # Caching between builds
      ###
      - name: Cache miniconda
        id: cache-miniconda
        uses: actions/cache@v2
        with:
          key: ubuntu-20.04-miniconda-${{hashFiles('.github/workflows/download_miniconda.sh')}}
          path: ~/.cache/miniconda

      ###
      # Install platform-level dependencies (OS-specific)
      ###
      - name: Install dependencies
        run: |
          sudo apt install -yq curl

      ###
      # Install Conda environment
      ###
      - name: Download miniconda
        run: .github/workflows/download_miniconda.sh ${{matrix.os}}
        if: steps.cache-miniconda.outputs.cache-hit != 'true'
      - name: Install dependencies
        run: |
          . .github/workflows/activate_miniconda.sh
          mamba install -y -q -c conda-forge anaconda-client

      - name: Download Katana conda packages
        uses: actions/download-artifact@v2
        with:
          name: conda-pkgs-ubuntu-20.04
          path: pkgs

      - name: Upload Katana conda packages to anaconda with label dev
        env:
          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
        run: |
          . .github/workflows/activate_miniconda.sh
          echo "Anaconda token has ${#ANACONDA_TOKEN} characters."
          echo "Uploading:"
          ls pkgs/*/*.tar.bz2
          anaconda --token "$ANACONDA_TOKEN" upload --no-progress --user KatanaGraph --package-type conda --label dev pkgs/*/*.tar.bz2

  test_build_in_container_conda:
    needs: do_python_test
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Run build_in_container
        run: scripts/build_in_container.py --build $HOME/build --type conda

  test_build_in_container_conda_ninja:
    needs: do_python_test
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Run build_in_container (with Ninja)
        run: scripts/build_in_container.py --build $HOME/build --type conda --cmake-generator Ninja

  lint:
    needs: do_python
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        os:
        - ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: 'recursive'
        # We need history so that we have tags to provide version information.
        fetch-depth: 0
        # 0 fetches all history. See https://github.com/actions/checkout#checkout-v2

    ###
    # Caching between builds
    ###
    - name: Cache miniconda
      id: cache-miniconda
      uses: actions/cache@v2
      with:
        key: ${{matrix.os}}-miniconda-${{hashFiles('.github/workflows/download_miniconda.sh')}}
        path: ~/.cache/miniconda

    ###
    # Install platform-level dependencies (OS-specific)
    ###
    - name: Install dependencies
      run: |
        sudo apt install -yq curl

    ###
    # Install Conda environment
    ###
    - name: Download miniconda
      run: .github/workflows/download_miniconda.sh ${{matrix.os}}
      if: steps.cache-miniconda.outputs.cache-hit != 'true'

    - name: Install dependencies
      run: |
        . .github/workflows/activate_miniconda.sh
        mamba install -y -q -c conda-forge black=19.10b0 pylint=2.8.3 pydocstyle=6.1.1

    - name: Check formatting
      run: |
        . .github/workflows/activate_miniconda.sh
        ./scripts/check_python_format.sh python test scripts

    - name: Check for lint
      run: |
        . .github/workflows/activate_miniconda.sh
        ./scripts/check_python_lint.sh python test scripts

  build_and_test:
    needs: do_python_test
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        os:
          - ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          submodules: 'recursive'
          # We need history so that we have tags to provide version information.
          fetch-depth: 0
          # 0 fetches all history. See https://github.com/actions/checkout#checkout-v2

      ###
      # Caching between builds
      ###
      - name: Cache miniconda
        id: cache-miniconda
        uses: actions/cache@v2
        with:
          key: ${{matrix.os}}-miniconda-${{hashFiles('.github/workflows/download_miniconda.sh')}}
          path: ~/.cache/miniconda
      - name: Cache ccache objects
        uses: actions/cache@v2
        with:
          key: ${{matrix.os}}-ccache-dev-arrow-6-v1-${{github.sha}}
          restore-keys: |
            ${{matrix.os}}-ccache-dev-arrow-6-v1-
          path: ~/.ccache

      ###
      # Install platform-level dependencies (OS-specific)
      ###
      - name: Install dependencies
        if: startsWith(matrix.os, 'ubuntu-')
        run: |
          sudo apt install -yq ccache curl libz-dev bzip2 make libnuma1

      - name: Setup ccache
        run: |
          .github/workflows/configure_ccache.sh
          ccache --max-files 350 --max-size=1.5G # Roughly two builds worth of objects, and limited in size as a safety net.

      ###
      # Install Conda environment
      ###
      - name: Download miniconda
        run: .github/workflows/download_miniconda.sh ${{matrix.os}}
        if: steps.cache-miniconda.outputs.cache-hit != 'true'
      - name: Create directories
        run: mkdir -p $HOME/build
      - name: Install Conda dependencies
        run: |
          . .github/workflows/activate_miniconda.sh
          mamba env update --quiet --file=conda_recipe/environment.yml
          mamba list

      ###
      # Standard CMake build process
      ###
      - name: Configure
        run: |
          . .github/workflows/activate_miniconda.sh
          export CMAKE_BUILD_PARALLEL_LEVEL=2
          .github/workflows/configure_cpp.sh \
            ${{matrix.os}} "$CXX" Release "$HOME/build" \
            -DKATANA_LANG_BINDINGS=python -DBUILD_DOCS=internal


      - name: Build
        run: |
          . .github/workflows/activate_miniconda.sh
          export CMAKE_BUILD_PARALLEL_LEVEL=2
          cmake --build $HOME/build --parallel 2

      - name: Build documentation
        run: |
          . .github/workflows/activate_miniconda.sh
          cmake --build $HOME/build --target doc

      - name: Clean ccache
        run: |
          ccache --show-stats
          ccache --cleanup

      # Upload path is relative to work directory and "~" and "${{env.HOME}}" do
      # not seem to be expanded, so hard code HOME for now.
      - name: Upload documentation artifact (C++ and Python)
        uses: actions/upload-artifact@v1
        if: startsWith(matrix.os, 'ubuntu-')
        with:
          name: katana-docs-${{matrix.os}}
          path: /home/runner/build/docs
      
      - name: Publish documentation
        uses: peaceiris/actions-gh-pages@v3
        if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'push' && github.repository == 'KatanaGraph/katana' }}
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: documentation
          publish_dir: /home/runner/build/docs/katana_python

      - name: Test python_env.sh
        if: startsWith(matrix.os, 'ubuntu-')
        run: |
          . .github/workflows/activate_miniconda.sh
          $HOME/build/python_env.sh python -c "import katana; katana.for_each;"
          $HOME/build/python_env.sh python -c "import katana.local.analytics; katana.local.analytics.sssp;"
          . $HOME/build/python_env.sh
          python -c "import katana; katana.for_each;"
          python -c "import katana.local.analytics; katana.local.analytics.sssp;"

      - name: Install
        run: |
          . .github/workflows/activate_miniconda.sh
          DESTDIR=/tmp cmake --install $HOME/build

      - name: Test
        if: startsWith(matrix.os, 'ubuntu-')
        run: |
          . .github/workflows/activate_miniconda.sh
          export AWS_EC2_METADATA_DISABLED=true
          # Default timeout is 1500 seconds. Quick tests should not take more
          # than a few seconds typically.
          cd $HOME/build && ctest --output-on-failure --label-regex quick --parallel 2 --timeout 350