pm-gh-actions.yml

############################################################################################
# Copyright (C) 2021 Prediction Machine Advisers, LLC
# This file is available under MIT license
# based on https://github.com/predictionmachine/pm-github-actions/blob/main/.github/workflows/pm-gh-actions.yml
# pm-version 0.4.0
############################################################################################

name: PM CI workflow
on:
  workflow_call:  # enables this workflow to be reusable for other repo. See: https://docs.github.com/en/actions/learn-github-actions/reusing-workflows
    secrets:
      CC_TEST_REPORTER_ID:
        description: 'a repo-specific test reporter id provided by Codeclimate for submitting coverage report'
        required: true
      CODECOV_TOKEN:
        description: 'a repo-specific token for uploading coverage reports to Codcov'
        required: true
  push:
    branches:
      - dev
      - main
  pull_request:
    types:
      - opened
      - edited
      - reopened
    branches:
      - main
      - dev
env:
  GUIDELINE_REPO: 'https://github.com/predictionmachine/pm-coding-template'

jobs:
  # Check branch name standard as per `pm-coding-template`
  # Reference: https://github.com/predictionmachine/pm-coding-template/blob/main/docs/coding-standard.md#github-branches-pull-requests
  check-branch-naming-convention:
    name: Check branch naming convention as per pm-coding-template
    runs-on: ubuntu-latest
    timeout-minutes: 10
    if: ${{ github.event_name == 'pull_request' }}
    env:
      BRANCH_NAME_RULE: 'https://github.com/predictionmachine/pm-coding-template/blob/main/docs/coding-standard.md#github-branches-pull-requests'
    steps:
      - name: Check branch name
        id: branchCheck
        run: |
          echo "branch name: $GITHUB_HEAD_REF"
          if [[ "$GITHUB_HEAD_REF" =~ "/" ]];then
            echo "::set-output name=is_valid_name::true"
          else
            echo "::set-output name=is_valid_name::false"
          fi
      - uses: marocchino/sticky-pull-request-comment@v2
        if: ${{ steps.branchCheck.outputs.is_valid_name == 'false' }}
        with:
          header: invalid-branch-name-comments
          message: |
            👋   @${{ github.event.pull_request.user.login }}
            Please use a valid [branch name](${{ env.BRANCH_NAME_RULE }}).
            Make sure you have followed our [contribution guidelines](${{ env.GUIDELINE_REPO }}).
      - name: Exit job for invalid branch name
        if: ${{ steps.branchCheck.outputs.is_valid_name == 'false' }}
        run: |
          echo "Invalid branch name: $GITHUB_HEAD_REF"
          echo "Please use the standard mentioned here: $BRANCH_NAME_RULE"
          exit 1

  enforce-pr-description:
    name: 'Check PR description, fail if empty'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      TEMPLATE_URL: 'https://github.com/predictionmachine/.github/blob/main/.github/pull_request_template.md'
    steps:
      - name: Check PR description
        if: ${{ github.event_name == 'pull_request' }}
        id: getPrBody
        env:
          PR_BODY: ${{ github.event.pull_request.body }}
        run: |
          echo ${PR_BODY:0:5}
          if [ "${PR_BODY:0:5}" == "" ];then
            echo "::set-output name=is_empty_body::true"
          fi
      - uses: marocchino/sticky-pull-request-comment@v2
        if: ${{ steps.getPrBody.outputs.is_empty_body == 'true' }}
        with:
          header: pr-empty-comments
          message: |
            👋   @${{ github.event.pull_request.user.login }}
            Please use the PR template mentioned [here](${{ env.TEMPLATE_URL }})
            Please make sure you have followed our [contributing guidelines](${{ env.GUIDELINE_REPO }}).
      - name: Exit the job for empty PR description
        if: ${{ steps.getPrBody.outputs.is_empty_body == 'true' }}
        run: |
          echo "Failed due to empty PR description"
          exit 1

  check-unwanted-files:
    name: Check for unwanted files - .zip etc.
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: 'Check if file exists, and set flag'
        id: fileCheck
        run: |
          file_count=$(find ./ -type f \( -iname \*.zip -o -iname \*.parque \) | wc -l ) # get file count of the unwanted files
          if [[ "$file_count==0" ]];then
            echo "No unwanted files found"
            exit 0
          else
            echo "::set-output name=is_unwanted_file::true"
            echo "::set-output name=file_paths::$(find $PWD  -type f \( -iname \*.zip -o -iname \*.parque \))"
            echo "Please remove unwanted files"
            exit 0
          fi
      - uses: marocchino/sticky-pull-request-comment@v2
        if: ${{ steps.fileCheck.outputs.is_unwanted_file == 'true' }}
        with:
          header: file-check
          message: |
            👋   @${{ github.event.pull_request.user.login }}
            Please remove unwanted files from the repo.
            Here is the detected file path: ${{ steps.fileCheck.outputs.file_paths }}
            Please make sure you have followed our [contributing guidelines](${{ env.GUIDELINE_REPO }}).

  # see [reviewdog/action-detect-secrets](https://github.com/Yelp/detect-secrets) and
  # [detect-secrets](https://github.com/Yelp/detect-secrets)
  check-hardcoded-credentials:
    name: Check hardcoded credentials in files
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: detect-secrets
        uses: reviewdog/action-detect-secrets@master
        with:
          github_token: '${{ secrets.GITHUB_TOKEN }}'
          reporter: github-pr-review
          fail_on_error: false

  linting:
    name: 'Linting: black, flake8'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      DOCSTRING_REPORT: output/docstring_report.txt
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: black formatting check
        uses: reviewdog/action-black@v2
        with:
          github_token: '${{ secrets.GITHUB_TOKEN }}'
          black_args: '--config=pyproject.toml'
          reporter: github-pr-review
          fail_on_error: true
      - name: flake8 lint check
        uses: reviewdog/action-flake8@v3
        with:
          github_token: '${{ secrets.GITHUB_TOKEN }}'
          flake8_args: '--config=setup.cfg'
          reporter: github-pr-review
          fail_on_error: true

  interrogate:
    name: 'Docstrings coverage check (interrogate)'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    env:
      DOCSTRING_REPORT: output/docstring_report.txt
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Install dependencies & create output directory for reports
        run: |
          pip install --upgrade pip interrogate
          mkdir -p output # for docstring report
      - name: Generate docstring report
        # see also https://github.com/JackMcKew/python-interrogate-check#full-example
        # if interrogate failed, the cov will fail separately, later
        run: |
          interrogate -o "$DOCSTRING_REPORT" || exit 0
      - name: Analyze docstring report
        id: docstringCheck
        run: |
          total_cov=$(grep -o 'actual: [0-9.]*' $DOCSTRING_REPORT | sed 's/actual: //') # get actual docstring coverage from docstring report file
          echo "::set-output name=total_cov::$cov_val"
          passed_or_failed=$(grep -o 'RESULT: \w*' $DOCSTRING_REPORT | sed 's/RESULT: //')
          echo "::set-output name=passed_or_failed::$passed_or_failed"
      - name: Reformat docstring report
        run: |
          sed -i "s!^==*!## Docstring Coverage :memo:!" "$DOCSTRING_REPORT"
          sed -i "s!.*-- Summary --.*!\n<details><summary>docstring coverage details</summary>\n!" "$DOCSTRING_REPORT"
          sed -i "s!.*-- RESULT:!</details>\n\n!" "$DOCSTRING_REPORT"
          sed -i "s!) --.*!)!" "$DOCSTRING_REPORT"
          sed -i "s/TOTAL    /**TOTAL**/" "$DOCSTRING_REPORT"
          sed -i "s/ PASSED / **PASSED** /" "$DOCSTRING_REPORT"
          sed -i "s/ FAILED / **FAILED** /" "$DOCSTRING_REPORT"
          # drop second separator line; TODO: run with -vv and format nicely
          gawk -i inplace '!/^[|]-/ || !f++' "$DOCSTRING_REPORT"
          cat "$DOCSTRING_REPORT"
      - name: Post docstring report as PR comment
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: docstring-comment
          path: ${{ env.DOCSTRING_REPORT }}
      - name: Exit the job for docstring coverage failed
        if: ${{ steps.docstringCheck.outputs.passed_or_failed == 'FAILED' }}
        run: |
          echo "Docstring coverage (interrogate): failed because coverage is low, ${{ steps.docstringCheck.outputs.total_cov }}"
          exit 1

  mypy-typecheck:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    name: 'Mypy Type check'
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Use cache if possible
        # see [cache](https://github.com/actions/cache) and
        # [pip example](https://github.com/actions/cache/blob/master/examples.md#simple-example))
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements**.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ requirements.txt ]; then pip install -r requirements.txt; fi
          if [ requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
      - name: mypy type check - reviewdog
        uses: tsuyoshicho/action-mypy@v3
        with:
          github_token: '${{ secrets.GITHUB_TOKEN }}'
          reporter: github-pr-review
          level: warning  # to prevent failing without error
          # fail_on_error: true
      - name: mypy type check
        run: |
          python -m mypy

  test-and-coverage:
    name: 'Pytest and Coverage'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      COVERAGE_OUTPUT_PATH: output/coverage.xml
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Use cache if possible
        # see [cache](https://github.com/actions/cache) and
        # [pip example](https://github.com/actions/cache/blob/master/examples.md#simple-example))
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements**.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ requirements.txt ]; then pip install -r requirements.txt; fi
          if [ requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
      - name: Test with pytest and generate cov
        # see also https://pytest-cov.readthedocs.io/en/latest/config.html#caveats
        run: |
          python -m pytest --cov-report xml:$COVERAGE_OUTPUT_PATH --cov=. --cov-config=pyproject.toml \
            --continue-on-collection-errors --no-cov-on-fail
      - name: Upload code coverage to codeclimate
        uses: paambaati/codeclimate-action@v2.7.5
        env:
          # see https://docs.codeclimate.com/docs/finding-your-test-coverage-token
          # get from https://codeclimate.com/repos/6094b3d1a32d9010d700292e/settings/test_reporter
          CC_TEST_REPORTER_ID: '${{ secrets.CC_TEST_REPORTER_ID }}'
        with:
          # see https://github.com/marketplace/actions/code-climate-coverage-action#inputs
          coverageLocations: |
            ${{ env.COVERAGE_OUTPUT_PATH }}:coverage.py
      - name: Upload codecov report
        # see https://github.com/codecov/codecov-action
        uses: codecov/codecov-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ${{ env.COVERAGE_OUTPUT_PATH }}
          fail_ci_if_error: true # optional (default = false)
          verbose: true # optional (default = false)