benchmarks.yml

name: Benchmark

on:
  pull_request:
    types: [opened, synchronize, reopened, labeled]

permissions:
  contents: read
  pull-requests: write

jobs:
  benchmarks:
    if: contains(github.event.pull_request.labels.*.name, 'request-benchmarks')
    runs-on: ubuntu-latest

    steps:
      - name: Install wrk
        run: |
          sudo apt-get update
          sudo apt-get install -y wrk

      - name: Checkout Source Branch
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          path: source

      - name: Setup Node.js for Source Branch
        uses: actions/setup-node@v3
        with:
          node-version: '22.x'

      - name: Install Dependencies for Source Branch
        run: |
          cd source
          npm install

      - name: Run Benchmarks on Source Branch
        run: |
          cd source/benchmarks
          make > source_results.log

      - name: Save Source Results
        id: save-source-results
        uses: actions/upload-artifact@v2
        with:
          name: source-results
          path: source/benchmarks/source_results.log

      - name: Checkout Target Branch
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.base.ref }}
          path: target

      - name: Setup Node.js for Target Branch
        uses: actions/setup-node@v3
        with:
          node-version: '22.x'

      - name: Install Dependencies for Target Branch
        run: |
          cd target
          npm install

      - name: Run Benchmarks on Target Branch
        run: |
          cd target/benchmarks
          make > target_results.log

      - name: Save Target Results
        id: save-target-results
        uses: actions/upload-artifact@v2
        with:
          name: target-results
          path: target/benchmarks/target_results.log

      - name: Download Source Results
        uses: actions/download-artifact@v2
        with:
          name: source-results
          path: source-results

      - name: Download Target Results
        uses: actions/download-artifact@v2
        with:
          name: target-results
          path: target-results

      - name: Compare Results and Comment on PR
        env:
            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
            PR_NUMBER: ${{ github.event.pull_request.number }}
            PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
            PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
            PR_COMMENTS_URL: ${{ github.event.pull_request.comments_url }}
        run: |
            set -e
            echo "Comparing benchmark results..."
            # Check if source results file exists
            if [ ! -f source-results/source_results.log ]; then
              echo "Source results file not found!"
              exit 1
            fi
            
            # Check if target results file exists
            if [ ! -f target-results/target_results.log ]; then
              echo "Target results file not found!"
              exit 1
            fi
            
            # Compare the benchmark results and generate a summary
            SOURCE_RESULTS=$(cat source-results/source_results.log)
            TARGET_RESULTS=$(cat target-results/target_results.log)
            DIFF=$(diff <(echo "$SOURCE_RESULTS") <(echo "$TARGET_RESULTS")) || true
            
            # Debugging output
            echo "Source Results:"
            echo "$SOURCE_RESULTS"
            echo "Target Results:"
            echo "$TARGET_RESULTS"
            echo "Differences:"
            echo "$DIFF"
            
            # Create a comment body
            COMMENT_BODY='### Benchmark Comparison\n\n'
            COMMENT_BODY+="**Source Branch (${PR_HEAD_REF}) Results:**\n\`\`\`\n${SOURCE_RESULTS}\n\`\`\`\n\n"
            COMMENT_BODY+="**Target Branch (${PR_BASE_REF}) Results:**\n\`\`\`\n${TARGET_RESULTS}\n\`\`\`\n\n"
            COMMENT_BODY+="**Differences:**\n\`\`\`\n${DIFF}\n\`\`\`\n"
  
            # Post the comment to the PR
            curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \
            -H "Content-Type: application/json" \
            -d "{\"body\":\"${COMMENT_BODY}\"}" \
            "$PR_COMMENTS_URL"
        continue-on-error: true

      - name: Remove request-benchmarks Label
        run: |
          curl -s -X DELETE -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
          -H "Content-Type: application/json" \
          "${{ github.event.pull_request.issue_url }}/labels/request-benchmarks"