/.github/{scripts,workflows}: working through some pr regressions lab…

…eling
dolthub · Jan 12, 2024 · baa9105 · baa9105
1 parent 70b785d
commit baa9105
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 3 deletions.
diff --git a/.github/scripts/sql-correctness/current_correctness.txt b/.github/scripts/sql-correctness/current_correctness.txt
@@ -0,0 +1 @@
+99.997743
diff --git a/.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh b/.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh
@@ -3,7 +3,7 @@
 set -e
 
 if [ "$#" -lt 5 ]; then
-    echo  "Usage: ./get-dolt-correctness-job-json.sh <jobname> <version> <timeprefix> <actorprefix> <format> <nomsBinFormat> <issueNumber>"
+    echo  "Usage: ./get-dolt-correctness-job-json.sh <jobname> <version> <timeprefix> <actorprefix> <format> <nomsBinFormat> <issueNumber> <regressComp>"
     exit 1
 fi
 
@@ -14,6 +14,7 @@ actorprefix="$4"
 format="$5"
 nomsBinFormat="$6"
 issueNumber="$7"
+regressComp="$8"
 
 precision="6"
 
@@ -25,6 +26,12 @@ if [ -n "$issueNumber" ]; then
   issueNumber="\"--issue-number=$issueNumber\","
 fi
 
+regressPrec=""
+if [ -n "$regressComp" ]; then
+  regressComp="\"--regress-compare=$regressComp\","
+  regressPrec="\"--regress-precision=$precision\","
+fi
+
 resultCountQuery="select version, result, count(*) as total from results where result != 'skipped' group by result;"
 testCountQuery="select version, count(*) as total_tests from results where result != 'skipped';"
 correctnessQuery="select ROUND(100.0 * (cast(ok_results.total as decimal) / (cast(all_results.total as decimal) + .000001)), $precision) as correctness_percentage from (select count(*) as total from results where result = 'ok') as ok_results join (select count(*) as total from results where result != 'skipped') as all_results"
@@ -72,6 +79,8 @@ echo '
               "--version='$version'",
               '"$nomsBinFormat"'
               '"$issueNumber"'
+              '"$regressComp"'
+              '"$regressPrec"'
               "--bucket=sql-correctness-github-actions-results",
               "--region=us-west-2",
               "--results-dir='$timeprefix'",

diff --git a/.github/scripts/sql-correctness/run-correctness.sh b/.github/scripts/sql-correctness/run-correctness.sh
@@ -32,6 +32,13 @@ if [ -z "$MODE" ]; then
     exit 1
 fi
 
+if [ -n "$PR_NUMBER" ]; then
+  if [ -z "$REGRESS_COMP" ]; then
+    echo "Must set REGRESS_COMP for PR correctness comparisons"
+    exit 1
+  fi
+fi
+
 # use first 8 characters of VERSION to differentiate
 # jobs
 short=${VERSION:0:8}
@@ -71,7 +78,8 @@ source \
   "$actorprefix" \
   "$format" \
   "$NOMS_BIN_FORMAT" \
-  "$issuenumber" > job.json
+  "$issuenumber" \
+  "$REGRESS_COMP" > job.json
 
 # delete existing job with same name if this is a pr job
 if [ -n "$PR_NUMBER" ]; then

diff --git a/.github/workflows/k8s-sql-correctness.yaml b/.github/workflows/k8s-sql-correctness.yaml
@@ -39,6 +39,7 @@ jobs:
       - name: Create SQL Correctness K8s Job
         run: ./.github/scripts/sql-correctness/run-correctness.sh
         env:
+          REGRESS_COMP: ${{ github.event.client_payload.regress_comp }}
           PR_NUMBER: ${{ github.event.client_payload.issue_number }}
           VERSION: ${{ github.event.client_payload.version }}
           MODE: ${{ github.event.client_payload.mode }}

diff --git a/.github/workflows/pull-report-test.yaml b/.github/workflows/pull-report-test.yaml
@@ -0,0 +1,41 @@
+on:
+  push:
+    branches:
+      - 'db/compare'
+
+jobs:
+  report-pull-request:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Configure AWS Credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Remove Passing Labels if regression detected
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { ACTOR, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env;
+            const issue_number = parseInt(ISSUE_NUMBER, 10);
+            const { owner, repo } = context.repo;
+            try {
+                const labels = await github.rest.issues.listLabelsOnIssue({
+                  issue_number,
+                  owner,
+                  repo,
+                });
+
+                console.log("labels are:", labels);
+
+            } catch(e) {
+              console.error(e)
+            }
+        env:
+          ACTOR: ${{ github.event.client_payload.actor }}
+          ISSUE_NUMBER: '7309'
+          LABEL: 'correctness_approved'
diff --git a/.github/workflows/pull-report.yaml b/.github/workflows/pull-report.yaml
@@ -48,3 +48,27 @@ jobs:
           ACTOR: ${{ github.event.client_payload.actor }}
           ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
           FORMAT: ${{ github.event.client_payload.noms_bin_format }}
+      - name: Remove Passing Labels if regression detected
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { ACTOR, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env;
+            const issue_number = parseInt(ISSUE_NUMBER, 10);
+            const { owner, repo } = context.repo;
+            try {
+                const labels = github.rest.issues.createComment({
+                issue_number,
+                owner,
+                repo,
+                body: `@${ACTOR} ${FORMAT}\n ${data}`
+              });
+            } catch(e) {
+              console.error(e)
+            }
+        env:
+          ACTOR: ${{ github.event.client_payload.actor }}
+          ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
+      # if this is a regression
+      # first remove all correctness_approved labels from the pr
+      # if this
diff --git a/.github/workflows/sql-regressions.yaml b/.github/workflows/sql-regressions.yaml
@@ -9,9 +9,20 @@ jobs:
     name: Set Version and Actor
     runs-on: ubuntu-22.04
     outputs:
+      regress_comp: ${{ steps.regress-comp.outputs.regress_comp }}
       version: ${{ steps.set-vars.outputs.version }}
       actor: ${{ steps.set-vars.outputs.actor }}
     steps:
+      - name: Checkout PR HEAD REF
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.base_ref }}
+      - name: Get Current Correctness
+        id: regress-comp
+        working-directory: ./.github/scripts/sql-correctness
+        run: |
+          out=$(cat current_correctness.txt)
+          echo "regress_comp=$out" >> $GITHUB_OUTPUT
       - name: Checkout PR HEAD REF
         uses: actions/checkout@v3
         with:
@@ -41,4 +52,4 @@ jobs:
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}
           event-type: sql-correctness
-          client-payload: '{"issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}'
+          client-payload: '{"issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "regress_comp": "${{ needs.set-version-actor.outputs.regress_comp }}", "version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}'