ci: cherry-pick workflow and container build improvements (#957)

* ci: remove redundant usages of the free-disk-space action Co-authored-by: Techassi <[email protected]> * ci: Add attempt number to the workflow run name, and URL in the slack message (#941) Co-authored-by: Techassi <[email protected]> * ci: register a dummy test_build_images workflow which will be completed in a separate PR (#947) * Use our build-repo for Maven downloads & other updates (#953) * Use our build-repo for Maven downloads * ci(build): Continue matrix builds on failure so there is less jobs to rerun when there are transient failures * Update Spark Mvn build as well * Hopefully silence Spark build * Update FMPP version This is because FMPP Maven Plugin depends on FMPP in version 0.9.14 which itself depends on a Freemarker version that has not been pinned. Instead it specifies a "range" which resolves to a SNAPSHOT version which we don't want. * Add Stackable repo to Kafka build as well * Update changelog * Update CHANGELOG.md Co-authored-by: Nick <[email protected]> --------- Co-authored-by: Nick Larsen <[email protected]> Co-authored-by: Nick <[email protected]> * ci: Fix static analysis errors (#955) * ci: Fix high severity errors * ci: Set persist-credentials to false * ci: Move step output to env var * chore: Add changelog entry * ci(build): stagger the scheduled builds to ease load on external systems (#954) --------- Co-authored-by: Techassi <[email protected]> Co-authored-by: Lars Francke <[email protected]>
stackabletech · Dec 13, 2024 · ec65c46 · ec65c46
1 parent 7e7f845
commit ec65c46
Show file tree

Hide file tree

Showing 41 changed files with 545 additions and 138 deletions.
diff --git a/.github/workflows/build_airflow.yaml b/.github/workflows/build_airflow.yaml
@@ -1,5 +1,7 @@
 ---
 name: Build Airflow
+run-name: |
+  Build Airflow (attempt #${{ github.run_attempt }})
 
 env:
   PRODUCT_NAME: airflow
@@ -8,7 +10,7 @@ env:
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 1 1/2 * *' # https://crontab.guru/#0_1_1/2_*_*
+    - cron: '0 0 1/2 * *' # https://crontab.guru/#0_0_1/2_*_*
   push:
     branches: [main]
     tags: ['*']
@@ -27,6 +29,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
       - id: shard
         uses: stackabletech/actions/shard@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -41,6 +45,7 @@ jobs:
       id-token: write
     runs-on: ${{ matrix.runner.name }}
     strategy:
+      fail-fast: false
       matrix:
         runner:
           - {name: "ubuntu-latest", arch: "amd64"}
@@ -49,6 +54,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
       - name: Free Disk Space
         uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
@@ -89,14 +96,15 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
-      - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
       - name: Publish and Sign Image Index Manifest to docker.stackable.tech
         uses: stackabletech/actions/publish-index-manifest@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -126,7 +134,7 @@ jobs:
           channel-id: "C07UG6JH44F" # notifications-container-images
           payload: |
             {
-              "text": "*${{ github.workflow }}* failed",
+              "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
               "attachments": [
                 {
                   "pretext": "See the details below for a summary of which job(s) failed.",
@@ -152,7 +160,7 @@ jobs:
                     {
                       "type": "button",
                       "text": "Go to workflow run",
-                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
                     }
                   ]
                 }

diff --git a/.github/workflows/build_druid.yaml b/.github/workflows/build_druid.yaml
@@ -1,5 +1,7 @@
 ---
 name: Build Druid
+run-name: |
+  Build Druid (attempt #${{ github.run_attempt }})
 
 env:
   PRODUCT_NAME: druid
@@ -29,6 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
       - id: shard
         uses: stackabletech/actions/shard@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -43,6 +47,7 @@ jobs:
       id-token: write
     runs-on: ${{ matrix.runner.name }}
     strategy:
+      fail-fast: false
       matrix:
         runner:
           - {name: "ubuntu-latest", arch: "amd64"}
@@ -51,6 +56,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
       - name: Free Disk Space
         uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
@@ -91,14 +98,15 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
-      - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
       - name: Publish and Sign Image Index Manifest to docker.stackable.tech
         uses: stackabletech/actions/publish-index-manifest@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -128,7 +136,7 @@ jobs:
           channel-id: "C07UG6JH44F" # notifications-container-images
           payload: |
             {
-              "text": "*${{ github.workflow }}* failed",
+              "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
               "attachments": [
                 {
                   "pretext": "See the details below for a summary of which job(s) failed.",
@@ -154,7 +162,7 @@ jobs:
                     {
                       "type": "button",
                       "text": "Go to workflow run",
-                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
                     }
                   ]
                 }

diff --git a/.github/workflows/build_hadoop.yaml b/.github/workflows/build_hadoop.yaml
@@ -1,5 +1,7 @@
 ---
 name: Build Hadoop
+run-name: |
+  Build Hadoop (attempt #${{ github.run_attempt }})
 
 env:
   PRODUCT_NAME: hadoop
@@ -8,7 +10,7 @@ env:
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 1 1/2 * *' # https://crontab.guru/#0_1_1/2_*_*
+    - cron: '0 2 1/2 * *' # https://crontab.guru/#0_2_1/2_*_*
   push:
     branches: [main]
     tags: ['*']
@@ -29,6 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
       - id: shard
         uses: stackabletech/actions/shard@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -43,6 +47,7 @@ jobs:
       id-token: write
     runs-on: ${{ matrix.runner.name }}
     strategy:
+      fail-fast: false
       matrix:
         runner:
           - {name: "ubuntu-latest", arch: "amd64"}
@@ -51,6 +56,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
       - name: Free Disk Space
         uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
@@ -91,14 +98,15 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
-      - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
       - name: Publish and Sign Image Index Manifest to docker.stackable.tech
         uses: stackabletech/actions/publish-index-manifest@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -128,7 +136,7 @@ jobs:
           channel-id: "C07UG6JH44F" # notifications-container-images
           payload: |
             {
-              "text": "*${{ github.workflow }}* failed",
+              "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
               "attachments": [
                 {
                   "pretext": "See the details below for a summary of which job(s) failed.",
@@ -154,7 +162,7 @@ jobs:
                     {
                       "type": "button",
                       "text": "Go to workflow run",
-                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
                     }
                   ]
                 }

diff --git a/.github/workflows/build_hbase.yaml b/.github/workflows/build_hbase.yaml
@@ -1,5 +1,7 @@
 ---
 name: Build HBase
+run-name: |
+  Build HBase (attempt #${{ github.run_attempt }})
 
 env:
   PRODUCT_NAME: hbase
@@ -8,7 +10,7 @@ env:
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 1 1/2 * *' # https://crontab.guru/#0_1_1/2_*_*
+    - cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_*
   push:
     branches: [main]
     tags: ['*']
@@ -30,6 +32,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
       - id: shard
         uses: stackabletech/actions/shard@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -44,6 +48,7 @@ jobs:
       id-token: write
     runs-on: ${{ matrix.runner.name }}
     strategy:
+      fail-fast: false
       matrix:
         runner:
           - {name: "ubuntu-latest", arch: "amd64"}
@@ -52,6 +57,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
       - name: Free Disk Space
         uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
@@ -92,14 +99,15 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
-      - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
       - name: Publish and Sign Image Index Manifest to docker.stackable.tech
         uses: stackabletech/actions/publish-index-manifest@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -129,7 +137,7 @@ jobs:
           channel-id: "C07UG6JH44F" # notifications-container-images
           payload: |
             {
-              "text": "*${{ github.workflow }}* failed",
+              "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
               "attachments": [
                 {
                   "pretext": "See the details below for a summary of which job(s) failed.",
@@ -155,7 +163,7 @@ jobs:
                     {
                       "type": "button",
                       "text": "Go to workflow run",
-                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
                     }
                   ]
                 }

diff --git a/.github/workflows/build_hello-world.yaml b/.github/workflows/build_hello-world.yaml
@@ -1,5 +1,7 @@
 ---
 name: Build Hello-World
+run-name: |
+  Build Hello-World (attempt #${{ github.run_attempt }})
 
 env:
   PRODUCT_NAME: hello-world
@@ -8,7 +10,7 @@ env:
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 1 1/2 * *' # https://crontab.guru/#0_1_1/2_*_*
+    - cron: '0 1 2/2 * *' # https://crontab.guru/#0_1_2/2_*_*
   push:
     branches: [main]
     tags: ['*']
@@ -25,6 +27,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
       - id: shard
         uses: stackabletech/actions/shard@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -39,6 +43,7 @@ jobs:
       id-token: write
     runs-on: ${{ matrix.runner.name }}
     strategy:
+      fail-fast: false
       matrix:
         runner:
           - {name: "ubuntu-latest", arch: "amd64"}
@@ -47,6 +52,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
       - name: Free Disk Space
         uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
@@ -87,14 +94,15 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
 
-      - name: Free Disk Space
-        uses: stackabletech/actions/free-disk-space@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
       - name: Publish and Sign Image Index Manifest to docker.stackable.tech
         uses: stackabletech/actions/publish-index-manifest@a3f7587879e9f12e04a29fd26435949aaa4fd59c # 0.2.0
         with:
@@ -124,7 +132,7 @@ jobs:
           channel-id: "C07UG6JH44F" # notifications-container-images
           payload: |
             {
-              "text": "*${{ github.workflow }}* failed",
+              "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
               "attachments": [
                 {
                   "pretext": "See the details below for a summary of which job(s) failed.",
@@ -150,7 +158,7 @@ jobs:
                     {
                       "type": "button",
                       "text": "Go to workflow run",
-                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                      "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
                     }
                   ]
                 }