From f43762fa7cc5a987be967b0a519ff714e9062841 Mon Sep 17 00:00:00 2001
From: Andy Linfoot <78757007+andy-neuma@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:12:08 -0400
Subject: [PATCH] expand benchmark to h100's (#371)

SUMMARY:
* updated "build test" to accept an array of benchmarking labels
* updated "remote push" and "nightly" workflows to include benchmarking
on h100's
* adjusted docker job to have same criteria as upload job. did this
since upload could fail, but for auth reasons and this shouldn't stop us
from push docker.

TEST PLAN:
runs on remote push

---------

Co-authored-by: andy-neuma <andy@neuralmagic.com>
---
 .github/workflows/nm-build-test.yml  | 11 +++++++----
 .github/workflows/nm-nightly.yml     |  2 +-
 .github/workflows/nm-remote-push.yml |  2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/nm-build-test.yml b/.github/workflows/nm-build-test.yml
index 1acc2d31717b3..8884bc3dc4a39 100644
--- a/.github/workflows/nm-build-test.yml
+++ b/.github/workflows/nm-build-test.yml
@@ -58,10 +58,10 @@ on:
         required: true
 
       # benchmark related parameters
-      benchmark_label:
-        description: "requested benchmark label (specifies instance)"
+      benchmark_labels:
+        description: "stringified Json array of benchmark labels"
         type: string
-        default: ""
+        required: true
       benchmark_config_list_file:
         description: "benchmark configs file, e.g. 'nm_benchmark_nightly_configs_list.txt'"
         type: string
@@ -136,9 +136,12 @@ jobs:
     BENCHMARK:
         needs: [BUILD]
         if: success()
+        strategy:
+            matrix:
+                benchmark_label: ${{ fromJson(inputs.benchmark_labels) }}
         uses: ./.github/workflows/nm-benchmark.yml
         with:
-            label: ${{ inputs.benchmark_label }}
+            label: ${{ matrix.benchmark_label }}
             benchmark_config_list_file: ${{ inputs.benchmark_config_list_file }}
             timeout: ${{ inputs.benchmark_timeout }}
             gitref: ${{ github.ref }}
diff --git a/.github/workflows/nm-nightly.yml b/.github/workflows/nm-nightly.yml
index 434f2b9032b1b..494198bf496fc 100644
--- a/.github/workflows/nm-nightly.yml
+++ b/.github/workflows/nm-nightly.yml
@@ -39,7 +39,7 @@ jobs:
                             {"python":"3.11.4","label":"gcp-k8s-l4-solo","test":"neuralmagic/tests/test_skip_env_vars/full.txt"}]'
             test_timeout: 480
 
-            benchmark_label: gcp-k8s-l4-solo
+            benchmark_labels: '["gcp-k8s-l4-solo", "k8s-h100-solo"]'
             benchmark_config_list_file: ./.github/data/nm_benchmark_base_config_list.txt
             benchmark_timeout: 480
             push_benchmark_results_to_gh_pages: "${{ github.event_name == 'schedule' || inputs.push_benchmark_results_to_gh_pages }}"
diff --git a/.github/workflows/nm-remote-push.yml b/.github/workflows/nm-remote-push.yml
index a44274d9e8a11..17108f345f891 100644
--- a/.github/workflows/nm-remote-push.yml
+++ b/.github/workflows/nm-remote-push.yml
@@ -25,7 +25,7 @@ jobs:
                             {"python":"3.11.4","label":"gcp-k8s-l4-solo","test":"neuralmagic/tests/test_skip_env_vars/smoke.txt"}]'
             test_timeout: 480
 
-            benchmark_label: gcp-k8s-l4-solo
+            benchmark_labels: '["gcp-k8s-l4-solo", "k8s-h100-solo"]'
             benchmark_config_list_file: ./.github/data/nm_benchmark_base_config_list.txt
             benchmark_timeout: 480