upload RELEASE wheel to pypi.org (#361)

Besides uploading all generated assets to our NeuralMagic pypi server,
we also want to push generated wheels to the external pypi.org server
for RELEASE versions only.

The structure of the workflows doesn't allow pre-testing of these
changes, since they should only apply when wf_category is "RELEASE".
Discussing w/ @andy-neuma , we'll just have to see if it fails/succeeds
when we're ready to release.

---------

Co-authored-by: Andy Linfoot <[email protected]>
Co-authored-by: andy-neuma <[email protected]>

.github/actions/nm-cp-assets/action.yml

@@ -1,18 +1,17 @@
 name: cp assets
 description: "cp whl and tarfile to Google storage 'neuralmagic-public-pypi/dist'"
+inputs:
+  whl:
+    description: "the wheel asset file path"
+    required: true
+  targz:
+    description: "the tar.gz asset file path"
+    required: true
 runs:
   using: composite
   steps:
     - id: cp_assets
       run: |
-        WHL=$(find assets -type f -name "*nm_vllm*.whl")
-        WHL_FILE=$(basename ${WHL})
-        echo "whl: ${WHL}"
-        echo "whl_file: ${WHL_FILE}"
-        TAR=$(find assets -path "*nm-vllm*.tar.gz" -type f -name "nm-vllm*.tar.gz")
-        TAR_FILE=$(basename ${TAR})
-        echo "tar: ${TAR}"
-        echo "tar_file: ${TAR_FILE}"
-        gcloud storage cp ${WHL} gs://neuralmagic-public-pypi/dist/${WHL_FILE}
-        gcloud storage cp ${TAR} gs://neuralmagic-public-pypi/dist/${TAR_FILE}
+        gcloud storage cp ${WHL} gs://neuralmagic-public-pypi/dist/${{ inputs.whl }}
+        gcloud storage cp ${TAR} gs://neuralmagic-public-pypi/dist/${{ inputs.targz }}
       shell: bash

.github/actions/nm_whl_tar_gz_names/action.yml

@@ -0,0 +1,25 @@
+name: get wheel and tar.gz names
+description: "retrieve the whl and tarfile names from existing assets"
+outputs:
+  whl:
+    description: "the wheel asset file path"
+    value: ${{ steps.whl_targz_names.outputs.whl }}
+  targz:
+    description: "the tar.gz asset file path"
+    value: ${{ steps.whl_targz_names.outputs.targz }}
+runs:
+  using: composite
+  steps:
+    - id: whl_targz_names
+      run: |
+        WHL=$(find assets -type f -name "*nm_vllm*.whl")
+        WHL_FILE=$(basename ${WHL})
+        echo "whl: ${WHL}"
+        echo "whl_file: ${WHL_FILE}"
+        TAR=$(find assets -path "*nm-vllm*.tar.gz" -type f -name "nm-vllm*.tar.gz")
+        TAR_FILE=$(basename ${TAR})
+        echo "tar: ${TAR}"
+        echo "tar_file: ${TAR_FILE}"
+        echo "whl=${WHL_FILE}" >> $GITHUB_OUTPUT
+        echo "targz=${TAR_FILE}" >> $GITHUB_OUTPUT
+      shell: bash

.github/workflows/nm-build-test.yml

@@ -167,11 +167,12 @@ jobs:
     UPLOAD:
         needs: [TEST, BENCHMARK, LM-EVAL]
         if: ${{ inputs.push_to_pypi }}
-        uses: ./.github/workflows/nm-upload-assets-to-gcp.yml
+        uses: ./.github/workflows/nm-upload-assets.yml
         with:
             label: gcp-k8s-util
             timeout: ${{ inputs.build_timeout }}
             gitref: ${{ github.ref }}
+            wf_category: ${{ inputs.wf_category }}
         secrets: inherit
 
     # update docker

.github/workflows/nm-upload-assets-to-gcp.yml → .github/workflows/nm-upload-assets.yml

@@ -16,6 +16,10 @@ on:
                 description: 'git commit hash or branch name'
                 type: string
                 required: true
+            wf_category:
+                description: "workflow category: REMOTE, NIGHTLY, RELEASE"
+                type: string
+                default: "REMOTE"
 
     workflow_dispatch:
         inputs:
@@ -31,6 +35,10 @@ on:
                 description: 'git commit hash or branch name'
                 type: string
                 required: true
+            wf_category:
+                description: "workflow category: REMOTE, NIGHTLY, RELEASE"
+                type: string
+                default: "REMOTE"
 
 jobs:
 
@@ -68,6 +76,30 @@ jobs:
               with:
                   path: assets
 
+            - name: get wheel and tar.gz names
+              id: whl_targz_names
+              uses: ./.github/actions/nm_whl_tar_gz_names
+
+            # this workflow is only run if push-to-pypi is True, and we only
+            # want to push RELEASE assets to the external pypi.org
+            # publish the wheel file
+            - name: push wheel to pypi.org
+              if: ${{ inputs.wf_category == "RELEASE" }}
+              uses: neuralmagic/nm-actions/actions/publish_whl/[email protected]
+              with:
+                username: ${{ secrets.PYPI_PUBLIC_USER }}
+                password: ${{ secrets.PYPI_PUBLIC_AUTH }}
+                whl: ${{ steps.whl_targz_names.outputs.whl }}
+
+            # publish the tar.gz file
+            - name: push tar.gz to pypi.org
+              if: ${{ inputs.wf_category == "RELEASE" }}
+              uses: neuralmagic/nm-actions/actions/publish_whl/[email protected]
+              with:
+                username: ${{ secrets.PYPI_PUBLIC_USER }}
+                password: ${{ secrets.PYPI_PUBLIC_AUTH }}
+                whl: ${{ steps.whl_targz_names.outputs.targz }}
+
             # GCP
             - name: 'Authenticate to Google Cloud'
               id: auth
@@ -85,6 +117,9 @@ jobs:
             - name: cp assets
               id: cp-assets
               uses: ./.github/actions/nm-cp-assets/
+              with:
+                whl: ${{ steps.whl_targz_names.outputs.whl }}
+                targz: ${{ steps.whl_targz_names.outputs.targz }}
 
             - name: trigger stratus nm-pypi update workflow to update nm-pypi index
               uses: actions/github-script@v6