diff --git a/.github/workflows/build-nightly.yml b/.github/workflows/build-nightly.yml index 40ca448f56eb..dbb0d05bd410 100644 --- a/.github/workflows/build-nightly.yml +++ b/.github/workflows/build-nightly.yml @@ -2,43 +2,21 @@ name: build-nightly run-name: ${{ github.workflow }} is to create nightly wheel file for pypi on: push: - branches: - - 'master' - schedule: - - cron: '0 0 * * *' - workflow_dispatch: + # branches: + # - 'main' + #schedule: + # - cron: '0 0 * * *' + #workflow_dispatch: + + jobs: - build-nightly: - runs-on: ubuntu-22.04 - permissions: - id-token: write - contents: read - steps: - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }} - aws-region: us-east-1 - - uses: actions/checkout@v3 - - run: | - pwd - sudo apt-get install python3-pip - pip3 --version - sudo pip3 install virtualenv - virtualenv venv - source venv/bin/activate - make -B build - deactivate - ls dist/ - aws s3 cp dist/*nightly*.whl s3://nm-github-actions/${{ github.event.repository.name }}/ - todaytime=`date +%Y%m%d` - date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7} - aws s3 cp log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/ - oldDate=`date --date='-2 month' +%Y%m%d` - oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"` - if [[ "${oldWhl}" != 'notfound' ]]; then - for oldwhl in $(echo "${oldWhl}" | awk '{print $4}') - do - echo "Remove old build ${oldwhl}" - aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl} - done - fi + + BUILD-SPARSEZOO_NIGHTLY: + + uses: ./.github/workflows/util.yml + with: + runs_on: ubuntu-22.04 + run_id: ${{ github.run_id }} + build_type: nightly + testmo_project_id: 9 + secrets: inherit diff --git a/.github/workflows/build-nightly.yml.bak b/.github/workflows/build-nightly.yml.bak new file mode 100644 index 000000000000..40ca448f56eb --- /dev/null +++ b/.github/workflows/build-nightly.yml.bak @@ -0,0 +1,44 @@ +name: build-nightly +run-name: ${{ github.workflow }} is to create nightly wheel file for pypi +on: + push: + branches: + - 'master' + schedule: + - cron: '0 0 * * *' + workflow_dispatch: +jobs: + build-nightly: + runs-on: ubuntu-22.04 + permissions: + id-token: write + contents: read + steps: + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }} + aws-region: us-east-1 + - uses: actions/checkout@v3 + - run: | + pwd + sudo apt-get install python3-pip + pip3 --version + sudo pip3 install virtualenv + virtualenv venv + source venv/bin/activate + make -B build + deactivate + ls dist/ + aws s3 cp dist/*nightly*.whl s3://nm-github-actions/${{ github.event.repository.name }}/ + todaytime=`date +%Y%m%d` + date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7} + aws s3 cp log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/ + oldDate=`date --date='-2 month' +%Y%m%d` + oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"` + if [[ "${oldWhl}" != 'notfound' ]]; then + for oldwhl in $(echo "${oldWhl}" | awk '{print $4}') + do + echo "Remove old build ${oldwhl}" + aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl} + done + fi diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index f09f76c10bf7..acd85a6c8abb 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -2,33 +2,18 @@ name: build-release run-name: ${{ github.workflow }} is to create release wheel file for pypi on: push: - branches: - - 'release/[0-9]+.[0-9]+' - workflow_dispatch: + # branches: + # - 'release/[0-9]+.[0-9]+' + # workflow_dispatch: + jobs: - build-release: - runs-on: ubuntu-22.04 - permissions: - id-token: write - contents: read - steps: - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }} - aws-region: us-east-1 - - uses: actions/checkout@v3 - - run: | - pwd - sudo apt-get install python3-pip - pip3 --version - sudo pip3 install virtualenv - virtualenv venv - source venv/bin/activate - sed -i 's/is_release = False/is_release = True/g' utils/neuralmagic/version.py - make -B build - deactivate - ls dist/ - aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/ - todaytime=`date +%Y%m%d` - date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7} - aws s3 cp log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/ + + BUILD-SPARSEZOO_RELEASE: + + uses: ./.github/workflows/util.yml + with: + runs_on: ubuntu-22.04 + run_id: ${{ github.run_id }} + build_type: release + testmo_project_id: 9 + secrets: inherit diff --git a/.github/workflows/build-release.yml.bak b/.github/workflows/build-release.yml.bak new file mode 100644 index 000000000000..f09f76c10bf7 --- /dev/null +++ b/.github/workflows/build-release.yml.bak @@ -0,0 +1,34 @@ +name: build-release +run-name: ${{ github.workflow }} is to create release wheel file for pypi +on: + push: + branches: + - 'release/[0-9]+.[0-9]+' + workflow_dispatch: +jobs: + build-release: + runs-on: ubuntu-22.04 + permissions: + id-token: write + contents: read + steps: + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }} + aws-region: us-east-1 + - uses: actions/checkout@v3 + - run: | + pwd + sudo apt-get install python3-pip + pip3 --version + sudo pip3 install virtualenv + virtualenv venv + source venv/bin/activate + sed -i 's/is_release = False/is_release = True/g' utils/neuralmagic/version.py + make -B build + deactivate + ls dist/ + aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/ + todaytime=`date +%Y%m%d` + date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7} + aws s3 cp log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/ diff --git a/.github/workflows/util.yml b/.github/workflows/util.yml new file mode 100644 index 000000000000..70d9268917dd --- /dev/null +++ b/.github/workflows/util.yml @@ -0,0 +1,123 @@ +name: report-to-testmo +on: + workflow_call: + inputs: + runs_on: + description: "runner label specifying instance running the job" + type: string + required: true + run_id: + description: "run id provided by GHA" + required: true + type: string + build_type: + description: "build type: nightly or release" + type: string + required: true + testmo_project_id: + description: "testmo project id" + type: string + required: true + +jobs: + + BUILD: + runs-on: ${{ inputs.runs_on }} + outputs: + status: ${{ steps.build.outputs.status }} + commitid: ${{ steps.build.outputs.commitid }} + permissions: + id-token: write + contents: read + steps: + + - name: repo checkout + uses: actions/checkout@v3 + + - name: s3 + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }} + aws-region: us-east-1 + + - name: build + id: build + run: | + pwd + sudo apt-get -y install python3-pip + pip3 --version + sudo pip3 install virtualenv + virtualenv venv + source venv/bin/activate + pip install -e . + if [[ "${{ inputs.build_type }}" = release ]]; then + sed -i 's/is_release = False/is_release = True/g' utils/neuralmagic/version.py + fi + status=$(make -B build || echo 'FAILED') + deactivate + echo "=========== Build log ===========" + echo "${status}" + echo "commitid=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" + echo "=========== Build status ===========" + if [[ "${status}" = "FAILED" ]]; then + echo "${{ github.event.repository.name }} build failed" + echo "status=failed" >> "$GITHUB_OUTPUT" + exit 1 + else + echo "${{ github.event.repository.name }} build success" + fi + echo "=========== Generated build ===========" + ls dist/ + echo "=========== Copy build to S3 ===========" + aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/ + if [ $? -eq 0 ]; then + echo "ok: copied to s3://nm-github-actions/${{ github.event.repository.name }}/" + echo "status=success" >> "$GITHUB_OUTPUT" + else + echo "failed: copied to s3://nm-github-actions/${{ github.event.repository.name }}/" + echo "status=failed" >> "$GITHUB_OUTPUT" + exit 1 + fi + oldDate=`date --date='-2 month' +%Y%m%d` + oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"` + if [[ "${oldWhl}" != 'notfound' ]]; then + for oldwhl in $(echo "${oldWhl}" | awk '{print $4}') + do + echo "Remove old build ${oldwhl} in S3" + aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl} + done + fi + + TESTMO: + if: success() || failure() + needs: BUILD + runs-on: ${{ inputs.runs_on }} + steps: + + - id: report + run: | + echo "node: $(node -v)" + echo "npm: $(npm -v)" + echo "Installing testmo cli..." + sudo npm install -g @testmo/testmo-cli + export TESTMO_TOKEN=${{ secrets.TESTMO_TEST_TOKEN }} + TESTMO_URL="https://neuralmagic.testmo.net" + todaytime=`date +%Y%m%d` + name="${{ github.event.repository.name }} ${{ inputs.build_type }} ${todaytime} ${{ needs.BUILD.outputs.commitid }} RunID:${{ inputs.run_id }}" + echo "========== Build info ===========" + echo "name: ${name}" + echo "build status: ${{ needs.BUILD.outputs.status }}" + echo "========== Report to testmo ===========" + echo "testmo automation:run:submit \\" + echo " --instance ${TESTMO_URL} \\" + echo " --project-id ${{ inputs.testmo_project_id }} \\" + echo " --name ${name} \\" + echo " --source ${{ github.event.repository.name }} \\" + echo " --results result.xml" + testmo automation:run:submit \ + --instance "${TESTMO_URL}" \ + --project-id ${{ inputs.testmo_project_id }} \ + --name "${name}" \ + --source "${{ github.event.repository.name }}" \ + --results result.xml \ + -- cat result.xml