Initial commit

.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Publish site
+
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+      - docs
+
+jobs:
+
+  publish:
+    name: Publish the site
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository normally
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Init docs
+        run: bash initdocs.sh
+
+      - name: Run Mkdocs deploy
+        run: mkdocs gh-deploy --force

.github/workflows/run-mlperf-inference-submission-checker.yml

@@ -0,0 +1,31 @@
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference submission checker
+
+on:
+  pull_request:
+    branches: [ "main", "v4.1" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    env:
+      CM_INDEX: "on"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.10" ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cm4mlops
+    - name: Run MLPerf Inference Submission Checker
+      run: |
+        cmr "run mlc mlcommons mlperf inference mlperf-inference submission checker submission-checker mlc-submission-checker" --adr.inference-src.tags=_repo.https://github.com/GATEOverflow/inference --quiet --extra_args=" --skip-extra-files-in-root-check" --submission_dir=./

.github/workflows/update-results.yml

@@ -0,0 +1,51 @@
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference results updater
+
+
+on:
+  push:
+    branches: [ "main", "v4.1" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    env:
+      CM_INDEX: "on"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.10" ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cm4mlops
+
+    - name: Run MLPerf Inference Submission Checker and generate results summary
+      run: |
+        cm run script --tags=run,mlperf,inference,submission-checker --adr.inference-src.tags=_repo.https://github.com/GATEOverflow/inference --quiet --extra_args=" --skip-extra-files-in-root-check"  --submission_dir=./ > >(tee -a out.txt) 2> >(tee -a checker_log.txt >&2)
+        cm run script --tags=convert,from-csv,to-md --csv_file=summary.csv --md_file=README.md
+
+        USER=GitHub Action
+        [email protected]
+
+        git config --global user.name "$USER"
+        git config --global user.email "$EMAIL"
+        git remote set-url origin https://x-access-token:${{ secrets.ACCESS_TOKEN }}@github.com/${{ github.repository }}
+        git add summary*
+        echo -e 'Please download [summary.xlsx](summary.xlsx) to view the most recent results. [This page](https://docs.google.com/spreadsheets/d/e/2PACX-1vSCu8F7Hwck-AGJ5kWxi2G3xhO5MJoc_igybvsxjCt-2fEEYyf2BIcR0rTXW0eUzg/pubhtml) shows the results which may not be the latest. \n ```' > temp
+        tail -n 16 checker_log.txt >> temp
+        echo -e '\n```\n' >> temp
+        cat temp | cat - README.md > temp1
+        head -n 100 temp1 > README.md
+        git add README.md
+        
+        git diff-index --quiet HEAD || (git commit -am "Updated results summary" && git push)

README.md

@@ -0,0 +1,23 @@
+Please download [summary.xlsx](summary.xlsx) to view the most recent results. [This page](https://docs.google.com/spreadsheets/d/e/2PACX-1vSCu8F7Hwck-AGJ5kWxi2G3xhO5MJoc_igybvsxjCt-2fEEYyf2BIcR0rTXW0eUzg/pubhtml) shows the results which may not be the latest. 
+ ```
+[2024-10-04 09:53:00,935 submission_checker.py:2936 INFO] Results=0, NoResults=0, Power Results=0
+[2024-10-04 09:53:00,935 submission_checker.py:2943 INFO] ---
+[2024-10-04 09:53:00,935 submission_checker.py:2944 INFO] Closed Results=0, Closed Power Results=0
+
+[2024-10-04 09:53:00,935 submission_checker.py:2949 INFO] Open Results=0, Open Power Results=0
+
+[2024-10-04 09:53:00,935 submission_checker.py:2954 INFO] Network Results=0, Network Power Results=0
+
+[2024-10-04 09:53:00,935 submission_checker.py:2959 INFO] ---
+[2024-10-04 09:53:00,935 submission_checker.py:2961 INFO] Systems=0, Power Systems=0
+[2024-10-04 09:53:00,935 submission_checker.py:2962 INFO] Closed Systems=0, Closed Power Systems=0
+[2024-10-04 09:53:00,935 submission_checker.py:2967 INFO] Open Systems=0, Open Power Systems=0
+[2024-10-04 09:53:00,935 submission_checker.py:2972 INFO] Network Systems=0, Network Power Systems=0
+[2024-10-04 09:53:00,935 submission_checker.py:2977 INFO] ---
+[2024-10-04 09:53:00,935 submission_checker.py:2982 INFO] SUMMARY: submission looks OK
+INFO:root:       ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-submission-checker/customize.py
+
+```
+
+| Organization   | Availability   | Division   | SystemType   | SystemName   | Platform   | Model   | MlperfModel   | Scenario   | Result   | Accuracy   | number_of_nodes   | host_processor_model_name   | host_processors_per_node   | host_processor_core_count   | accelerator_model_name   | accelerators_per_node   | Location   | framework   | operating_system   | notes   | compliance   | errors   | version   | inferred   | has_power   | Units   | weight_data_types   |
+|----------------|----------------|------------|--------------|--------------|------------|---------|---------------|------------|----------|------------|-------------------|-----------------------------|----------------------------|-----------------------------|--------------------------|-------------------------|------------|-------------|--------------------|---------|--------------|----------|-----------|------------|-------------|---------|---------------------|

docinit.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+if [ ! -e docs ]; then
+    git clone https://github.com/GATEOverflow/inference_results_visualization_template.git docs
+    test $? -eq 0 || exit $?
+fi
+
+python3 -m pip install -r docs/requirements.txt
+
+if [ ! -e overrides ]; then
+    cp -r docs/overrides overrides
+    test $? -eq 0 || exit $?
+fi
+
+repo_owner=${INFERENCE_RESULTS_REPO_OWNER:-mlcommons}
+repo_branch=${INFERENCE_RESULTS_REPO_BRANCH:-main}
+repo_name=${INFERENCE_RESULTS_REPO_NAME:-inference_results_${INFERENCE_RESULTS_VERSION}}
+ver_num=$(cat dbversion)
+let ver_num++
+echo "ver_num=$ver_num" > dbversion
+if [ ! -e docs/javascripts/config.js ]; then
+    if [ -n "${INFERENCE_RESULTS_VERSION}" ]; then
+         echo "const results_version=\"${INFERENCE_RESULTS_VERSION}\";" > docs/javascripts/config.js;
+         echo "var repo_owner=\"${repo_owner}\";" >> docs/javascripts/config.js;
+         echo "var repo_branch=\"${repo_branch}\";" >> docs/javascripts/config.js;
+         echo "var repo_name=\"${repo_name}\";" >> docs/javascripts/config.js;
+         #ver_num=`echo ${INFERENCE_RESULTS_VERSION} | tr -cd '0-9'`
+         echo "const dbVersion =\"${ver_num}\";" >> docs/javascripts/config.js;
+    else
+       echo "Please export INFERENCE_RESULTS_VERSION=v4.1 or the corresponding version";
+       exit 1
+    fi
+fi
+
+if [ ! -e docs/thirdparty/tablesorter ]; then
+    cd docs/thirdparty && git clone https://github.com/Mottie/tablesorter.git && cd -
+    test $? -eq 0 || exit $?
+fi
+
+python3 process.py
+test $? -eq 0 || exit $?
+python3 process_results_table.py
+test $? -eq 0 || exit $?

mkdocs.yml

@@ -0,0 +1,42 @@
+site_name: MLPerf Inference Results Comparison
+repo_url: https://github.com/mlcommons/inference_results_v4.0
+theme:
+  name: material
+  logo: img/logo_v2.svg
+  favicon: img/logo_v2.svg
+  palette:
+    primary: deep purple
+    accent: yellow
+  custom_dir: overrides
+  features:
+    - content.tabs.link
+    - content.code.copy
+    - navigation.expand
+    - navigation.sections
+    - navigation.indexes
+    - navigation.instant
+    - navigation.tabs
+    - navigation.tabs.sticky
+    - navigation.top
+    - toc.follow
+nav:
+  - Results: index.md
+  - Compare: compare/index.md
+
+markdown_extensions:
+  - pymdownx.tasklist:
+      custom_checkbox: true
+  - pymdownx.details
+  - admonition
+  - attr_list
+  - def_list
+  - footnotes
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
+  - pymdownx.tabbed:
+      alternate_style: true
+plugins:
+  - macros