Documentation / Website for Leaderboard (#107)

* ruff: lint

* initial docusaurus

* Filled out commands

* update docs, update workflow

* update dependencies

* update dependencies

* Finished CUDA section

* minor typo

* add preview vid

.github/workflows/deploy-docs.yml

@@ -0,0 +1,69 @@
+# Workflow that builds and deploys the documentation website
+# Adapted from https://github.com/All-Hands-AI/OpenHands/blob/main/.github/workflows/deploy-docs.yml
+name: Deploy Docs to GitHub Pages
+
+# * Always run on "main"
+# * Run on PRs that target the "main" branch and have changes in the "docs" folder or this workflow
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - 'docs/**'
+      - '.github/workflows/deploy-docs.yml'
+    branches:
+      - main
+
+# If triggered by a PR, it will be in the same group. However, each commit on main will be in its own unique group
+concurrency:
+  group: ${{ github.workflow }}-${{ (github.head_ref && github.ref) || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Build the documentation website
+  build:
+    if: github.repository == 'gpu-mode/discord-cluster-manager'
+    name: Build Docusaurus
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+          cache: yarn
+          cache-dependency-path: docs/yarn.lock
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile --non-interactive
+      - name: Upload Build Artifact
+        if: github.ref == 'refs/heads/main'
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/build
+
+  # Deploy the documentation website
+  deploy:
+    if: github.ref == 'refs/heads/main' && github.repository == 'gpu-mode/discord-cluster-manager'
+    name: Deploy to GitHub Pages
+    runs-on: ubuntu-latest
+    # This job only runs on "main" so only run one of these jobs at a time
+    # otherwise it will fail if one is already running
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+    needs: build
+    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
+    permissions:
+      pages: write # to deploy to Pages
+      id-token: write # to verify the deployment originates from an appropriate source
+    # Deploy to the github-pages environment
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

docs/.gitignore

@@ -0,0 +1,20 @@
+# Dependencies
+/node_modules
+
+# Production
+/build
+
+# Generated files
+.docusaurus
+.cache-loader
+
+# Misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

docs/README.md

@@ -0,0 +1,41 @@
+# Website
+
+This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
+
+### Installation
+
+```
+$ yarn
+```
+
+### Local Development
+
+```
+$ yarn start
+```
+
+This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
+
+### Build
+
+```
+$ yarn build
+```
+
+This command generates static content into the `build` directory and can be served using any static contents hosting service.
+
+### Deployment
+
+Using SSH:
+
+```
+$ USE_SSH=true yarn deploy
+```
+
+Not using SSH:
+
+```
+$ GIT_USER=<Your GitHub username> yarn deploy
+```
+
+If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.

docs/blog/authors.yml

@@ -0,0 +1,23 @@
+yangshun:
+  name: Yangshun Tay
+  title: Front End Engineer @ Facebook
+  url: https://github.com/yangshun
+  image_url: https://github.com/yangshun.png
+  page: true
+  socials:
+    x: yangshunz
+    github: yangshun
+
+slorber:
+  name: Sébastien Lorber
+  title: Docusaurus maintainer
+  url: https://sebastienlorber.com
+  image_url: https://github.com/slorber.png
+  page:
+    # customize the url of the author page at /blog/authors/<permalink>
+    permalink: '/all-sebastien-lorber-articles'
+  socials:
+    x: sebastienlorber
+    linkedin: sebastienlorber
+    github: slorber
+    newsletter: https://thisweekinreact.com

docs/blog/tags.yml

@@ -0,0 +1,19 @@
+facebook:
+  label: Facebook
+  permalink: /facebook
+  description: Facebook tag description
+
+hello:
+  label: Hello
+  permalink: /hello
+  description: Hello tag description
+
+docusaurus:
+  label: Docusaurus
+  permalink: /docusaurus
+  description: Docusaurus tag description
+
+hola:
+  label: Hola
+  permalink: /hola
+  description: Hola tag description

docs/docs/available-discord-commands.md

@@ -0,0 +1,154 @@
+---
+sidebar_position: 4
+---
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Available Discord Commands
+We advise that you read this section carefully -- it is short, but will be extremely useful for you
+to remember. These `/` commands allow you to interface with the leaderboard, as well as extract any
+relevant information you may need. We allow participants to view all aspects of the evaluation
+pipeline, including leaderboard reference code and our evaluation scripts. We separate the commands
+into three categories, which you can open in the tabs below.
+
+
+<Tabs>
+  <TabItem value="submission" label="Leaderboard Submission Commands" default>
+        ## Submission Commands
+
+        The following commands allow participants to submit their kernel code to a specific GPU on a
+        specific leaderboard. We currently support two runners, **GitHub** and **Modal**, which each contain their
+        own set of available GPUs. After running a submission command, **a UI selection window will pop up asking the
+        participant to select which GPUs to submit to** (multiple can be selected at once).
+
+        ---
+        ### `/leaderboard submit modal {leaderboard_name} {script}`
+        **Description:** Submit leaderboard data on a Modal instance, which currently allows submissions on `NVIDIA T4`,
+        `NVIDIA L4`, `NVIDIA A100`, and `NVIDIA H100` GPUs.
+
+        **Arguments:**
+        - `leaderboard_name` *(required)*: Name of the leaderboard to submit to.
+        - `script` *(required)*: Script to be submitted. Note, a Python leaderboard expects a Python
+        submission file, and a CUDA leaderboard expects a CUDA submission file.
+
+        ---
+
+        ### `/leaderboard submit github {leaderboard_name} {script}`
+        **Description:** Submit leaderboard data on the GitHub runners, which currently allow submissions on `NVIDIA T4`
+        and `AMD` GPUs.
+
+        **Arguments:**
+        - `leaderboard_name` *(required)*: Name of the leaderboard to submit to.
+        - `script` *(required)*: Script to be submitted. Note, a Python leaderboard expects a Python
+        submission file, and a CUDA leaderboard expects a CUDA submission file.
+  </TabItem>
+  <TabItem value="tools" label="Useful Info Commands">
+
+        ## Useful Info Commands
+        These commands are particularly useful for leaderboard participants to query for information
+        about the leaderboards that they are submitting to. For example, listing the available
+        leaderboards to submit to, viewing the source code for the reference kernel, etc.
+
+        ---
+
+        ### `/leaderboard list`
+        **Description:** Lists all available leaderboards. Each leaderboard shows its name,
+        deadline, and all GPUs that can be submitted to on that leaderboard.
+
+        **Arguments:** None.
+
+        *Example output*:
+        ```rust title="/leaderboard list"
+        Name                    Deadline             GPU Types     
+        --------------------------------------------------------------------------------
+        softmax_py              2025-01-31 00:00     NVIDIA T4, AMD MI250
+        softmax_cuda            2024-12-31 00:00     NVIDIA T4
+        matmul_cuda             2025-12-12 00:00     NVIDIA T4, NVIDIA A100, NVIDIA H100
+        llama3-inference_py     2025-12-31 00:00     NVIDIA A100, NVIDIA H100
+        ```
+        For example, for the `llama3-inferece_py` leaderboard, users may only submit to `NVIDIA
+        A100` and `NVIDIA H100` GPUs.
+
+        ---
+
+        ### `/leaderboard show {leaderboard_name}`
+        **Description:** Display rankings for a particular leaderboard. After running this command,
+        a UI will pop up asking the user which GPUs for that particular leaderboard should be displayed (can select multiple).
+
+
+        **Arguments:** 
+        - `leaderboard_name` *(required)*: Name of the leaderboard to show information for.
+
+        ---
+
+        ### `/leaderboard show-personal {leaderboard_name}`
+        **Description:** Display rankings of **your own submissions** for a particular leaderboard. After running this command,
+        a UI will pop up asking the user which GPUs for that particular leaderboard should be displayed (can select multiple).
+
+        **Arguments:** 
+        - `leaderboard_name` *(required)*: Name of the leaderboard to show information for.
+
+        ---
+
+        ### `/leaderboard eval-code {language}`
+        **Description:** On the leaderboard side, we take the reference kernel and user submission
+        kernel and verify that both produce the same outputs and time the user submission. This
+        command allows participants to view this script for Python / CUDA leaderboards.
+
+        **Arguments:** 
+        - `language` *(required)*: `cuda` or `python` evaluation script to retrieve.
+
+        ---
+
+        ### `/leaderboard reference-code {leaderboard_name}`
+        **Description:** Each leaderboard has a set of functions, including the reference kernel,
+        that the leaderboard creator implemented. As a participant, you can view this code using
+        this command.
+
+        **Arguments:** 
+        - `leaderboard_name` *(required)*: Name of the leaderboard to retrieve reference code for.
+
+
+
+  </TabItem>
+  <TabItem value="creation" label="Leaderboard Creation Commands">
+
+        ## Leaderboard Creation Commands
+        The following commands allow leaderboard creators to specify algorithms / kernels that they
+        want participants to compete on. Only Discord users with the `Leaderboard Admin` or `Leaderboard
+        Creator` role can run these commands on a particular server such as GPU MODE. All reference
+        code must satisfy a particular format outlined in [Creating a Leaderboard](#).
+
+        After running a creation command, **a UI selection window will pop up asking the
+        creator to select which GPUs this leaderboard should run on** (multiple can be selected at once).
+
+        ---
+
+        ### `/leaderboard create`
+        **Description:** Creates a new leaderboard according to the defined reference code. Each
+        leaderboard uniquely defines the input data to evaluate on, a reference kernel to compare user
+        submissions to, and a comparison function that checks that two outputs match. Most standard
+        kernels will assume a list of Tensors, but we enable this flexibility for leaderboard creators to
+        define anything as the input and output types.
+
+        **Arguments:** 
+        - `leaderboard_name` *(required)*: name of the leaderboard to create.
+        - `deadline` *(required)*: name of the leaderboard to delete.
+        - `reference_code` *(required)*: The reference code that defines the leaderboard. This file
+        must obey a set of implemented function signatures according to [Creating a Leaderboard](#),
+        and its file extension (e.g. `.py` or `.cu`) determines whether the leaderboard submissions must be
+        in Python or CUDA/C++.
+
+        ---
+
+        ### `/leaderboard delete {leaderboard_name}`
+        **Description:** ⚠️  Delete an existing leaderboard and all submissions information to that
+        leaderboard. Use with extreme care -- a message will pop up asking the user to verify that they want
+        to do this.
+
+        **Arguments:** 
+        - `leaderboard_name` *(required)*: Name of the leaderboard to delete.
+  </TabItem>
+</Tabs>
+
+

docs/docs/creating-a-leaderboard/_category_.json

@@ -0,0 +1,11 @@
+{
+  "label": "Creating a Leaderboard",
+  "position": 3,
+  "link": {
+    "type": "generated-index",
+    "description": "Creating a leaderboard requires special permissions on servers like GPU MODE,
+    but you can also simply request for them to be made. This section is also useful for participants
+    to understand what the user submission pipeline looks like. The whole pipeline is designed to be extremely simple,
+    with the goal of quickly evaluating and profiling kernels that people want to see made faster."
+  }
+}

docs/docs/creating-a-leaderboard/cuda-creations.md

@@ -0,0 +1,6 @@
+---
+sidebar_position: 2
+---
+
+# Creating a CUDA Leaderboard
+WIP... Will be done soon!

docs/docs/creating-a-leaderboard/python-creations.md

@@ -0,0 +1,6 @@
+---
+sidebar_position: 1
+---
+
+# Creating a Python Leaderboard
+WIP... Will be done soon!