From 68a044bf7e1987882c1c4aab2ce96eb3f409479e Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:11:04 +0000
Subject: [PATCH 01/14] Keep updating v5 for catalogs

---
 .github/workflows/.metadata.yml               |  7 +++
 .github/workflows/update-aws-catalog.yml      | 43 +++++++++++--------
 .../workflows/update-fluidstack-catalog.yml   | 36 ++++++++++------
 .github/workflows/update-gcp-catalog.yml      | 37 ++++++++++------
 .github/workflows/update-lambda-catalog.yml   | 33 ++++++++------
 5 files changed, 99 insertions(+), 57 deletions(-)
 create mode 100644 .github/workflows/.metadata.yml

diff --git a/.github/workflows/.metadata.yml b/.github/workflows/.metadata.yml
new file mode 100644
index 00000000..5a76e10c
--- /dev/null
+++ b/.github/workflows/.metadata.yml
@@ -0,0 +1,7 @@
+# Mapping between version and SkyPilot commit. The CI will update the catalogs
+# for all versions listed here. We keep older versions for backward
+# compatibility.
+# TODO(zhwu): Remove v5 support after 0.10.0 release.
+version_commit:
+  v5: 1ed40e3174646ba835423b9308d8d6489f83b6bc
+  v6: latest
diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 3bee7c9d..de12bd4e 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -24,36 +24,45 @@ jobs:
           fetch-depth: 0
           path: catalogs
           token: ${{ secrets.GH_ACTION_PAT }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: '3.10'
-          cache: 'pip' # caching pip dependencies
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          cd sky
-          pip install ".[aws]"
-
+          version: "latest"
+          python-version: "3.10"
       - name: Run fetch_aws
+        id: fetch_catalogs
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
-          mkdir -p catalogs/catalogs/$version
-          cd catalogs/catalogs/$version
-          # Uses --check-all-regions-enabled-for-account to ensure that the catalog fetched has all regions, otherwise fail the job.
-          python -m sky.clouds.service_catalog.data_fetchers.fetch_aws --no-az-mappings --check-all-regions-enabled-for-account
+          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          # Loop over all versions and commit hashes
+          for version in $versions; do
+            # Find commit hash for the version
+            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            cd sky
+            git checkout $commit_hash
+            uv pip install ".[aws]"
+            cd -
+            echo "Fetching AWS catalog for version $version, SkyPilot commit hash $commit_hash"
+            mkdir -p catalogs/catalogs/$version
+            cd catalogs/catalogs/$version
+            # Uses --check-all-regions-enabled-for-account to ensure that the catalog fetched has all regions, otherwise fail the job.
+            python -m sky.clouds.service_catalog.data_fetchers.fetch_aws --no-az-mappings --check-all-regions-enabled-for-account
+          done
+          echo "versions=$versions" >> $GITHUB_OUTPUT
+          echo "commit_hashes=$commit_hashes" >> $GITHUB_OUTPUT
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Commit catalog
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update AWS catalog $version (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update AWS catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
+        env:
+          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}
+      
\ No newline at end of file
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 2d22afb1..4c8ecd6a 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -23,35 +23,43 @@ jobs:
           fetch-depth: 0
           path: catalogs
           token: ${{ secrets.GH_ACTION_PAT }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v4
         with:
+          version: "latest"
           python-version: "3.10"
-          cache: "pip" # caching pip dependencies
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           cd sky
-          pip install .
-
+          uv pip install ".[fluidstack]"
       - name: Run fetch_fluidstack
+        id: fetch_catalogs
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
-          mkdir -p catalogs/catalogs/$version
-          cd catalogs/catalogs/$version
-          mkdir -p ~/.fluidstack
-          echo "$FLUIDSTACK_API_KEY" > ~/.fluidstack/api_key
-          python -m  sky.clouds.service_catalog.data_fetchers.fetch_fluidstack
+          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          for version in $versions; do
+            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            cd sky
+            git checkout $commit_hash
+            uv pip install ".[fluidstack]"
+            cd -
+            mkdir -p catalogs/catalogs/$version
+            cd catalogs/catalogs/$version
+            mkdir -p ~/.fluidstack
+            echo "$FLUIDSTACK_API_KEY" > ~/.fluidstack/api_key
+            python -m  sky.clouds.service_catalog.data_fetchers.fetch_fluidstack
+          done
+          echo "versions=$versions" >> $GITHUB_OUTPUT
         env:
           FLUIDSTACK_API_KEY: ${{ secrets.FLUIDSTACK_API_KEY }}
       - name: Commit catalog
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update FluidStack catalog $version (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update FluidStack catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
+        env:
+          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 9dc6316e..4835bf5a 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -24,16 +24,15 @@ jobs:
           fetch-depth: 0
           path: catalogs
           token: ${{ secrets.GH_ACTION_PAT }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: '3.10'
-          cache: 'pip' # caching pip dependencies
+          version: "latest"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           cd sky
-          pip install ".[gcp]" &
+          uv pip install ".[gcp]"
           cd ~
           curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-436.0.0-linux-x86_64.tar.gz
           tar -xf google-cloud-cli-436.0.0-linux-x86_64.tar.gz
@@ -41,6 +40,7 @@ jobs:
           wait
 
       - name: Run fetch_gcp
+        id: fetch_catalogs
         run: |
           echo "> Sourcing gcloud path ..."
           source ~/google-cloud-sdk/path.bash.inc
@@ -53,24 +53,33 @@ jobs:
           echo "> Configuring project ..."
           gcloud config set project $GCP_PROJECT_ID > /dev/null
           echo "> Fetching catalog schema version ..."
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
-          echo "> Creating catalog directory ..."
-          mkdir -p catalogs/catalogs/$version
-          cd catalogs/catalogs/$version
-          echo "> Fetching GCP catalog ..."
-          python -u -m sky.clouds.service_catalog.data_fetchers.fetch_gcp --all-regions --single-threaded
+          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          for version in $versions; do
+            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            cd sky
+            git checkout $commit_hash
+            uv pip install ".[gcp]"
+            cd -
+            echo "> Creating catalog directory ..."
+            mkdir -p catalogs/catalogs/$version
+            cd catalogs/catalogs/$version
+            echo "> Fetching GCP catalog ..."
+            python -u -m sky.clouds.service_catalog.data_fetchers.fetch_gcp --all-regions --single-threaded
+          done
+          echo "versions=$versions" >> $GITHUB_OUTPUT
         env:
           GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
 
       - name: Commit catalog
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update GCP catalog $version (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update GCP catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
+        env:
+          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 23cb37c5..aca837b1 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -24,34 +24,43 @@ jobs:
           fetch-depth: 0
           path: catalogs
           token: ${{ secrets.GH_ACTION_PAT }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: '3.10'
-          cache: 'pip' # caching pip dependencies
+          version: "latest"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           cd sky
-          pip install ".[lambda]"
+          uv pip install ".[lambda]"
 
       - name: Run fetch_lambda
+        id: fetch_catalogs
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
-          mkdir -p catalogs/catalogs/$version
-          cd catalogs/catalogs/$version
-          python -m sky.clouds.service_catalog.data_fetchers.fetch_lambda_cloud --api-key ${LAMBDA_API_KEY}
+          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          for version in $versions; do
+            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            cd sky
+            git checkout $commit_hash
+            uv pip install ".[lambda]"
+            cd -
+            mkdir -p catalogs/catalogs/$version
+            cd catalogs/catalogs/$version
+            python -m sky.clouds.service_catalog.data_fetchers.fetch_lambda_cloud --api-key ${LAMBDA_API_KEY}
+          done
+          echo "versions=$versions" >> $GITHUB_OUTPUT
         env:
           LAMBDA_API_KEY: ${{ secrets.LAMBDA_API_KEY }}
 
       - name: Commit catalog
         run: |
-          version=$(python -c 'import sky; print(sky.clouds.service_catalog.constants.CATALOG_SCHEMA_VERSION)')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update Lambda catalog $version (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update Lambda catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
+        env:
+          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}

From 5d21c9dd9fbc2dc8ab52ed044bd05b0965522d32 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:14:50 +0000
Subject: [PATCH 02/14] fix latest commit

---
 .github/workflows/update-aws-catalog.yml         | 8 ++++++--
 .github/workflows/update-fluidstack-catalog.yml  | 7 +++++--
 .github/workflows/update-gcp-catalog.yml         | 7 +++++--
 .github/workflows/update-lambda-catalog.yml      | 7 +++++--
 .github/workflows/.metadata.yml => .metadata.yml | 0
 5 files changed, 21 insertions(+), 8 deletions(-)
 rename .github/workflows/.metadata.yml => .metadata.yml (100%)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index de12bd4e..ea306061 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -32,11 +32,15 @@ jobs:
       - name: Run fetch_aws
         id: fetch_catalogs
         run: |
-          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           # Loop over all versions and commit hashes
           for version in $versions; do
             # Find commit hash for the version
-            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            if [ "$commit_hash" == "latest" ]; then
+              # Find the latest commit of the remote SkyPilot repo
+              commit_hash=$(git rev-parse origin/master)
+            fi
             cd sky
             git checkout $commit_hash
             uv pip install ".[aws]"
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 4c8ecd6a..2174ff46 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -35,9 +35,12 @@ jobs:
       - name: Run fetch_fluidstack
         id: fetch_catalogs
         run: |
-          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            if [ "$commit_hash" == "latest" ]; then
+              commit_hash=$(git rev-parse origin/master)
+            fi
             cd sky
             git checkout $commit_hash
             uv pip install ".[fluidstack]"
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 4835bf5a..00030f78 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -53,9 +53,12 @@ jobs:
           echo "> Configuring project ..."
           gcloud config set project $GCP_PROJECT_ID > /dev/null
           echo "> Fetching catalog schema version ..."
-          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            if [ "$commit_hash" == "latest" ]; then
+              commit_hash=$(git rev-parse origin/master)
+            fi
             cd sky
             git checkout $commit_hash
             uv pip install ".[gcp]"
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index aca837b1..b99a3d59 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -37,9 +37,12 @@ jobs:
       - name: Run fetch_lambda
         id: fetch_catalogs
         run: |
-          versions=$(cat .metadata.yml | yq -r '.version_commit | keys[]')
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            commit_hash=$(cat .metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            if [ "$commit_hash" == "latest" ]; then
+              commit_hash=$(git rev-parse origin/master)
+            fi
             cd sky
             git checkout $commit_hash
             uv pip install ".[lambda]"
diff --git a/.github/workflows/.metadata.yml b/.metadata.yml
similarity index 100%
rename from .github/workflows/.metadata.yml
rename to .metadata.yml

From b14c85e10ba5abb1acf6d3b3e87f83cdd7ed011c Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:20:14 +0000
Subject: [PATCH 03/14] Add venv

---
 .github/workflows/update-aws-catalog.yml        | 2 ++
 .github/workflows/update-fluidstack-catalog.yml | 3 +++
 .github/workflows/update-gcp-catalog.yml        | 3 +++
 .github/workflows/update-lambda-catalog.yml     | 3 +++
 4 files changed, 11 insertions(+)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index ea306061..5994e9f2 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -32,6 +32,8 @@ jobs:
       - name: Run fetch_aws
         id: fetch_catalogs
         run: |
+          uv venv --seed ~/catalogs-venv
+          source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           # Loop over all versions and commit hashes
           for version in $versions; do
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 2174ff46..4bcdb8c3 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -30,11 +30,14 @@ jobs:
           python-version: "3.10"
       - name: Install dependencies
         run: |
+          uv venv --seed ~/catalogs-venv
+          source ~/catalogs-venv/bin/activate
           cd sky
           uv pip install ".[fluidstack]"
       - name: Run fetch_fluidstack
         id: fetch_catalogs
         run: |
+          source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
             commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 00030f78..51a9ef62 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -31,6 +31,8 @@ jobs:
           python-version: "3.10"
       - name: Install dependencies
         run: |
+          uv venv --seed ~/catalogs-venv
+          source ~/catalogs-venv/bin/activate
           cd sky
           uv pip install ".[gcp]"
           cd ~
@@ -42,6 +44,7 @@ jobs:
       - name: Run fetch_gcp
         id: fetch_catalogs
         run: |
+          source ~/catalogs-venv/bin/activate
           echo "> Sourcing gcloud path ..."
           source ~/google-cloud-sdk/path.bash.inc
           echo "> Writing service account key to file ..."
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index b99a3d59..290ee6e6 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -31,12 +31,15 @@ jobs:
           python-version: "3.10"
       - name: Install dependencies
         run: |
+          uv venv --seed ~/catalogs-venv
+          source ~/catalogs-venv/bin/activate
           cd sky
           uv pip install ".[lambda]"
 
       - name: Run fetch_lambda
         id: fetch_catalogs
         run: |
+          source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
             commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')

From 11c574b3ecfdc5a6cd7ca81278d63a27e7043960 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:28:57 +0000
Subject: [PATCH 04/14] test with ref

---
 .github/workflows/update-aws-catalog.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 5994e9f2..ca0537d3 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -24,6 +24,8 @@ jobs:
           fetch-depth: 0
           path: catalogs
           token: ${{ secrets.GH_ACTION_PAT }}
+          # Use the same commit hash as the workflow is triggered by
+          ref: ${{ github.event.workflow_run.head_sha }}
       - name: Install the latest version of uv
         uses: astral-sh/setup-uv@v4
         with:

From 46bd99ae78c8487b851713b6fda399587f8eeaeb Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:37:25 +0000
Subject: [PATCH 05/14] fix

---
 .github/workflows/update-aws-catalog.yml        | 1 +
 .github/workflows/update-fluidstack-catalog.yml | 1 +
 .github/workflows/update-gcp-catalog.yml        | 3 +++
 .github/workflows/update-lambda-catalog.yml     | 1 +
 4 files changed, 6 insertions(+)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index ca0537d3..8a6a0f76 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -54,6 +54,7 @@ jobs:
             cd catalogs/catalogs/$version
             # Uses --check-all-regions-enabled-for-account to ensure that the catalog fetched has all regions, otherwise fail the job.
             python -m sky.clouds.service_catalog.data_fetchers.fetch_aws --no-az-mappings --check-all-regions-enabled-for-account
+            cd -
           done
           echo "versions=$versions" >> $GITHUB_OUTPUT
           echo "commit_hashes=$commit_hashes" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 4bcdb8c3..2962ede5 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -53,6 +53,7 @@ jobs:
             mkdir -p ~/.fluidstack
             echo "$FLUIDSTACK_API_KEY" > ~/.fluidstack/api_key
             python -m  sky.clouds.service_catalog.data_fetchers.fetch_fluidstack
+            cd -
           done
           echo "versions=$versions" >> $GITHUB_OUTPUT
         env:
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 51a9ef62..32b04c90 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -58,10 +58,12 @@ jobs:
           echo "> Fetching catalog schema version ..."
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
+            echo "> Fetching catalog schema version $version ..."
             commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi
+            echo "> Checking out SkyPilot to commit hash $commit_hash ..."
             cd sky
             git checkout $commit_hash
             uv pip install ".[gcp]"
@@ -71,6 +73,7 @@ jobs:
             cd catalogs/catalogs/$version
             echo "> Fetching GCP catalog ..."
             python -u -m sky.clouds.service_catalog.data_fetchers.fetch_gcp --all-regions --single-threaded
+            cd -
           done
           echo "versions=$versions" >> $GITHUB_OUTPUT
         env:
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 290ee6e6..0c6b0400 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -53,6 +53,7 @@ jobs:
             mkdir -p catalogs/catalogs/$version
             cd catalogs/catalogs/$version
             python -m sky.clouds.service_catalog.data_fetchers.fetch_lambda_cloud --api-key ${LAMBDA_API_KEY}
+            cd -
           done
           echo "versions=$versions" >> $GITHUB_OUTPUT
         env:

From e09d9e0ed3052ebf3763d7e8471983bb2a59923f Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:45:09 +0000
Subject: [PATCH 06/14] authorize with github action for GCP

---
 .github/workflows/update-gcp-catalog.yml | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 32b04c90..d0e63b95 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -36,25 +36,14 @@ jobs:
           cd sky
           uv pip install ".[gcp]"
           cd ~
-          curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-436.0.0-linux-x86_64.tar.gz
-          tar -xf google-cloud-cli-436.0.0-linux-x86_64.tar.gz
-          ./google-cloud-sdk/install.sh -q
-          wait
-
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}'
       - name: Run fetch_gcp
         id: fetch_catalogs
         run: |
           source ~/catalogs-venv/bin/activate
-          echo "> Sourcing gcloud path ..."
-          source ~/google-cloud-sdk/path.bash.inc
-          echo "> Writing service account key to file ..."
-          echo "$GCP_SERVICE_ACCOUNT_KEY" > $HOME/service_account_key.json
-          echo "> Exporting service account key ..."
-          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/service_account_key.json
-          echo "> Activating service account ..."
-          gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS > /dev/null
-          echo "> Configuring project ..."
-          gcloud config set project $GCP_PROJECT_ID > /dev/null
           echo "> Fetching catalog schema version ..."
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do

From 31dd9f5f0f2c52950777b7fc8d0c07e2b85bf4e2 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 19:52:26 +0000
Subject: [PATCH 07/14] Add commit hash

---
 .github/workflows/update-gcp-catalog.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index d0e63b95..1bf04f84 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -44,11 +44,10 @@ jobs:
         id: fetch_catalogs
         run: |
           source ~/catalogs-venv/bin/activate
-          echo "> Fetching catalog schema version ..."
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            echo "> Fetching catalog schema version $version ..."
             commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            echo "> Fetching catalog schema version $version with SkyPilot commit hash $commit_hash ..."
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi

From fdc72dfa02df5163d355cb033f8f844ece0e7623 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 20:02:16 +0000
Subject: [PATCH 08/14] fix commit hash fetcher

---
 .github/workflows/update-aws-catalog.yml        | 2 +-
 .github/workflows/update-fluidstack-catalog.yml | 2 +-
 .github/workflows/update-gcp-catalog.yml        | 2 +-
 .github/workflows/update-lambda-catalog.yml     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 8a6a0f76..e7cc6bca 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -40,7 +40,7 @@ jobs:
           # Loop over all versions and commit hashes
           for version in $versions; do
             # Find commit hash for the version
-            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             if [ "$commit_hash" == "latest" ]; then
               # Find the latest commit of the remote SkyPilot repo
               commit_hash=$(git rev-parse origin/master)
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 2962ede5..b4cbe208 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -40,7 +40,7 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 1bf04f84..d327ae6e 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -46,7 +46,7 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             echo "> Fetching catalog schema version $version with SkyPilot commit hash $commit_hash ..."
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 0c6b0400..1bd788e9 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -42,7 +42,7 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            commit_hash=$(cat catalogs/.metadata.yml | yq -r '.version_commit | .[] | select(. == $version)')
+            commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi

From 9f6120d2e04aaef210d6f33a718a46e35a9c42f9 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 20:13:20 +0000
Subject: [PATCH 09/14] add history for sky

---
 .github/workflows/update-aws-catalog.yml        | 3 +--
 .github/workflows/update-fluidstack-catalog.yml | 1 +
 .github/workflows/update-gcp-catalog.yml        | 1 +
 .github/workflows/update-lambda-catalog.yml     | 1 +
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index e7cc6bca..609413d1 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -16,6 +16,7 @@ jobs:
       - name: Clone SkyPilot repo
         uses: actions/checkout@v4
         with:
+          fetch-depth: 0
           repository: skypilot-org/skypilot
           path: sky
       - name: Clone Catalog repo
@@ -24,8 +25,6 @@ jobs:
           fetch-depth: 0
           path: catalogs
           token: ${{ secrets.GH_ACTION_PAT }}
-          # Use the same commit hash as the workflow is triggered by
-          ref: ${{ github.event.workflow_run.head_sha }}
       - name: Install the latest version of uv
         uses: astral-sh/setup-uv@v4
         with:
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index b4cbe208..6fc06527 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -15,6 +15,7 @@ jobs:
       - name: Clone SkyPilot repo
         uses: actions/checkout@v4
         with:
+          fetch-depth: 0
           repository: skypilot-org/skypilot
           path: sky
       - name: Clone Catalog repo
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index d327ae6e..aa2ab8da 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -16,6 +16,7 @@ jobs:
       - name: Clone SkyPilot repo
         uses: actions/checkout@v4
         with:
+          fetch-depth: 0
           repository: skypilot-org/skypilot
           path: sky
       - name: Clone Catalog repo
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 1bd788e9..68962410 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -16,6 +16,7 @@ jobs:
       - name: Clone SkyPilot repo
         uses: actions/checkout@v4
         with:
+          fetch-depth: 0
           repository: skypilot-org/skypilot
           path: sky
       - name: Clone Catalog repo

From ed3bf1de3390f9003f5ab870c982189b2462452b Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 20:20:48 +0000
Subject: [PATCH 10/14] cd sky

---
 .github/workflows/update-aws-catalog.yml        | 2 +-
 .github/workflows/update-fluidstack-catalog.yml | 2 +-
 .github/workflows/update-gcp-catalog.yml        | 2 +-
 .github/workflows/update-lambda-catalog.yml     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 609413d1..9ae7ff47 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -39,12 +39,12 @@ jobs:
           # Loop over all versions and commit hashes
           for version in $versions; do
             # Find commit hash for the version
+            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             if [ "$commit_hash" == "latest" ]; then
               # Find the latest commit of the remote SkyPilot repo
               commit_hash=$(git rev-parse origin/master)
             fi
-            cd sky
             git checkout $commit_hash
             uv pip install ".[aws]"
             cd -
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 6fc06527..4fd89dc6 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -41,11 +41,11 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
+            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi
-            cd sky
             git checkout $commit_hash
             uv pip install ".[fluidstack]"
             cd -
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index aa2ab8da..0ceb2130 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -47,13 +47,13 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
+            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             echo "> Fetching catalog schema version $version with SkyPilot commit hash $commit_hash ..."
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi
             echo "> Checking out SkyPilot to commit hash $commit_hash ..."
-            cd sky
             git checkout $commit_hash
             uv pip install ".[gcp]"
             cd -
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 68962410..d58090ba 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -43,11 +43,11 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
+            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi
-            cd sky
             git checkout $commit_hash
             uv pip install ".[lambda]"
             cd -

From 0d29879ccb33335d7113465ad55061e9b1a353c4 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 20:24:48 +0000
Subject: [PATCH 11/14] move sky

---
 .github/workflows/update-aws-catalog.yml        | 2 +-
 .github/workflows/update-fluidstack-catalog.yml | 2 +-
 .github/workflows/update-gcp-catalog.yml        | 2 +-
 .github/workflows/update-lambda-catalog.yml     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 9ae7ff47..45c5ed76 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -39,8 +39,8 @@ jobs:
           # Loop over all versions and commit hashes
           for version in $versions; do
             # Find commit hash for the version
-            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
+            cd sky
             if [ "$commit_hash" == "latest" ]; then
               # Find the latest commit of the remote SkyPilot repo
               commit_hash=$(git rev-parse origin/master)
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 4fd89dc6..8d1f7bb8 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -41,8 +41,8 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
+            cd sky
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index 0ceb2130..e3e43147 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -47,8 +47,8 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
+            cd sky
             echo "> Fetching catalog schema version $version with SkyPilot commit hash $commit_hash ..."
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index d58090ba..6d8a6ab4 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -43,8 +43,8 @@ jobs:
           source ~/catalogs-venv/bin/activate
           versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           for version in $versions; do
-            cd sky
             commit_hash=$(cat catalogs/.metadata.yml | yq -r ".version_commit.$version")
+            cd sky
             if [ "$commit_hash" == "latest" ]; then
               commit_hash=$(git rev-parse origin/master)
             fi

From 97e6ce4ea8cd51ab63bf7c43d40a8093b23b47bf Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 21:07:01 +0000
Subject: [PATCH 12/14] fix

---
 .github/workflows/update-aws-catalog.yml        | 3 +--
 .github/workflows/update-fluidstack-catalog.yml | 2 +-
 .github/workflows/update-gcp-catalog.yml        | 2 +-
 .github/workflows/update-lambda-catalog.yml     | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 45c5ed76..41b3b76d 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -55,8 +55,7 @@ jobs:
             python -m sky.clouds.service_catalog.data_fetchers.fetch_aws --no-az-mappings --check-all-regions-enabled-for-account
             cd -
           done
-          echo "versions=$versions" >> $GITHUB_OUTPUT
-          echo "commit_hashes=$commit_hashes" >> $GITHUB_OUTPUT
+          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 8d1f7bb8..33869b23 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -56,7 +56,7 @@ jobs:
             python -m  sky.clouds.service_catalog.data_fetchers.fetch_fluidstack
             cd -
           done
-          echo "versions=$versions" >> $GITHUB_OUTPUT
+          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           FLUIDSTACK_API_KEY: ${{ secrets.FLUIDSTACK_API_KEY }}
       - name: Commit catalog
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index e3e43147..fa23b562 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -64,7 +64,7 @@ jobs:
             python -u -m sky.clouds.service_catalog.data_fetchers.fetch_gcp --all-regions --single-threaded
             cd -
           done
-          echo "versions=$versions" >> $GITHUB_OUTPUT
+          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 6d8a6ab4..9ce385c8 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -56,7 +56,7 @@ jobs:
             python -m sky.clouds.service_catalog.data_fetchers.fetch_lambda_cloud --api-key ${LAMBDA_API_KEY}
             cd -
           done
-          echo "versions=$versions" >> $GITHUB_OUTPUT
+          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           LAMBDA_API_KEY: ${{ secrets.LAMBDA_API_KEY }}
 

From 989a462f0f76fe1dce7d57c55ad2a2499527e8c8 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Fri, 13 Dec 2024 21:47:55 +0000
Subject: [PATCH 13/14] fix versions

---
 .github/workflows/update-aws-catalog.yml        | 8 +++-----
 .github/workflows/update-fluidstack-catalog.yml | 6 ++----
 .github/workflows/update-gcp-catalog.yml        | 6 ++----
 .github/workflows/update-lambda-catalog.yml     | 6 ++----
 4 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/update-aws-catalog.yml b/.github/workflows/update-aws-catalog.yml
index 41b3b76d..07badb11 100644
--- a/.github/workflows/update-aws-catalog.yml
+++ b/.github/workflows/update-aws-catalog.yml
@@ -55,21 +55,19 @@ jobs:
             python -m sky.clouds.service_catalog.data_fetchers.fetch_aws --no-az-mappings --check-all-regions-enabled-for-account
             cd -
           done
-          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Commit catalog
         run: |
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update AWS catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update AWS catalog $versions (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
-        env:
-          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}
-      
\ No newline at end of file
+      
diff --git a/.github/workflows/update-fluidstack-catalog.yml b/.github/workflows/update-fluidstack-catalog.yml
index 33869b23..58c0f95f 100644
--- a/.github/workflows/update-fluidstack-catalog.yml
+++ b/.github/workflows/update-fluidstack-catalog.yml
@@ -56,18 +56,16 @@ jobs:
             python -m  sky.clouds.service_catalog.data_fetchers.fetch_fluidstack
             cd -
           done
-          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           FLUIDSTACK_API_KEY: ${{ secrets.FLUIDSTACK_API_KEY }}
       - name: Commit catalog
         run: |
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update FluidStack catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update FluidStack catalog $versions (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
-        env:
-          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}
diff --git a/.github/workflows/update-gcp-catalog.yml b/.github/workflows/update-gcp-catalog.yml
index fa23b562..5b4caa28 100644
--- a/.github/workflows/update-gcp-catalog.yml
+++ b/.github/workflows/update-gcp-catalog.yml
@@ -64,20 +64,18 @@ jobs:
             python -u -m sky.clouds.service_catalog.data_fetchers.fetch_gcp --all-regions --single-threaded
             cd -
           done
-          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
 
       - name: Commit catalog
         run: |
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update GCP catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update GCP catalog $versions (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
-        env:
-          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}
diff --git a/.github/workflows/update-lambda-catalog.yml b/.github/workflows/update-lambda-catalog.yml
index 9ce385c8..d036ad89 100644
--- a/.github/workflows/update-lambda-catalog.yml
+++ b/.github/workflows/update-lambda-catalog.yml
@@ -56,19 +56,17 @@ jobs:
             python -m sky.clouds.service_catalog.data_fetchers.fetch_lambda_cloud --api-key ${LAMBDA_API_KEY}
             cd -
           done
-          echo "versions=\"$versions\"" >> $GITHUB_OUTPUT
         env:
           LAMBDA_API_KEY: ${{ secrets.LAMBDA_API_KEY }}
 
       - name: Commit catalog
         run: |
+          versions=$(cat catalogs/.metadata.yml | yq -r '.version_commit | keys[]')
           cd catalogs
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
           git add .
-          git commit -m"[Bot] Update Lambda catalog $VERSIONS (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
+          git commit -m"[Bot] Update Lambda catalog $versions (scheduled at $(date))" || { echo "No changes to commit" && exit 0; }
           git fetch origin
           git rebase origin/master
           git push
-        env:
-          VERSIONS: ${{ steps.fetch_catalogs.outputs.versions }}

From 84eb1a29a07d73b907ed9135c4ea07df2ca5bd85 Mon Sep 17 00:00:00 2001
From: Zhanghao Wu <zhanghao.wu@outlook.com>
Date: Tue, 17 Dec 2024 16:03:49 -0800
Subject: [PATCH 14/14] Address comment

---
 .metadata.yml |  2 ++
 README.md     | 12 ++++++------
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.metadata.yml b/.metadata.yml
index 5a76e10c..0e0c996b 100644
--- a/.metadata.yml
+++ b/.metadata.yml
@@ -3,5 +3,7 @@
 # compatibility.
 # TODO(zhwu): Remove v5 support after 0.10.0 release.
 version_commit:
+  # This commit should be fixed, as any newer commit will be fetching v6
+  # catalogs
   v5: 1ed40e3174646ba835423b9308d8d6489f83b6bc
   v6: latest
diff --git a/README.md b/README.md
index 254f1d3f..7daa2d4b 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,10 @@
 # SkyPilot Catalogs
 
-**Latest catalog schema version**: v5
+**Latest catalog schema version**: v6
 
-**Supported catalog schema versions**: v1, v2, v3, v4, v5
+**Supported catalog schema versions**: v1, v2, v3, v4, v5, v6
+
+**The versions that are still being updated periodically**: v5, v6
 
 ## Automatic Catalog Fetching
 
@@ -12,11 +14,9 @@ Catalogs are updated **every 7 hours**.
 
 
 
+## Schema V6
 
-
-## Schema V5
-
-The catalogs for each cloud in [v5](v5) include the following files:
+The catalogs for each cloud in [v6](v6) include the following files:
 1. `vms.csv`: the catalog for the VMs, including the instance and the accelerators.
 2. `images.csv`: the catalog for the images, which contains the mapping from the SkyPilot image tag to the image ID that can be used to find the image in the clouds.