From cabdc83d60a3b8c594496a8b49a4c9ae3b49b1b4 Mon Sep 17 00:00:00 2001 From: Michael Norris Date: Wed, 15 Jan 2025 17:47:00 -0800 Subject: [PATCH] Migration off defaults to conda-forge channel (#4126) Summary: Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch Explanation of changes: - - changed to miniforge from miniconda: this ensures we only pull in from conda-defaults when creating the environment - architecture change: ARM64 and aarch64 are the same thing. But there is no miniforge package for ARM64, so we need to make it check for aarch64 instead. - mkl 2023.1 change: pulling in 2023.2 causes problems and all current builds pull in 2023.1. So it is specified as 2023.1. The meta.yaml changes are the ones that narrow it to 2023.1 during `conda build faiss`. Differential Revision: D68043874 --- .github/actions/build_cmake/action.yml | 19 ++++++++++++++++--- .github/actions/build_conda/action.yml | 20 +++++++++++++++----- conda/faiss/meta.yaml | 9 ++++++--- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index fa20974af5..71fc95c17e 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -23,12 +23,19 @@ runs: uses: conda-incubator/setup-miniconda@v3 with: python-version: '3.11' - miniconda-version: latest + miniforge-version: latest # ensures conda-forge channel is used. + channels: conda-forge + conda-remove-defaults: 'true' + # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64. + # They are the same thing, just named differently. + architecture: ${{ runner.arch == 'ARM64' && 'aarch64' || runner.arch }} - name: Configure build environment shell: bash run: | # initialize Conda conda config --set solver libmamba + # Ensure starting packages are from conda-forge. + conda list --show-channel-urls conda update -y -q conda echo "$CONDA/bin" >> $GITHUB_PATH @@ -43,7 +50,7 @@ runs: if [ "${{ runner.arch }}" = "X64" ]; then # TODO: merge this with ARM64 conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17 - conda install -y -q mkl=2023 mkl-devel=2023 + conda install -y -q mkl=2023.1.0 mkl-devel=2023.1.0 fi # no CUDA needed for ROCm so skip this @@ -56,12 +63,13 @@ runs: elif [ "${{ inputs.cuvs }}" = "ON" ]; then conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge fi + # install test packages if [ "${{ inputs.rocm }}" = "ON" ]; then : # skip torch install via conda, we need to install via pip to get # ROCm-enabled version until it's supported in conda by PyTorch elif [ "${{ inputs.gpu }}" = "ON" ]; then - conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0" + conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.1" else conda install -y -q "pytorch<2.5" -c pytorch fi @@ -174,3 +182,8 @@ runs: with: name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }} path: test-results + - name: Check installed packages channel + shell: bash + run: | + # Shows that all installed packages are from conda-forge. + conda list --show-channel-urls diff --git a/.github/actions/build_conda/action.yml b/.github/actions/build_conda/action.yml index ff860007b2..bf7818ab70 100644 --- a/.github/actions/build_conda/action.yml +++ b/.github/actions/build_conda/action.yml @@ -26,20 +26,25 @@ runs: else echo "shell=pwsh" >> "$GITHUB_OUTPUT" fi + echo "${{ runner.arch }}" - name: Setup miniconda uses: conda-incubator/setup-miniconda@v3 with: python-version: '3.11' - miniconda-version: latest + miniforge-version: latest # ensures conda-forge channel is used. + channels: conda-forge + conda-remove-defaults: 'true' + # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64. + # They are the same thing, just named differently. + architecture: ${{ runner.arch == 'ARM64' && 'aarch64' || runner.arch }} - name: Install conda build tools shell: ${{ steps.choose_shell.outputs.shell }} run: | + # Ensure starting packages are from conda-forge. + conda list --show-channel-urls + conda install -y -q "conda!=24.11.0" conda install -y -q "conda-build!=24.11.0" - - name: Fix CI failure - shell: ${{ steps.choose_shell.outputs.shell }} - if: runner.os != 'Windows' - run: conda remove conda-anaconda-telemetry - name: Enable anaconda uploads if: inputs.label != '' shell: ${{ steps.choose_shell.outputs.shell }} @@ -94,3 +99,8 @@ runs: run: | conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia + - name: Check installed packages channel + shell: bash + run: | + # Shows that all installed packages are from conda-forge. + conda list --show-channel-urls diff --git a/conda/faiss/meta.yaml b/conda/faiss/meta.yaml index fe7612c23b..402a84f94a 100644 --- a/conda/faiss/meta.yaml +++ b/conda/faiss/meta.yaml @@ -41,12 +41,15 @@ outputs: - llvm-openmp # [osx] - cmake >=3.24.0 - make =4.2 # [not win] - - mkl-devel =2023 # [x86_64] + - mkl =2023.1.0 # [x86_64] + - _openmp_mutex =4.5=2_gnu # [x86_64 and not win] host: - - mkl =2023 # [x86_64] + - mkl =2023.1.0 # [x86_64] + - _openmp_mutex =4.5=2_gnu # [x86_64 and not win] - openblas =0.3 # [not x86_64] run: - - mkl =2023 # [x86_64] + - mkl =2023.1.0 # [x86_64] + - _openmp_mutex =4.5=2_gnu # [x86_64 and not win] - openblas =0.3 # [not x86_64] test: requires: