January 2025 rebuild using rattler-build (#501)

* Initial commit to move to rattler * Further fixes * Complete local osx-arm64 rebuild * Near complete linux-aarch64 local rebuild * Fix teb-local-planner * Fix remaining build issues on linux-aarch64 * Do not use env -i * Obsolete patch * Fixes * Windows workaround for rattler-build * Create ros-noetic-topic-tools.patch * Build additional recipes in PR * Update ros-noetic-topic-tools.patch * Pull in vs2019 for OpenCV compat * Also set c_compiler * Create ros-noetic-xmlrpcpp.patch * Update ros-noetic-xmlrpcpp.patch * Update ros-noetic-xmlrpcpp.patch * Update ros-noetic-xmlrpcpp.patch * Update ros-noetic-xmlrpcpp.patch * Fixes * Update ros-noetic-cv-bridge.patch * Fix * Unvendor gtsam * Add gtsam in PRs * Try fix rviz issues * fix * Fix rtabmap-conversions patch * Add verbose options for python-qt-binding * Debug messages * Fixes * need sip 6.8 * Fixes * Fix * Fixes * Test * Try fix sip issues * Try fix sip issues * Fix * Fixes * Fix * Fixes * Fix workflow * Fix workflow * Fixes linux * Fixup ninja: error: '%PREFIX%/Library/lib/lz4.lib', needed by 'devel/bin/pcl_ros_tf.dll', missing and no known rule to make it * Full successful local build on linux-64 * Undo non-working lz4 patch * Loccal linux-64 rebuild of all packages except jsk stuff * Full local linux-64 rebuild * Full local linux-64 rebuild * time::min not available on Win because of our patch * Updates for Windows compatibility * Fix plotjuggler line endings / tabs or whatever it is ;) --------- Co-authored-by: Silvio Traversaro <[email protected]>
RoboStack · Jan 10, 2025 · 33d3128 · 33d3128
1 parent 142310f
commit 33d3128
Show file tree

Hide file tree

Showing 72 changed files with 6,045 additions and 1,044 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true
diff --git a/.github/testpr_environment.yml b/.github/testpr_environment.yml
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,7 +1,12 @@
+name: Generate CI pipelines for ROS packages
+
 permissions:
   actions: write
   contents: write
 
+env:
+  ROS_VERSION: 1
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -15,7 +20,7 @@ jobs:
         python-version: '3.11' # Version range or exact version of a Python version to use, using SemVer's version range syntax
     - name: Install vinca
       run: |
-        pip install git+https://github.com/RoboStack/vinca.git
+        pip install git+https://github.com/RoboStack/vinca.git@rattler-build-humble
 
     - name: Generate recipes for linux-64
       run: |

diff --git a/.github/workflows/testpr.yml b/.github/workflows/testpr.yml
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,7 @@ recipes/
 *.ps1
 *.json
 .DS_Store
+# pixi environments
+.pixi
+*.egg-info
+output/
diff --git a/.scripts/build_linux.sh b/.scripts/build_linux.sh
@@ -9,71 +9,25 @@ set -xeuo pipefail
 export PYTHONUNBUFFERED=1
 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}"
 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}"
-# export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support"
-# export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml"
 
-cat >~/.condarc <<CONDARC
-conda-build:
-    root-dir: /opt/conda/build_artifacts
-CONDARC
+curl -fsSL https://pixi.sh/install.sh | bash
+export PATH="$HOME/.pixi/bin:$PATH"
 
-# This doesn't work, see e.g. https://dev.azure.com/robostack/ros_pipelines/_build/results?buildId=188&view=logs&j=4e20d398-0572-5e54-89c7-6bdb9c00a59a&t=f5885ff8-badf-54b3-1543-35164851bdf3
-# if grep -q libgl "recipes/${CURRENT_BUILD_PKG_NAME}.yaml"; then
-# 	sudo yum install -y install mesa-libGL-devel
-# fi
-
-export "CONDA_BLD_PATH=/opt/conda/build_artifacts"
-
-mkdir -p $CONDA_BLD_PATH
-conda index $CONDA_BLD_PATH
-
-conda config --set remote_max_retries 5
-conda config --add channels conda-forge
-conda config --add channels robostack-staging
-conda config --add channels $CONDA_BLD_PATH
-conda config --remove channels defaults
-# conda config --set channel_priority strict
-
-mamba update conda --yes --quiet -c conda-forge
-mamba install --yes --quiet pip conda-build anaconda-client mamba boa
-
-# setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
-# export PATH="$HOME/miniconda/bin:$PATH"
-conda config --set anaconda_upload yes
-conda config --set show_channel_urls true
-conda config --set auto_update_conda false
-conda config --set add_pip_as_python_dependency false
-
-conda info
-conda config --show-sources
-conda list --show-channel-urls
+export CONDA_BLD_PATH="/opt/conda/build_artifacts"
 
 pwd
 
-for recipe in ${CURRENT_RECIPES[@]}; do
-	cd ${FEEDSTOCK_ROOT}/recipes/${recipe}
-	if [[ ${recipe} == *"gazebo-ros" || ${recipe} == *"gazebo-ros-control" ]]; then
-		boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config_old_linux_compiler.yaml
-	else
-		boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml
-	fi
-done
-
-anaconda -t ${ANACONDA_API_TOKEN} upload /opt/conda/build_artifacts/linux-*/*.tar.bz2 --force
-# quetz-client "${QUETZ_URL}" /opt/conda/build_artifacts --force
-
-# set up the condarc
+cd ${FEEDSTOCK_ROOT}
 
-# source run_conda_forge_build_setup
-
-# # make the build number clobber
-# make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
+for recipe in ${CURRENT_RECIPES[@]}; do
+	pixi run -v rattler-build build \
+		--recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \
+		-m ${FEEDSTOCK_ROOT}/conda_build_config.yaml \
+		-c robostack-jazzy -c conda-forge \
+		--output-dir $CONDA_BLD_PATH
 
-# conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \
-#     --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml"
+		# -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml \
 
-# if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then
-#     upload_package  "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
-# fi
+done
 
-# touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}"
+pixi run upload ${CONDA_BLD_PATH}/linux-*/*.conda --force
diff --git a/.scripts/build_osx.sh b/.scripts/build_osx.sh
@@ -1,35 +1,25 @@
 #!/bin/bash
 
 set -x
+set -e
 
 export FEEDSTOCK_ROOT=`pwd`
+export "CONDA_BLD_PATH=$HOME/conda-bld/"
 
-"${SHELL}" <(curl -L micro.mamba.pm/install.sh)
-source ~/.bash_profile
-
-micromamba config set remote_max_retries 5
+curl -fsSL https://pixi.sh/install.sh | bash
+export PATH="$HOME/.pixi/bin:$PATH"
 
-echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build."
-micromamba create -n devenv --quiet --yes conda-forge-ci-setup=3 conda-build pip boa quetz-client -c conda-forge
-micromamba activate devenv
-
-set -e
-
-export "CONDA_BLD_PATH=$CONDA_PREFIX/conda-bld/"
-
-mkdir -p $CONDA_BLD_PATH
-# micromamba index $CONDA_BLD_PATH
-micromamba config append channels conda-forge --env
-micromamba config append channels robostack-staging --env
-micromamba config append channels $CONDA_BLD_PATH --env
+# Remove homebrew from $PATH
+export PATH=$(echo $PATH | tr ":" "\n" | grep -v 'homebrew' | xargs | tr ' ' ':')
 
 for recipe in ${CURRENT_RECIPES[@]}; do
-	cd ${FEEDSTOCK_ROOT}/recipes/${recipe}
-	if [[ ${recipe} == *"rviz" || ${recipe} == *"moveit-setup-assistant" || ${recipe} == *"turtlesim" ]]; then
-		boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config_old_osx.yaml
-	else
-		boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml
-	fi
+	pixi run -v rattler-build build \
+		--recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \
+		-m ${FEEDSTOCK_ROOT}/conda_build_config.yaml \
+		-c robostack-jazzy -c conda-forge \
+		--output-dir $CONDA_BLD_PATH
+
+	# -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml \
 done
 
-anaconda -t ${ANACONDA_API_TOKEN} upload ${CONDA_BLD_PATH}/osx-64/*.tar.bz2 --force
+pixi run upload ${CONDA_BLD_PATH}/osx-*/*.conda --force
diff --git a/.scripts/build_osx_arm64.sh b/.scripts/build_osx_arm64.sh
@@ -1,31 +1,25 @@
 #!/bin/bash
 
 set -x
-
-export FEEDSTOCK_ROOT=`pwd`
-
-"${SHELL}" <(curl -L micro.mamba.pm/install.sh)
-source ~/.bash_profile
-
-micromamba config set remote_max_retries 5
-
-echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build."
-micromamba create -n devenv --quiet --yes conda-forge-ci-setup=3 conda-build pip boa quetz-client -c conda-forge
-micromamba activate devenv
-
 set -e
 
-export "CONDA_BLD_PATH=$CONDA_PREFIX/conda-bld/"
+export FEEDSTOCK_ROOT=`pwd`
+export "CONDA_BLD_PATH=$HOME/conda-bld/"
 
-mkdir -p $CONDA_BLD_PATH
+curl -fsSL https://pixi.sh/install.sh | bash
+export PATH="$HOME/.pixi/bin:$PATH"
 
-micromamba config append channels conda-forge --env
-micromamba config append channels robostack-staging --env
-micromamba config append channels $CONDA_BLD_PATH --env
+# Remove homebrew from $PATH
+export PATH=$(echo $PATH | tr ":" "\n" | grep -v 'homebrew' | xargs | tr ' ' ':')
 
 for recipe in ${CURRENT_RECIPES[@]}; do
-	cd ${FEEDSTOCK_ROOT}/recipes/${recipe}
-	boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml
+	pixi run -v rattler-build build \
+		--recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \
+		-m ${FEEDSTOCK_ROOT}/conda_build_config.yaml \
+		-c robostack-jazzy -c conda-forge \
+		--output-dir $CONDA_BLD_PATH
+
+	# -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml \
 done
 
-anaconda -t ${ANACONDA_API_TOKEN} upload ${CONDA_BLD_PATH}/osx-arm64/*.tar.bz2 --force
+pixi run upload ${CONDA_BLD_PATH}/osx-*/*.conda --force
diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat
@@ -1,35 +1,29 @@
 setlocal EnableExtensions EnableDelayedExpansion
-call %CONDA%\condabin\conda_hook.bat
-call %CONDA%\condabin\conda.bat activate base
 
+set CONDA_BLD_PATH=C:\bld
 echo "PATH is %PATH%"
-echo "CONDA_BLD_PATH is %CONDA_BLD_PATH%"
 
 rmdir /Q/S C:\Strawberry\
 rmdir /Q/S "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\"
 
 set "FEEDSTOCK_ROOT=%cd%"
 
 mkdir %CONDA_BLD_PATH%
-call conda index %CONDA_BLD_PATH%
-
-rem call conda config --remove channels defaults
-call conda config --add channels conda-forge
-call conda config --add channels robostack-staging
-call conda config --add channels %CONDA_BLD_PATH%
-:: call conda config --set channel_priority strict
 
 :: Enable long path names on Windows
 reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f
 
-:: conda remove --force m2-git
-
 for %%X in (%CURRENT_RECIPES%) do (
     echo "BUILDING RECIPE %%X"
     cd %FEEDSTOCK_ROOT%\recipes\%%X\
-    boa build . -m %FEEDSTOCK_ROOT%\.ci_support\conda_forge_pinnings.yaml -m %FEEDSTOCK_ROOT%\conda_build_config.yaml
+    pixi run -v rattler-build build --recipe %FEEDSTOCK_ROOT%\recipes\%%X\ ^
+        -m %FEEDSTOCK_ROOT%\conda_build_config.yaml ^
+        -c robostack-jazzy -c conda-forge ^
+        --output-dir %CONDA_BLD_PATH%
+
     if errorlevel 1 exit 1
+    rem -m %FEEDSTOCK_ROOT%\.ci_support\conda_forge_pinnings.yaml
 )
 
-anaconda -t %ANACONDA_API_TOKEN% upload "C:\bld\win-64\*.tar.bz2" --force
+pixi run upload "%CONDA_BLD_PATH%\win-64\*.conda" --force
 if errorlevel 1 exit 1
diff --git a/additional_recipes/build_catkin.sh b/additional_recipes/build_catkin.sh
@@ -1,6 +1,8 @@
-# Generated by vinca http://github.com/seanyen/vinca.
+# Generated by vinca http://github.com/RoboStack/vinca.
 # DO NOT EDIT!
 
+set -eo pipefail
+
 CATKIN_BUILD_BINARY_PACKAGE="ON"
 
 if [ "${PKG_NAME}" == "ros-noetic-catkin" ]; then
@@ -20,19 +22,38 @@ export LINK=$CXX
 if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then
   PYTHON_EXECUTABLE=$PREFIX/bin/python
   PKG_CONFIG_EXECUTABLE=$PREFIX/bin/pkg-config
+  OSX_DEPLOYMENT_TARGET="10.15"
 else
   PYTHON_EXECUTABLE=$BUILD_PREFIX/bin/python
   PKG_CONFIG_EXECUTABLE=$BUILD_PREFIX/bin/pkg-config
+  OSX_DEPLOYMENT_TARGET="11.0"
+fi
+
+if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then
+  export QT_HOST_PATH="$BUILD_PREFIX"
+else
+  export QT_HOST_PATH="$PREFIX"
 fi
+
 echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
 echo "USING PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_EXECUTABLE}"
 
 export ROS_PYTHON_VERSION=`$PYTHON_EXECUTABLE -c "import sys; print('%i.%i' % (sys.version_info[0:2]))"`
 echo "Using Python $ROS_PYTHON_VERSION"
+# Fix up SP_DIR which for some reason might contain a path to a wrong Python version
+FIXED_SP_DIR=$(echo $SP_DIR | sed -E "s/python[0-9]+\.[0-9]+/python$ROS_PYTHON_VERSION/")
+echo "Using site-package dir ${FIXED_SP_DIR}"
 
 # see https://github.com/conda-forge/cross-python-feedstock/issues/24
 if [[ "$CONDA_BUILD_CROSS_COMPILATION" == "1" ]]; then
-  find $PREFIX/lib/cmake -type f -exec sed -i "s~\${_IMPORT_PREFIX}/lib/python$ROS_PYTHON_VERSION/site-packages~$BUILD_PREFIX/lib/python$ROS_PYTHON_VERSION/site-packages~g" {} +
+  find $PREFIX/lib/cmake -type f -exec sed -i "s~\${_IMPORT_PREFIX}/lib/python$ROS_PYTHON_VERSION/site-packages~$BUILD_PREFIX/lib/python$ROS_PYTHON_VERSION/site-packages~g" {} + || true
+  find $PREFIX/share/rosidl* -type f -exec sed -i "s~$PREFIX/lib/python$ROS_PYTHON_VERSION/site-packages~$BUILD_PREFIX/lib/python$ROS_PYTHON_VERSION/site-packages~g" {} + || true
+  find $PREFIX/share/rosidl* -type f -exec sed -i "s~\${_IMPORT_PREFIX}/lib/python$ROS_PYTHON_VERSION/site-packages~$BUILD_PREFIX/lib/python$ROS_PYTHON_VERSION/site-packages~g" {} + || true
+  find $PREFIX/lib/cmake -type f -exec sed -i "s~message(FATAL_ERROR \"The imported target~message(WARNING \"The imported target~g" {} + || true
+  # way around bad CPU type issues
+  # rm $PREFIX/bin/doxygen || echo "doxygen not found"
+  # rm $PREFIX/bin/dia || echo "dia not found"
+  # rm $PREFIX/bin/dot || echo "dot not found"
 fi
 
 # NOTE: there might be undefined references occurring
@@ -42,25 +63,34 @@ fi
 export CXXFLAGS="$CXXFLAGS -DBOOST_ERROR_CODE_HEADER_ONLY"
 
 if [[ $target_platform =~ linux.* ]]; then
+    export CFLAGS="$CFLAGS -D__STDC_FORMAT_MACROS=1";
     export CXXFLAGS="$CXXFLAGS -D__STDC_FORMAT_MACROS=1";
     # I am too scared to turn this on for now ...
     # export LDFLAGS="$LDFLAGS -lrt";
+    # Some qt stuff uses g++ directly - fix these use cases
+    ln -s $GXX $BUILD_PREFIX/bin/g++
 fi
 
 export SKIP_TESTING=ON
 
-cmake ${CMAKE_ARGS} .. -DCMAKE_INSTALL_PREFIX=$PREFIX \
+cmake ${CMAKE_ARGS} --compile-no-warning-as-error \
+         -DCMAKE_INSTALL_PREFIX=$PREFIX \
          -DCMAKE_PREFIX_PATH=$PREFIX \
          -DCMAKE_BUILD_TYPE=Release \
          -DCMAKE_INSTALL_LIBDIR=lib \
          -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON \
          -DCMAKE_FIND_FRAMEWORK=LAST \
          -DBUILD_SHARED_LIBS=ON \
          -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
+         -DPython_EXECUTABLE=$PYTHON_EXECUTABLE \
+         -DPython3_EXECUTABLE=$PYTHON_EXECUTABLE \
+         -DPython3_FIND_STRATEGY=LOCATION \
+         -DPYTHON_INSTALL_DIR=$FIXED_SP_DIR \
          -DPKG_CONFIG_EXECUTABLE=$PKG_CONFIG_EXECUTABLE \
          -DSETUPTOOLS_DEB_LAYOUT=OFF \
          -DCATKIN_SKIP_TESTING=$SKIP_TESTING \
          -DCATKIN_BUILD_BINARY_PACKAGE=$CATKIN_BUILD_BINARY_PACKAGE \
+         -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \
          -G "Ninja" \
          $SRC_DIR/$PKG_NAME/src/work
 
@@ -80,11 +110,22 @@ if [ "${PKG_NAME}" == "ros-noetic-catkin" ]; then
         mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d"
         cp "${RECIPE_DIR}/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh"
     done
+fi
+
+if [ "${PKG_NAME}" == "ros-noetic-environment" ]; then
+   for SCRIPT in "1.ros_distro.sh" "1.ros_etc_dir.sh" "1.ros_package_path.sh" "1.ros_python_version.sh" "1.ros_version.sh"
+   do
+       mkdir -p "${PREFIX}/etc/conda/activate.d"
+       cp "${PREFIX}/etc/catkin/profile.d/${SCRIPT}" "${PREFIX}/etc/conda/activate.d/${SCRIPT}"
+   done
+fi
 
-   if [ "${PKG_NAME}" == "ros-noetic-environment" ]; then
-       for SCRIPT in "1.ros_distro.sh" "1.ros_etc_dir.sh" "1.ros_package_path.sh" "1.ros_python_version.sh" "1.ros_version.sh"
-       do
-           cp "${PREFIX}/etc/catkin/profile.d/${SCRIPT}" "${PREFIX}/etc/conda/activate.d/${SCRIPT}"
-       done
-   fi
+if [ "${PKG_NAME}" == "ros-noetic-ros-workspace" ]; then
+    # Copy the [de]activate scripts to $PREFIX/etc/conda/[de]activate.d.
+    # This will allow them to be run on environment activation.
+    for CHANGE in "activate" "deactivate"
+    do
+        mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d"
+        cp "${RECIPE_DIR}/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh"
+    done
 fi
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# GitHub syntax highlighting
		pixi.lock linguist-language=YAML linguist-generated=true