diff --git a/.github/workflows/build-ci.sh b/.github/workflows/build-ci.sh index 24e7a96..260d3ba 100755 --- a/.github/workflows/build-ci.sh +++ b/.github/workflows/build-ci.sh @@ -2,8 +2,28 @@ set -e +function status { + echo -e "\033[1m$1\033[0m" +} + +function info { + echo -e "\033[1;34m$1\033[0m" +} + +function warning { + echo -e "\033[1;33m$1\033[0m" +} + +function verbose_cmd { + if [ $verbose -eq 1 ]; then + "$@" + else + "$@" > /dev/null + fi +} + project_root="$( cd -- "$(dirname "$0")/../.." >/dev/null 2>&1 ; pwd -P )" -echo "Project root: $project_root" +status "Project root: $project_root" py_venv_path="$project_root/.venv" cinnamon_path="$project_root/cinnamon" @@ -11,6 +31,7 @@ llvm_path="$project_root/llvm" torch_mlir_path="$project_root/torch-mlir" upmem_path="$project_root/upmem" +verbose=0 reconfigure=0 setup_python_venv=1 @@ -27,72 +48,96 @@ enable_roc=0 ############################################################### if [ -n "$LLVM_BUILD_DIR" ]; then - checkout_and_build_llvm=0 + checkout_and_build_llvm=external TORCH_MLIR_CMAKE_OPTIONS="$TORCH_MLIR_CMAKE_OPTIONS -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm" CINNAMON_CMAKE_OPTIONS="$CINNAMON_CMAKE_OPTIONS -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm" - echo "Using environment variable LLVM_BUILD_DIR, project is reconfigured automatically, any warnings from this script regarding llvm config can be ignored" + info "Using environment variable LLVM_BUILD_DIR for configuration" + info "Dependent targets will use '$LLVM_BUILD_DIR'" + + if [ ! -d "$LLVM_BUILD_DIR" ]; then + warning "Directory '$LLVM_BUILD_DIR' does not exist" + fi fi if [ -n "$TORCH_MLIR_INSTALL_DIR" ]; then - checkout_and_build_torch_mlir=0 + checkout_and_build_torch_mlir=external CINNAMON_CMAKE_OPTIONS="$CINNAMON_CMAKE_OPTIONS -DTORCH_MLIR_DIR=$TORCH_MLIR_INSTALL_DIR" - echo "Using environment variable TORCH_MLIR_INSTALL_DIR, project is reconfigured automatically, any warnings from this script regarding torch-mlir config can be ignored" + info "Using environment variable TORCH_MLIR_INSTALL_DIR for configuration" + info "Dependent targets will use '$TORCH_MLIR_INSTALL_DIR'" + + if [ ! -d "$TORCH_MLIR_INSTALL_DIR" ]; then + warning "Directory '$TORCH_MLIR_INSTALL_DIR' does not exist" + fi fi if [ -n "$UPMEM_HOME" ]; then - checkout_upmem=0 + checkout_upmem=external CINNAMON_CMAKE_OPTIONS="$CINNAMON_CMAKE_OPTIONS -DUPMEM_DIR=$UPMEM_HOME" - echo "Using environment variable UPMEM_HOME, project is reconfigured automatically, any warnings from this script regarding upmem config can be ignored" + info "Using environment variable UPMEM_HOME for configuration" + info "Dependent targets will use '$UPMEM_HOME'" + + if [ ! -d "$UPMEM_HOME" ]; then + warning "Directory '$UPMEM_HOME' does not exist" + fi fi ############################################################### -if echo "$@" | grep -q "-reconfigure"; then +if echo "$@" | grep -q -- "-verbose"; then + verbose=1 +else + info "Some steps will be run in quiet mode, use -verbose to see all output" +fi + +if echo "$@" | grep -q -- "-reconfigure"; then reconfigure=1 fi -if echo "$@" | grep -q "-no-python-venv"; then +if echo "$@" | grep -q -- "-no-python-venv"; then setup_python_venv=0 fi -if echo "$@" | grep -q "-no-llvm"; then +if echo "$@" | grep -q -- "-no-llvm"; then checkout_and_build_llvm=0 fi -if echo "$@" | grep -q "-no-torch-mlir"; then +if echo "$@" | grep -q -- "-no-torch-mlir"; then checkout_and_build_torch_mlir=0 fi -if echo "$@" | grep -q "-no-upmem"; then +if echo "$@" | grep -q -- "-no-upmem"; then checkout_upmem=0 fi -if echo "$@" | grep -q "-no-cinnamon-wheel"; then +if echo "$@" | grep -q -- "-no-cinnamon-wheel"; then build_cinnamon_wheel=0 fi -if echo "$@" | grep -q "-enable-cuda"; then +if echo "$@" | grep -q -- "-enable-cuda"; then enable_cuda=1 fi -if echo "$@" | grep -q "-enable-roc"; then +if echo "$@" | grep -q -- "-enable-roc"; then enable_roc=1 fi if [[ $setup_python_venv -eq 1 ]]; then reconfigure_python_venv=0 if [ ! -d "$py_venv_path" ]; then - python3 -m venv "$py_venv_path" + status "Creating Python venv" + python3.12 -m venv "$py_venv_path" source "$py_venv_path/bin/activate" reconfigure_python_venv=1 else + status "Enabling Python venv" source "$py_venv_path/bin/activate" fi if [ $reconfigure -eq 1 ] || [ $reconfigure_python_venv -eq 1 ]; then + status "Installing Python dependencies" # https://pytorch.org/get-started/locally/ if [[ $enable_cuda -eq 1 ]]; then torch_source=https://download.pytorch.org/whl/cu124 @@ -102,24 +147,24 @@ if [[ $setup_python_venv -eq 1 ]]; then torch_source=https://download.pytorch.org/whl/cpu fi - pip install --upgrade pip - pip install torch torchvision torchaudio --index-url $torch_source - pip install pybind11 - pip install build + verbose_cmd pip install --upgrade pip + verbose_cmd pip install torch torchvision torchaudio --index-url $torch_source + verbose_cmd pip install pybind11 + verbose_cmd pip install build fi -else - echo "Skipping Python venv setup" - echo "Make sure to have a correct Python environment set up" +elif [[ $setup_python_venv -eq 0 ]]; then + warning "Skipping Python venv setup" + warning "Make sure to have a correct Python environment set up" fi if [[ $checkout_and_build_llvm -eq 1 ]]; then reconfigure_llvm=0 if [ ! -d "$llvm_path" ]; then - git clone https://github.com/llvm/llvm-project "$llvm_path" + status "Checking out LLVM" + git clone --branch llvmorg-19.1.3 --depth 1 https://github.com/llvm/llvm-project "$llvm_path" + status "Applying patches to LLVM" cd "$llvm_path" - git checkout llvmorg-19.1.3 - patch_dir="$project_root/patches/llvm" for patch in $(ls $patch_dir); do git apply $patch_dir/$patch @@ -131,7 +176,9 @@ if [[ $checkout_and_build_llvm -eq 1 ]]; then cd "$llvm_path" if [ $reconfigure -eq 1 ] || [ $reconfigure_llvm -eq 1 ]; then + status "Configuring LLVM" cmake -S llvm -B build \ + -Wno-dev \ -DLLVM_ENABLE_PROJECTS="mlir;llvm;clang" \ -DLLVM_TARGETS_TO_BUILD="host" \ -DLLVM_ENABLE_ASSERTIONS=ON \ @@ -145,20 +192,23 @@ if [[ $checkout_and_build_llvm -eq 1 ]]; then $LLVM_CMAKE_OPTIONS fi + status "Building LLVM" cmake --build build --target all llc opt export PATH=$llvm_path/build/bin:$PATH -else - echo "Skipping LLVM checkout and build" - echo "The following steps will need LLVM_DIR and MLIR_DIR to be set in their respective _CMAKE_OPTIONS" +elif [[ $checkout_and_build_llvm -eq 0 ]]; then + warning "Skipping LLVM checkout and build" + warning "The following steps will need LLVM_DIR and MLIR_DIR to be set in their respective _CMAKE_OPTIONS" fi if [[ $checkout_and_build_torch_mlir -eq 1 ]]; then reconfigure_torch_mlir=0 if [ ! -d "$torch_mlir_path" ]; then - git clone https://github.com/llvm/torch-mlir "$torch_mlir_path" + status "Checking out Torch-MLIR" + git clone --depth 1 https://github.com/llvm/torch-mlir "$torch_mlir_path" cd "$torch_mlir_path" + git fetch --depth 1 origin 98e08023bbf71e00ab81e980eac9f7c96f1f24b4 git checkout 98e08023bbf71e00ab81e980eac9f7c96f1f24b4 reconfigure_torch_mlir=1 @@ -167,8 +217,13 @@ if [[ $checkout_and_build_torch_mlir -eq 1 ]]; then cd "$torch_mlir_path" if [ $reconfigure -eq 1 ] || [ $reconfigure_torch_mlir -eq 1 ]; then + status "Configuring Torch-MLIR" dependency_paths="" + if [[ $setup_python_venv -eq 1 ]]; then + dependency_paths="$dependency_paths -DPython3_FIND_VIRTUALENV=ONLY" + fi + if [[ $checkout_and_build_llvm -eq 1 ]]; then dependency_paths="$dependency_paths -DLLVM_DIR=$llvm_path/build/lib/cmake/llvm" dependency_paths="$dependency_paths -DMLIR_DIR=$llvm_path/build/lib/cmake/mlir" @@ -176,44 +231,52 @@ if [[ $checkout_and_build_torch_mlir -eq 1 ]]; then cmake -S . -B build \ $dependency_paths \ + -Wno-dev \ -DCMAKE_BUILD_TYPE=Release \ -DTORCH_MLIR_OUT_OF_TREE_BUILD=ON \ -DTORCH_MLIR_ENABLE_STABLEHLO=OFF \ $TORCH_MLIR_CMAKE_OPTIONS fi + status "Building Torch-MLIR" cmake --build build --target all TorchMLIRPythonModules - cmake --install build --prefix install + verbose_cmd cmake --install build --prefix install if [[ $setup_python_venv -eq 1 ]]; then + status "Building and installing Torch-MLIR Python package" python_package_dir=build/tools/torch-mlir/python_packages/torch_mlir python_package_rel_build_dir=../../../python_packages/torch_mlir mkdir -p $(dirname $python_package_dir) ln -s "$python_package_rel_build_dir" "$python_package_dir" 2> /dev/null || true - TORCH_MLIR_CMAKE_ALREADY_BUILT=1 TORCH_MLIR_CMAKE_BUILD_DIR=build python setup.py build install + TORCH_MLIR_CMAKE_ALREADY_BUILT=1 TORCH_MLIR_CMAKE_BUILD_DIR=build PYTHONWARNINGS=ignore verbose_cmd python setup.py build install + elif [[ $setup_python_venv -eq 0 ]]; then + warning "Skipping Torch-MLIR Python package build" + warning "Make sure to have a correct Python environment set up" fi -else - echo "Skipping Torch-MLIR checkout and build" - echo "The following steps will need TORCH_MLIR_DIR to be set in their respective _CMAKE_OPTIONS" +elif [[ $checkout_and_build_torch_mlir -eq 0 ]]; then + warning "Skipping Torch-MLIR checkout and build" + warning "The following steps will need TORCH_MLIR_DIR to be set in their respective _CMAKE_OPTIONS" fi if [[ $checkout_upmem -eq 1 ]]; then if [ ! -d "$upmem_path" ]; then + status "Downloading UpMem SDK" upmem_archive="upmem.tar.gz" curl http://sdk-releases.upmem.com/2024.1.0/ubuntu_22.04/upmem-2024.1.0-Linux-x86_64.tar.gz --output "$upmem_archive" mkdir "$upmem_path" tar xf "$upmem_archive" -C "$upmem_path" --strip-components=1 rm "$upmem_archive" fi -else - echo "Skipping UpMem checkout" - echo "The following steps will need UPMEM_DIR to be set in their respective _CMAKE_OPTIONS" +elif [[ $checkout_upmem -eq 0 ]]; then + warning "Skipping UpMem checkout" + warning "The following steps will need UPMEM_DIR to be set in their respective _CMAKE_OPTIONS" fi cd "$cinnamon_path" if [ ! -d "build" ] || [ $reconfigure -eq 1 ]; then + status "Configuring Cinnamon" ln -s "$project_root/LICENSE" "$cinnamon_path/python/" 2>/dev/null || true dependency_paths="" @@ -239,9 +302,11 @@ if [ ! -d "build" ] || [ $reconfigure -eq 1 ]; then $CINNAMON_CMAKE_OPTIONS fi +status "Building Cinnamon" cmake --build build --target all if [[ $setup_python_venv -eq 1 ]] && [[ -n "$llvm_path" ]] && [[ -n "$torch_mlir_path" ]]; then + status "Building Cinnamon Python package" site_packages_dir="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" cinnamon_python_package_dir_src="$project_root/cinnamon/python/src/cinnamon" cinnamon_python_package_dir_dest="$site_packages_dir/cinnamon" @@ -269,6 +334,9 @@ if [[ $setup_python_venv -eq 1 ]] && [[ -n "$llvm_path" ]] && [[ -n "$torch_mlir if [[ $build_cinnamon_wheel -eq 1 ]]; then cd "$cinnamon_path/python" - python -m build + PYTHONWARNINGS=ignore verbose_cmd python -m build fi +elif [[ $setup_python_venv -eq 0 ]]; then + warning "Skipping Cinnamon Python package build" + warning "Make sure to have a correct Python environment set up" fi \ No newline at end of file