Merge pull request #12259 from vkuznet/disutils2setuptools

Switch from distutils to setuptools:
- we agreed on WMCore meeting that we'll merge this PR to test new CI/CD pipeline
- we also agreed that after the merge we will create new RC
- and we will need to follow up with additional issue to merge existing CI/CD pipeline and new one

.github/workflows/build_and_install_locally.yaml

@@ -0,0 +1,63 @@
+# This workflow will build and install WMCore core services locally within
+# virtual environment
+
+on:
+  # this section fires workflow on a specific tag which matches some pattern
+  push:
+    tags:
+      - '*.*.*'
+  # this section forces manual builds
+  workflow_dispatch:
+    inputs:
+      name:
+        description: 'WMCore services'
+
+jobs:
+  # first job performs build and upload of packages to PyPI
+  build_and_publish_services:
+    name: Build_and_upload_to_pypi
+    runs-on: ubuntu-latest
+    environment:
+      name: production
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    strategy:
+      matrix:
+        target: [wmagent, wmagent-devtools, wmcore, reqmon, reqmgr2, global-workqueue, acdcserver, reqmgr2ms-unmerged,
+                 reqmgr2ms-output, reqmgr2ms-pileup, reqmgr2ms-rulecleaner, reqmgr2ms-transferor, reqmgr2ms-monitor]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup python 3.8
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Upgrade pip3
+        run: |
+          python3 -m pip install --upgrade pip
+
+      - name: Update the setup script template with package name
+        run: |
+          sed "s/PACKAGE_TO_BUILD/${{ matrix.target }}/" setup_template.py > setup.py
+
+      - name: Create requirements file
+        run: |
+          echo "create new requirements.txt without gfal dependencies for CI/CD"
+          cat requirements.txt | egrep -v "gfal" requirements.$pkg.txt
+          awk "/(${{ matrix.target }}$)|(${{ matrix.target }},)/ {print \$1}" requirements.$pkg.txt > requirements.txt
+
+      - name: Build sdist and wheels
+        run: python3 setup.py clean sdist bdist_wheel
+
+      - name: Verify built package
+        run: ls -lh dist/
+
+      - name: Install package locally from built artifacts
+        run: |
+          pip install -r requirements.txt
+          pip install --no-index --find-links=dist/ ${{ matrix.target }}
+
+      - name: Test package installation
+        run: |
+          pip list --format=freeze | grep ${{ matrix.target }}

MANIFEST.in

@@ -4,3 +4,8 @@ recursive-include bin *
 include requirements.txt
 include LICENSE
 include NOTICE
+graft src/couchapps
+graft src/css
+graft src/html
+graft src/javascript
+graft src/templates

bin/patchComponent.sh

@@ -80,7 +80,7 @@ echo "INFO: Current WMCoreTag: $currTag"
 # Find all possible locations for the component source
 # NOTE: We always consider PYTHONPATH first
 pythonLibPaths=$(echo $PYTHONPATH |sed -e "s/\:/ /g")
-pythonLibPaths="$pythonLibPaths $(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")"
+pythonLibPaths="$pythonLibPaths $(python -c "import sysconfig; print(sysconfig.get_path('purelib'))")"
 
 for path in $pythonLibPaths
 do

bin/test_local_build_and_install.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+restore() {
+    echo
+    echo "Restore setup.py and requirements.txt files from git repository..."
+    git checkout -- setup.py
+    git checkout -- requirements.txt
+    echo "Clean-up any build files..."
+    rm -rf build src/python/*.egg-info
+}
+
+# always execute restore function before script ends either by exit call or
+# through Ctrl-C (shell interrupt)
+trap restore EXIT
+trap restore SIGINT
+
+set -e  # Exit on any error
+
+# Define package list
+packages=("wmagent" "wmagent-devtools" "wmcore" "reqmon" "reqmgr2" "global-workqueue" "acdcserver" \
+          "reqmgr2ms-unmerged" "reqmgr2ms-output" "reqmgr2ms-pileup" "reqmgr2ms-rulecleaner" \
+          "reqmgr2ms-transferor" "reqmgr2ms-monitor")
+
+if [ -d venv ]; then
+    echo "WARNING: Found venv area, please either delete or relocate it to proceed..."
+    exit 1
+fi
+
+# Create and activate virtual environment
+python3 -m venv venv
+source venv/bin/activate
+
+# Upgrade build tools
+pip install --upgrade pip setuptools wheel
+
+# extract python version
+pyver=`python -V | awk '{print $2}' | cut -d'.' -f1,2`
+export PYTHONPATH=$PYTHONPATH:$PWD:$PWD/venv/lib/python$pyver/site-packages
+echo
+echo "PYTHONPATH=$PYTHONPATH"
+
+# preserve setup.py and requirements.txt files we they will be overwritten
+# below for every WM package we build
+echo
+echo "### Preserving setup.py and requirements.txt files..."
+# first restore these files from git
+restore
+# now let's make local copy to use in our builds
+cp setup.py setup.py.orig
+cp requirements.txt requirements.txt.orig
+
+# Loop through each package
+for pkg in "${packages[@]}"; do
+    echo
+    echo "### Building and testing package: $pkg ..."
+    sleep 2
+
+    # Clean previous builds
+    python3 setup.py clean
+    rm -rf dist build *.egg-info
+
+    # copy original files
+    /bin/cp -f setup.py.orig setup.py
+    /bin/cp -f requirements.txt.orig requirements.txt
+
+    # Update setup.py
+    sed "s/PACKAGE_TO_BUILD/${pkg}/" setup_template.py > setup.py
+
+    # Define temporary requirements file
+    TMP_REQUIREMENTS="/tmp/requirements_${pkg}.txt"
+
+    # Create package-specific requirements file
+    # NOTE: we will skip gfal and htcondor (on macOS) due to their special nature
+    # to avoid dependency failure
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        # no htcondor on macos, we will skip it during test
+        cat requirements.txt | egrep -v "htcondor|gfal" > requirements.$pkg.txt
+    else
+        cat requirements.txt | egrep -v "gfal" > requirements.$pkg.txt
+    fi
+    awk "/(${pkg}$)|(${pkg},)/ {print \$1}" requirements.$pkg.txt > requirements.txt
+
+    # Build package
+    python3 setup.py sdist bdist_wheel
+
+    # verify dist area
+    ls -lh dist/
+
+    # Install package locally
+    pip install -r requirements.txt
+    pip install --no-index --find-links=dist/ ${pkg}
+
+    # installed package installation
+    installed=`pip list | grep $pkg | awk '{print $1}'`
+    echo
+    if [ "$installed" == "$pkg" ]; then
+        echo "### Successfully built and installed: ${pkg} ..."
+        rm requirements.$pkg.txt
+        /bin/cp -f setup.py.orig setup.py
+        /bin/cp -f requirements.txt.orig requirements.txt
+        sleep 2
+    else
+        echo "ERROR: fail to install $pkg ..."
+        exit 1
+    fi
+done
+
+# now list all installed packages
+echo
+echo "### List installed WM packages:"
+pip list --format=freeze | grep -E "^($(IFS='|'; echo "${packages[*]}"))="
+
+echo
+echo "SUCCESS: All packages are built and installed successfully..."

bin/wmagent-component-standalone

@@ -20,13 +20,10 @@ import importlib
 import runpy
 import pkgutil
 import inspect
-import subprocess
-from pprint import pprint,pformat
-from distutils.sysconfig import get_python_lib
+import sysconfig
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from WMCore.Configuration import loadConfigurationFile
 from WMCore.ResourceControl.ResourceControl import ResourceControl
-from WMCore.Services.CRIC.CRIC import CRIC
 from WMCore.WMInit import connectToDB
 from Utils.FileTools import loadEnvFile
 
@@ -150,7 +147,7 @@ if __name__ == '__main__':
     # First find all possible locations for the component source
     # NOTE: We always consider PYTHONPATH first
     pythonLibPaths = os.getenv('PYTHONPATH', '').split(':')
-    pythonLibPaths.append(get_python_lib())
+    pythonLibPaths.append(sysconfig.get_path("purelib"))
     # Normalize paths and remove empty ones
     pythonLibPaths = [x for x in pythonLibPaths if x]
     for index, libPath in enumerate(pythonLibPaths):