From 2997e3ffa3c58cb320b0f9039a45462ef534a0c5 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 22 Jan 2025 12:07:46 +0100 Subject: [PATCH] Implement prebuilt-wheel hook The `prebuilt_wheel` hook complements the `post_build` hook. While the post build hook runs after a wheel has been built, the prebuilt wheel hook runs after a wheel has been downloaded. The new hook makes it possible to upload prebuilt wheels to an internal Python index. Fixes: #535 Signed-off-by: Christian Heimes --- .github/workflows/test.yaml | 1 + .gitignore | 2 + docs/customization.md | 34 ++++++++++ .../build/lib/package_plugins/post_build.py | 0 .../pyproject.toml | 3 +- .../src/package_plugins/hooks.py} | 15 +++++ e2e/prebuilt_settings/setuptools.yaml | 3 + e2e/test_build.sh | 2 +- e2e/test_build_settings.sh | 2 +- e2e/test_prebuilt_wheel_hook.sh | 65 +++++++++++++++++++ src/fromager/commands/build.py | 7 ++ src/fromager/hooks.py | 20 ++++++ 12 files changed, 151 insertions(+), 3 deletions(-) rename e2e/{post_build_hook => fromager_hooks}/build/lib/package_plugins/post_build.py (100%) rename e2e/{post_build_hook => fromager_hooks}/pyproject.toml (88%) rename e2e/{post_build_hook/src/package_plugins/post_build.py => fromager_hooks/src/package_plugins/hooks.py} (59%) create mode 100644 e2e/prebuilt_settings/setuptools.yaml create mode 100755 e2e/test_prebuilt_wheel_hook.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 665c9021..f5676750 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -100,6 +100,7 @@ jobs: - optimize_build - extra_metadata - elfdeps + - prebuilt_wheel_hook os: - ubuntu-latest - macos-latest diff --git a/.gitignore b/.gitignore index 1a7ddd56..4b36b777 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ __pycache__/ /package.json /e2e-output/ /dist/ +/e2e/fromager_hooks/build +/e2e/fromager_hooks/dist /e2e/pyo3_test/.cargo /e2e/pyo3_test/build /e2e/pyo3_test/dist diff --git a/docs/customization.md b/docs/customization.md index 0fe400cf..05043d85 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -276,6 +276,8 @@ Fromager supports plugging in Python hooks to be run after build events. The `post_build` hook runs after a wheel is successfully built and can be used to publish that wheel to a package index or take other post-build actions. +NOTE: The hook is not run for prebuilt wheels. + Configure a `post_build` hook in your `pyproject.toml` like this: ```toml @@ -302,3 +304,35 @@ def post_build( f"{req.name}: running post build hook for {sdist_filename} and {wheel_filename}" ) ``` + +### prebuilt_wheel + +The `prebuilt_wheel` hook runs after a prebuilt wheel has been downloaded and +can be used to publish that wheel to a package index or take other post-build +actions. + +Configure a `prebuilt_wheel` hook in your `pyproject.toml` like this: + +```toml +[project.entry-points."fromager.hooks"] +prebuilt_wheel = "package_plugins.module:function" +``` + +The input arguments to the `prebuilt_build` hook are the `WorkContext`, +`Requirement` being built, the distribution name and version, and the wheel +filename. + +NOTE: The files should not be renamed or moved. + +```python +def prebuilt_wheel( + ctx: context.WorkContext, + req: Requirement, + dist_name: str, + dist_version: str, + wheel_filename: pathlib.Path, +): + logger.info( + f"{req.name}: running prebuilt wheel hook for {wheel_filename}" + ) +``` diff --git a/e2e/post_build_hook/build/lib/package_plugins/post_build.py b/e2e/fromager_hooks/build/lib/package_plugins/post_build.py similarity index 100% rename from e2e/post_build_hook/build/lib/package_plugins/post_build.py rename to e2e/fromager_hooks/build/lib/package_plugins/post_build.py diff --git a/e2e/post_build_hook/pyproject.toml b/e2e/fromager_hooks/pyproject.toml similarity index 88% rename from e2e/post_build_hook/pyproject.toml rename to e2e/fromager_hooks/pyproject.toml index 54fde3e5..27a61f00 100644 --- a/e2e/post_build_hook/pyproject.toml +++ b/e2e/fromager_hooks/pyproject.toml @@ -29,4 +29,5 @@ requires-python = ">=3.11" dependencies = [] [project.entry-points."fromager.hooks"] -post_build = "package_plugins.post_build:after_build_wheel" +post_build = "package_plugins.hooks:after_build_wheel" +prebuilt_wheel = "package_plugins.hooks:after_prebuilt_wheel" diff --git a/e2e/post_build_hook/src/package_plugins/post_build.py b/e2e/fromager_hooks/src/package_plugins/hooks.py similarity index 59% rename from e2e/post_build_hook/src/package_plugins/post_build.py rename to e2e/fromager_hooks/src/package_plugins/hooks.py index a45030ee..37b15d6c 100644 --- a/e2e/post_build_hook/src/package_plugins/post_build.py +++ b/e2e/fromager_hooks/src/package_plugins/hooks.py @@ -22,3 +22,18 @@ def after_build_wheel( test_file = sdist_filename.parent / "test-output-file.txt" logger.info(f"{req.name}: post-build hook writing to {test_file}") test_file.write_text(f"{dist_name}=={dist_version}") + + +def after_prebuilt_wheel( + ctx: context.WorkContext, + req: Requirement, + dist_name: str, + dist_version: str, + wheel_filename: pathlib.Path, +): + logger.info( + f"{req.name}: running post build hook in {__name__} for {wheel_filename}" + ) + test_file = ctx.work_dir / "test-prebuilt.txt" + logger.info(f"{req.name}: prebuilt-wheel hook writing to {test_file}") + test_file.write_text(f"{dist_name}=={dist_version}") diff --git a/e2e/prebuilt_settings/setuptools.yaml b/e2e/prebuilt_settings/setuptools.yaml new file mode 100644 index 00000000..56894659 --- /dev/null +++ b/e2e/prebuilt_settings/setuptools.yaml @@ -0,0 +1,3 @@ +variants: + cpu: + pre_built: True diff --git a/e2e/test_build.sh b/e2e/test_build.sh index 5e6f11d1..ebafbc95 100755 --- a/e2e/test_build.sh +++ b/e2e/test_build.sh @@ -12,7 +12,7 @@ DIST="stevedore" VERSION="5.2.0" # Install hook for test -pip install e2e/post_build_hook +pip install e2e/fromager_hooks OS=$(uname) if [ "$OS" = "Darwin" ]; then diff --git a/e2e/test_build_settings.sh b/e2e/test_build_settings.sh index c371af29..a14e5eac 100755 --- a/e2e/test_build_settings.sh +++ b/e2e/test_build_settings.sh @@ -12,7 +12,7 @@ DIST="stevedore" VERSION="5.2.0" # Install hook for test -pip install e2e/post_build_hook +pip install e2e/fromager_hooks # Bootstrap the test project fromager \ diff --git a/e2e/test_prebuilt_wheel_hook.sh b/e2e/test_prebuilt_wheel_hook.sh new file mode 100755 index 00000000..37143ea4 --- /dev/null +++ b/e2e/test_prebuilt_wheel_hook.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test build-sequence prebuilt-wheel hook + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# What are we building? +DIST="setuptools" +VERSION="75.8.0" + +# Install hook for test +pip install e2e/fromager_hooks + +# Bootstrap the test project +fromager \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-dir="$SCRIPTDIR/prebuilt_settings" \ + bootstrap "${DIST}==${VERSION}" + +# Save the build order file but remove everything else. +cp "$OUTDIR/work-dir/build-order.json" "$OUTDIR/" + +# Remove downloaded wheels to trigger hook +rm -rf "$OUTDIR/wheels-repo" + +log="$OUTDIR/build-logs/${DIST}-build.log" +fromager \ + --log-file "$log" \ + --work-dir "$OUTDIR/work-dir" \ + --sdists-repo "$OUTDIR/sdists-repo" \ + --wheels-repo "$OUTDIR/wheels-repo" \ + --settings-dir="$SCRIPTDIR/prebuilt_settings" \ + build-sequence "$OUTDIR/build-order.json" + +if ! grep -q "downloading prebuilt wheel ${DIST}==${VERSION}" "$log"; then + echo "Lack of message indicating download of prebuilt wheel" 1>&2 + pass=false +fi + + +EXPECTED_FILES=" +wheels-repo/simple/${DIST}/${DIST}-${VERSION}-py3-none-any.whl +work-dir/test-prebuilt.txt +" + +pass=true +for f in $EXPECTED_FILES; do + if [ ! -f "$OUTDIR/$f" ]; then + echo "FAIL: Did not find $OUTDIR/$f" 1>&2 + pass=false + fi +done + +if $pass; then + if ! grep -q "${DIST}==${VERSION}" $OUTDIR/work-dir/test-prebuilt.txt; then + echo "FAIL: Did not find content in post-build hook output file" 1>&2 + pass=false + fi +fi + +$pass diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 0fbb5cc6..dfe564fa 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -203,6 +203,13 @@ def build_sequence( wheel_url=source_download_url, output_directory=wkctx.wheels_build, ) + hooks.run_prebuilt_wheel_hooks( + ctx=wkctx, + req=req, + dist_name=dist_name, + dist_version=str(resolved_version), + wheel_filename=wheel_filename, + ) else: logger.info( "%s: building %s==%s", dist_name, dist_name, resolved_version diff --git a/src/fromager/hooks.py b/src/fromager/hooks.py index 9dcc6cdf..ed52894f 100644 --- a/src/fromager/hooks.py +++ b/src/fromager/hooks.py @@ -59,3 +59,23 @@ def run_post_build_hooks( sdist_filename=sdist_filename, wheel_filename=wheel_filename, ) + + +def run_prebuilt_wheel_hooks( + ctx: context.WorkContext, + req: Requirement, + dist_name: str, + dist_version: str, + wheel_filename: pathlib.Path, +) -> None: + hook_mgr = _get_hooks("prebuilt_wheel") + if hook_mgr.names(): + logger.info(f"{req.name}: starting prebuilt-wheel hooks") + for ext in hook_mgr: + ext.plugin( + ctx=ctx, + req=req, + dist_name=dist_name, + dist_version=dist_version, + wheel_filename=wheel_filename, + )