Merge pull request #536 from tiran/prebuilt-wheel-hook

Implement prebuilt-wheel hook
python-wheel-build · Jan 23, 2025 · e6f94c9 · e6f94c9
2 parents d7f5395 + 3634fa2
commit e6f94c9
Show file tree

Hide file tree

Showing 13 changed files with 163 additions and 12 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -100,6 +100,7 @@ jobs:
           - optimize_build
           - extra_metadata
           - elfdeps
+          - prebuilt_wheel_hook
         os:
           - ubuntu-latest
           - macos-latest

diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,8 @@ __pycache__/
 /package.json
 /e2e-output/
 /dist/
+/e2e/fromager_hooks/build
+/e2e/fromager_hooks/dist
 /e2e/pyo3_test/.cargo
 /e2e/pyo3_test/build
 /e2e/pyo3_test/dist

diff --git a/.mergify.yml b/.mergify.yml
@@ -27,11 +27,11 @@ pull_request_rules:
     conditions:
       - and:
           - check-success=e2e (3.11, 1.75, bootstrap, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_build_tags, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, build_settings, ubuntu-latest)
@@ -44,21 +44,22 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, optimize_build, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, override, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, pep517_build_sdist, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, prebuilt_wheel_hook, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_extras, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_extras, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_build_tags, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_extras, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_extras, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
+          - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, build, macos-latest)
           - check-success=e2e (3.12, 1.75, build, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, build_order, macos-latest)
@@ -83,6 +84,8 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, override, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, pep517_build_sdist, macos-latest)
           - check-success=e2e (3.12, 1.75, pep517_build_sdist, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, prebuilt_wheel_hook, macos-latest)
+          - check-success=e2e (3.12, 1.75, prebuilt_wheel_hook, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, prebuilt_wheels_alt_server, macos-latest)
           - check-success=e2e (3.12, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, report_missing_dependency, macos-latest)

diff --git a/docs/customization.md b/docs/customization.md
@@ -276,6 +276,8 @@ Fromager supports plugging in Python hooks to be run after build events.
 The `post_build` hook runs after a wheel is successfully built and can be used
 to publish that wheel to a package index or take other post-build actions.
 
+NOTE: The hook is not run for prebuilt wheels.
+
 Configure a `post_build` hook in your `pyproject.toml` like this:
 
 ```toml
@@ -302,3 +304,35 @@ def post_build(
         f"{req.name}: running post build hook for {sdist_filename} and {wheel_filename}"
     )
 ```
+
+### prebuilt_wheel
+
+The `prebuilt_wheel` hook runs after a prebuilt wheel has been downloaded and
+can be used to publish that wheel to a package index or take other post-build
+actions.
+
+Configure a `prebuilt_wheel` hook in your `pyproject.toml` like this:
+
+```toml
+[project.entry-points."fromager.hooks"]
+prebuilt_wheel = "package_plugins.module:function"
+```
+
+The input arguments to the `prebuilt_build` hook are the `WorkContext`,
+`Requirement` being built, the distribution name and version, and the wheel
+filename.
+
+NOTE: The files should not be renamed or moved.
+
+```python
+def prebuilt_wheel(
+    ctx: context.WorkContext,
+    req: Requirement,
+    dist_name: str,
+    dist_version: str,
+    wheel_filename: pathlib.Path,
+):
+    logger.info(
+        f"{req.name}: running prebuilt wheel hook for {wheel_filename}"
+    )
+```
diff --git a/...k/build/lib/package_plugins/post_build.py → ...s/build/lib/package_plugins/post_build.py b/...k/build/lib/package_plugins/post_build.py → ...s/build/lib/package_plugins/post_build.py
diff --git a/e2e/post_build_hook/pyproject.toml → e2e/fromager_hooks/pyproject.toml b/e2e/post_build_hook/pyproject.toml → e2e/fromager_hooks/pyproject.toml
@@ -29,4 +29,5 @@ requires-python = ">=3.11"
 dependencies = []
 
 [project.entry-points."fromager.hooks"]
-post_build = "package_plugins.post_build:after_build_wheel"
+post_build = "package_plugins.hooks:after_build_wheel"
+prebuilt_wheel = "package_plugins.hooks:after_prebuilt_wheel"
diff --git a/...ld_hook/src/package_plugins/post_build.py → ...omager_hooks/src/package_plugins/hooks.py b/...ld_hook/src/package_plugins/post_build.py → ...omager_hooks/src/package_plugins/hooks.py
@@ -22,3 +22,18 @@ def after_build_wheel(
     test_file = sdist_filename.parent / "test-output-file.txt"
     logger.info(f"{req.name}: post-build hook writing to {test_file}")
     test_file.write_text(f"{dist_name}=={dist_version}")
+
+
+def after_prebuilt_wheel(
+    ctx: context.WorkContext,
+    req: Requirement,
+    dist_name: str,
+    dist_version: str,
+    wheel_filename: pathlib.Path,
+):
+    logger.info(
+        f"{req.name}: running post build hook in {__name__} for {wheel_filename}"
+    )
+    test_file =  ctx.work_dir / "test-prebuilt.txt"
+    logger.info(f"{req.name}: prebuilt-wheel hook writing to {test_file}")
+    test_file.write_text(f"{dist_name}=={dist_version}")
diff --git a/e2e/prebuilt_settings/setuptools.yaml b/e2e/prebuilt_settings/setuptools.yaml
@@ -0,0 +1,3 @@
+variants:
+  cpu:
+    pre_built: True
diff --git a/e2e/test_build.sh b/e2e/test_build.sh
@@ -12,7 +12,7 @@ DIST="stevedore"
 VERSION="5.2.0"
 
 # Install hook for test
-pip install e2e/post_build_hook
+pip install e2e/fromager_hooks
 
 OS=$(uname)
 if [ "$OS" = "Darwin" ]; then

diff --git a/e2e/test_build_settings.sh b/e2e/test_build_settings.sh
@@ -12,7 +12,7 @@ DIST="stevedore"
 VERSION="5.2.0"
 
 # Install hook for test
-pip install e2e/post_build_hook
+pip install e2e/fromager_hooks
 
 # Bootstrap the test project
 fromager \

diff --git a/e2e/test_prebuilt_wheel_hook.sh b/e2e/test_prebuilt_wheel_hook.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test build-sequence prebuilt-wheel hook
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+# What are we building?
+DIST="setuptools"
+VERSION="75.8.0"
+
+# Install hook for test
+pip install e2e/fromager_hooks
+
+# Bootstrap the test project
+fromager \
+    --sdists-repo="$OUTDIR/sdists-repo" \
+    --wheels-repo="$OUTDIR/wheels-repo" \
+    --work-dir="$OUTDIR/work-dir" \
+    --settings-dir="$SCRIPTDIR/prebuilt_settings" \
+    bootstrap "${DIST}==${VERSION}"
+
+# Save the build order file but remove everything else.
+cp "$OUTDIR/work-dir/build-order.json" "$OUTDIR/"
+
+# Remove downloaded wheels to trigger hook
+rm -rf "$OUTDIR/wheels-repo"
+
+log="$OUTDIR/build-logs/${DIST}-build.log"
+fromager \
+    --log-file "$log" \
+    --work-dir "$OUTDIR/work-dir" \
+    --sdists-repo "$OUTDIR/sdists-repo" \
+    --wheels-repo "$OUTDIR/wheels-repo" \
+    --settings-dir="$SCRIPTDIR/prebuilt_settings" \
+    build-sequence "$OUTDIR/build-order.json"
+
+if ! grep -q "downloading prebuilt wheel ${DIST}==${VERSION}" "$log"; then
+  echo "Lack of message indicating download of prebuilt wheel" 1>&2
+  pass=false
+fi
+
+
+EXPECTED_FILES="
+wheels-repo/simple/${DIST}/${DIST}-${VERSION}-py3-none-any.whl
+work-dir/test-prebuilt.txt
+"
+
+pass=true
+for f in $EXPECTED_FILES; do
+  if [ ! -f "$OUTDIR/$f" ]; then
+    echo "FAIL: Did not find $OUTDIR/$f" 1>&2
+    pass=false
+  fi
+done
+
+if $pass; then
+  if ! grep -q "${DIST}==${VERSION}" $OUTDIR/work-dir/test-prebuilt.txt; then
+    echo "FAIL: Did not find content in post-build hook output file" 1>&2
+    pass=false
+  fi
+fi
+
+$pass
diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py
@@ -203,6 +203,13 @@ def build_sequence(
                     wheel_url=source_download_url,
                     output_directory=wkctx.wheels_build,
                 )
+                hooks.run_prebuilt_wheel_hooks(
+                    ctx=wkctx,
+                    req=req,
+                    dist_name=dist_name,
+                    dist_version=str(resolved_version),
+                    wheel_filename=wheel_filename,
+                )
             else:
                 logger.info(
                     "%s: building %s==%s", dist_name, dist_name, resolved_version

diff --git a/src/fromager/hooks.py b/src/fromager/hooks.py
@@ -59,3 +59,23 @@ def run_post_build_hooks(
             sdist_filename=sdist_filename,
             wheel_filename=wheel_filename,
         )
+
+
+def run_prebuilt_wheel_hooks(
+    ctx: context.WorkContext,
+    req: Requirement,
+    dist_name: str,
+    dist_version: str,
+    wheel_filename: pathlib.Path,
+) -> None:
+    hook_mgr = _get_hooks("prebuilt_wheel")
+    if hook_mgr.names():
+        logger.info(f"{req.name}: starting prebuilt-wheel hooks")
+    for ext in hook_mgr:
+        ext.plugin(
+            ctx=ctx,
+            req=req,
+            dist_name=dist_name,
+            dist_version=dist_version,
+            wheel_filename=wheel_filename,
+        )