Migrated to pyproject.toml; Improved building of the Docker container

.dockerignore

@@ -1,3 +1,5 @@
+# Note: /dist/ is needed in the docker build to access the distribution archive
+
 # User specific configuration files
 /myconfig/
 
@@ -10,7 +12,6 @@
 
 # Make build
 /build/
-/dist/
 /*.egg-info/
 /AUTHORS
 /ChangeLog

.github/workflows/publish.yml

@@ -9,6 +9,7 @@ on:
   push:  # When pushing a tag
     tags:
     - "*"
+    - "!*a0"  # Exclude initial tag for a new version
 
 jobs:
   publish:

.gitignore

@@ -8,6 +8,7 @@
 # Make build
 /build/
 /dist/
+/zhmc_os_forwarder/_version_scm.py
 /*.egg-info/
 /AUTHORS
 /ChangeLog

.readthedocs.yaml

@@ -21,4 +21,6 @@ sphinx:
 # Python requirements required to build the docs
 python:
    install:
-     - requirements: dev-requirements.txt
+     - requirements: requirements-base.txt
+     - requirements: requirements.txt
+     - requirements: requirements-develop.txt

Dockerfile

@@ -1,26 +1,77 @@
 # Dockerfile for zhmc-os-forwarder project
 #
 # This image runs the zhmc_os_forwarder command.
-# The standard metric definition file is provided in its default location, so
-# the -m option does not need to be specified.
 #
-# The HMC credentials file still needs to be made available to the container
+# The os forwarder config file needs to be made available to the container
 # using some mount option and specified with -c.
 #
-# Example docker command to run the forwarder using a locally built version of this image:
-#
-#   docker run -it --rm -v $(pwd)/myconfig:/root/myconfig -p 514:514 zhmcosforwarder -c /root/myconfig/config.yaml -v
+#   docker run --rm -v $(pwd)/myconfig:/root/myconfig -p 514:514 zhmc_os_forwarder -c /root/myconfig/config.yaml -v
+
+FROM python:3.12-alpine as builder
+
+# Path name of binary distribution archive of zhmc-os-forwarder package
+ARG bdist_file
+RUN : "${bdist_file:?Build argument bdist_file is required}"
+
+# Install some packages onto this minimal Alpine image:
+# - git - in case the Python requirements use git+https links
+# - gcc musl-dev - in case Python wheels based on C need to be built (e.g. for rpds)
+RUN apk add git gcc musl-dev
+
+# Make sure the installed Python commands are found
+ENV PATH=/root/.local/bin:$PATH
+
+# Install the Python package of this project
+COPY ${bdist_file} /tmp/${bdist_file}
+RUN pip install --user /tmp/${bdist_file}
+
+# Show the installed Linux packages
+RUN echo "Installed Linux packages:" \
+  && apk info -v
+
+# Show the installed Python packages
+RUN echo "Installed Python packages:" \
+  && pip list
+
+# Display files in 'rpds' Python package (verifying that it can be imported)
+RUN echo "Files in rpds Python package:" \
+  && python -c "import rpds, os, sys; rpds_dir=os.path.dirname(rpds.__file__); print(rpds_dir); sys.stdout.flush(); os.system(f'ls -al {rpds_dir}')"
+
+# The Python 'rpds' package (used by 'jsonschema') has a shared library that is
+# built during its installation, and thus depends on APIs of the system and
+# the C library of the builder OS used in the first stage of this Dockerfile.
+# Therefore, the OS used in the final stage needs to be compatible with the
+# builder OS. We use the same OS image to make sure.
+FROM python:3.12-alpine
+
+# Version of the zhmc-os-forwarder package
+ARG package_version
+RUN : "${package_version:?Build argument package_version is required}"
+
+# Image build date in ISO-8601 format
+ARG build_date
+RUN : "${build_date:?Build argument build_date is required}"
+
+# Git commit ID of the zhmc-os-forwarder repo used to build the image
+ARG git_commit
+RUN : "${git_commit:?Build argument git_commit is required}"
 
-FROM python:3.9-slim
+# Set image metadata
+LABEL org.opencontainers.image.title="IBM Z HMC OS Message Forwarder"
+LABEL org.opencontainers.image.version="${package_version}"
+LABEL org.opencontainers.image.authors="Andreas Maier"
+LABEL org.opencontainers.image.created="${build_date}"
+LABEL org.opencontainers.image.url="https://github.com/zhmcclient/zhmc-os-forwarder"
+LABEL org.opencontainers.image.documentation="https://zhmc-os-forwarder.readthedocs.io"
+LABEL org.opencontainers.image.source="https://github.com/zhmcclient/zhmc-os-forwarder"
+LABEL org.opencontainers.image.licenses="Apache Software License 2.0"
+LABEL org.opencontainers.image.revision="${git_commit}"
 
-# Install this package
-ENV TMP_DIR=/tmp/zhmc-os-forwarder
-WORKDIR $TMP_DIR
-COPY . $TMP_DIR
-RUN pip install . && rm -rf $TMP_DIR
+# Copy the installed Python packages from the builder image
+COPY --from=builder /root/.local /root/.local
 
-# Set the current directory when running this image
-WORKDIR /root
+# Make sure the installed Python commands are found
+ENV PATH=/root/.local/bin:$PATH
 
 EXPOSE 9291
 ENTRYPOINT ["zhmc_os_forwarder"]

Makefile

@@ -95,13 +95,28 @@ else
 endif
 
 package_name := zhmc_os_forwarder
-package_version := $(shell $(PYTHON_CMD) setup.py --version)
-docker_registry := zhmcosforwarder
+
+# Package version (e.g. "1.0.0a1.dev10+gd013028e" during development, or "1.0.0"
+# when releasing).
+# Note: The package version is automatically calculated by setuptools_scm based
+# on the most recent tag in the commit history, increasing the least significant
+# version indicator by 1.
+package_version := $(shell $(PYTHON_CMD) -m setuptools_scm)
+
+# Docker image
+docker_image_name := zhmc_os_forwarder
+docker_image_tag := latest
 
 python_mn_version := $(shell $(PYTHON_CMD) -c "import sys; sys.stdout.write('{}.{}'.format(sys.version_info[0], sys.version_info[1]))")
 pymn := $(shell $(PYTHON_CMD) -c "import sys; sys.stdout.write('py{}{}'.format(sys.version_info[0], sys.version_info[1]))")
 
 package_dir := $(package_name)
+
+# The version file is recreated by setuptools-scm on every build, so it is
+# excluded from git, and also from some dependency lists.
+version_file := $(package_dir)/_version_scm.py
+
+# Source files in the package
 package_py_files := \
     $(wildcard $(package_dir)/*.py) \
     $(wildcard $(package_dir)/*/*.py) \
@@ -112,17 +127,18 @@ test_py_files := \
     $(wildcard $(test_dir)/*/*.py) \
 
 dist_dir := dist
-bdist_file := $(dist_dir)/$(package_name)-$(package_version)-py2.py3-none-any.whl
+bdist_file := $(dist_dir)/$(package_name)-$(package_version)-py3-none-any.whl
 sdist_file := $(dist_dir)/$(package_name)-$(package_version).tar.gz
 
-# This is also used for 'include' statements in MANIFEST.in.
-# Wildcards can be used directly (i.e. without wildcard function).
-dist_included_files := \
-    setup.py \
+# Dependencies of the distribution archives. Since the $(version_file) is
+# created when building the distribution archives, this must not contain
+# the $(version_file).
+dist_dependent_files := \
+    pyproject.toml \
     LICENSE \
     README.md \
     requirements.txt \
-    $(package_py_files) \
+    $(filter-out $(version_file), $(package_py_files)) \
 
 doc_dir := docs
 doc_build_dir := build_docs
@@ -135,8 +151,7 @@ doc_dependent_files := \
 
 # Source files for checks (with PyLint and Flake8, etc.)
 check_py_files := \
-    setup.py \
-    $(package_py_files) \
+    $(filter-out $(version_file), $(package_py_files)) \
     $(test_py_files) \
     $(doc_dir)/conf.py \
 
@@ -148,7 +163,7 @@ check_reqs_packages := pip_check_reqs pipdeptree build pytest coverage coveralls
 
 # Safety policy file
 safety_install_policy_file := .safety-policy-install.yml
-safety_all_policy_file := .safety-policy-all.yml
+safety_develop_policy_file := .safety-policy-develop.yml
 
 # Bandit config file
 bandit_rc_file := .bandit.toml
@@ -165,7 +180,7 @@ pylint_rc_file := .pylintrc
 pytest_cov_opts := --cov $(package_name) --cov-config .coveragerc --cov-report=html:htmlcov
 
 ifeq ($(PACKAGE_LEVEL),minimum)
-  pip_level_opts := -c minimum-constraints.txt
+  pip_level_opts := -c minimum-constraints-develop.txt -c minimum-constraints-install.txt
 else
   ifeq ($(PACKAGE_LEVEL),latest)
     pip_level_opts := --upgrade --upgrade-strategy eager
@@ -186,13 +201,13 @@ help:
 	@echo "  check      - Perform flake8 checks"
 	@echo "  ruff       - Perform ruff checks (an alternate lint tool)"
 	@echo "  pylint     - Perform pylint checks"
-	@echo "  safety     - Run safety for install and all"
+	@echo "  safety     - Run safety checker"
 	@echo "  bandit     - Run bandit checker"
 	@echo "  test       - Perform unit tests including coverage checker"
 	@echo "  build      - Build the distribution files in $(dist_dir)"
 	@echo "  builddoc   - Build the documentation in $(doc_build_dir)"
 	@echo "  all        - Do all of the above"
-	@echo "  docker     - Build local Docker image in registry $(docker_registry)"
+	@echo "  docker     - Build local Docker image $(docker_image_name):$(docker_image_tag)"
 	@echo "  authors    - Generate AUTHORS.md file from git log"
 	@echo "  upload     - Upload the package to Pypi"
 	@echo "  clean      - Remove any temporary files"
@@ -203,7 +218,7 @@ help:
 	@echo "  PACKAGE_LEVEL - Package level to be used for installing dependent Python"
 	@echo "      packages in 'install' and 'develop' targets:"
 	@echo "        latest - Latest package versions available on Pypi"
-	@echo "        minimum - A minimum version as defined in minimum-constraints.txt"
+	@echo "        minimum - A minimum version as defined in minimum-constraints-develop.txt"
 	@echo "      Optional, defaults to 'latest'."
 	@echo '  PYTHON_CMD=... - Name of python command. Default: python'
 	@echo '  PIP_CMD=... - Name of pip command. Default: pip'
@@ -255,15 +270,15 @@ pylint: $(done_dir)/pylint_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo '$@ done.'
 
 .PHONY: safety
-safety: $(done_dir)/safety_all_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done
+safety: $(done_dir)/safety_develop_$(pymn)_$(PACKAGE_LEVEL).done $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo "Makefile: $@ done."
 
 .PHONY: bandit
 bandit: $(done_dir)/bandit_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo "Makefile: $@ done."
 
 .PHONY: check_reqs
-check_reqs: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done minimum-constraints.txt minimum-constraints-install.txt requirements.txt
+check_reqs: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done minimum-constraints-develop.txt minimum-constraints-install.txt requirements.txt
 	@echo "Makefile: Checking missing dependencies of this package"
 	pip-missing-reqs $(package_name) --requirements-file=requirements.txt
 	pip-missing-reqs $(package_name) --requirements-file=minimum-constraints-install.txt
@@ -273,7 +288,9 @@ ifeq ($(PLATFORM),Windows_native)
 	@echo "Makefile: Warning: Skipping the checking of missing dependencies of site-packages directory on native Windows" >&2
 else
 	@echo "Makefile: Checking missing dependencies of some development packages in our minimum versions"
-	@rc=0; for pkg in $(check_reqs_packages); do dir=$$($(PYTHON_CMD) -c "import $${pkg} as m,os; dm=os.path.dirname(m.__file__); d=dm if not dm.endswith('site-packages') else m.__file__; print(d)"); cmd="pip-missing-reqs $${dir} --requirements-file=minimum-constraints.txt"; echo $${cmd}; $${cmd}; rc=$$(expr $${rc} + $${?}); done; exit $${rc}
+	bash -c "cat minimum-constraints-develop.txt minimum-constraints-install.txt >tmp_minimum-constraints-all.txt"
+	@rc=0; for pkg in $(check_reqs_packages); do dir=$$($(PYTHON_CMD) -c "import $${pkg} as m,os; dm=os.path.dirname(m.__file__); d=dm if not dm.endswith('site-packages') else m.__file__; print(d)"); cmd="pip-missing-reqs $${dir} --requirements-file=tmp_minimum-constraints-all.txt"; echo $${cmd}; $${cmd}; rc=$$(expr $${rc} + $${?}); done; exit $${rc}
+	-$(call RM_FUNC,tmp_minimum-constraints-all.txt)
 	@echo "Makefile: Done checking missing dependencies of some development packages in our minimum versions"
 endif
 	@echo "Makefile: $@ done."
@@ -329,7 +346,7 @@ clean:
 	-$(call RM_R_FUNC,*.pyc)
 	-$(call RM_R_FUNC,*.tmp)
 	-$(call RM_R_FUNC,tmp_*)
-	-$(call RM_FUNC,.coverage AUTHORS ChangeLog)
+	-$(call RM_FUNC,.coverage MANIFEST MANIFEST.in AUTHORS ChangeLog)
 	-$(call RMDIR_R_FUNC,__pycache__)
 	-$(call RMDIR_FUNC,build $(package_name).egg-info .pytest_cache)
 	@echo "Makefile: $@ done."
@@ -340,25 +357,24 @@ clobber: clean
 	-$(call RM_R_FUNC,*.done)
 	@echo "Makefile: $@ done."
 
-$(done_dir)/install_base_$(pymn)_$(PACKAGE_LEVEL).done: minimum-constraints.txt minimum-constraints-install.txt
+$(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done: minimum-constraints-develop.txt minimum-constraints-install.txt requirements-base.txt
 	@echo "Makefile: Installing base packages with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
 	-$(call RM_FUNC,$@)
-	bash -c 'pv=$$($(PYTHON_CMD) -m pip --version); if [[ $$pv =~ (^pip [1-8]\..*) ]]; then $(PYTHON_CMD) -m pip install pip==9.0.1; fi'
-	$(PYTHON_CMD) -m pip install $(pip_level_opts) pip setuptools wheel
+	$(PYTHON_CMD) -m pip install $(pip_level_opts) -r requirements-base.txt
 	@echo "Makefile: Done installing base packages"
 	echo "done" >$@
 
-$(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/install_base_$(pymn)_$(PACKAGE_LEVEL).done requirements.txt minimum-constraints.txt minimum-constraints-install.txt setup.py
+$(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/base_$(pymn)_$(PACKAGE_LEVEL).done requirements.txt minimum-constraints-develop.txt minimum-constraints-install.txt pyproject.toml
 	@echo "Makefile: Installing package (non-editable) and its prerequisites with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
 	-$(call RM_FUNC,$@)
 	$(PYTHON_CMD) -m pip install $(pip_level_opts) .
 	@echo "Makefile: Done installing package and its prerequisites"
 	echo "done" >$@
 
-$(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done dev-requirements.txt requirements.txt minimum-constraints.txt minimum-constraints-install.txt
+$(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/install_$(pymn)_$(PACKAGE_LEVEL).done requirements-develop.txt minimum-constraints-develop.txt minimum-constraints-install.txt
 	@echo "Makefile: Installing prerequisites for development with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
 	-$(call RM_FUNC,$@)
-	$(PYTHON_CMD) -m pip install $(pip_level_opts) -r dev-requirements.txt
+	$(PYTHON_CMD) -m pip install $(pip_level_opts) -r requirements-develop.txt
 	@echo "Makefile: Done installing prerequisites for development"
 	echo "done" >$@
 
@@ -367,32 +383,17 @@ $(doc_build_file): $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done $(doc_depen
 	sphinx-build -b html -v $(doc_dir) $(doc_build_dir)
 	@echo "Makefile: Done generating HTML documentation"
 
-# Note: distutils depends on the right files specified in MANIFEST.in, even when
-# they are already specified e.g. in 'package_data' in setup.py.
-# We generate the MANIFEST.in file automatically, to have a single point of
-# control (this Makefile) for what gets into the distribution archive.
-MANIFEST.in: Makefile $(dist_included_files)
-	@echo "Makefile: Creating the manifest input file"
-	echo "# MANIFEST.in file generated by Makefile - DO NOT EDIT!!" >$@
-ifeq ($(PLATFORM),Windows_native)
-	for %%f in ($(dist_included_files)) do (echo include %%f >>$@)
-else
-	echo "$(dist_included_files)" | xargs -n 1 echo include >>$@
-endif
-	@echo "Makefile: Done creating the manifest input file: $@"
-
-# Distribution archives.
-# Note: Deleting MANIFEST causes distutils (setup.py) to read MANIFEST.in and to
-# regenerate MANIFEST. Otherwise, changes in MANIFEST.in will not be used.
-# Note: Deleting build is a safeguard against picking up partial build products
-# which can lead to incorrect hashbangs in scripts in wheel archives.
-$(bdist_file) $(sdist_file): _check_version $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile MANIFEST.in $(dist_included_files)
-	-$(call RM_FUNC,MANIFEST)
-	-$(call RMDIR_FUNC,build $(package_name).egg-info .eggs)
-	$(PYTHON_CMD) -m build --outdir $(dist_dir)
-	@echo 'Done: Created distribution archives: $@'
-
-$(done_dir)/flake8_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done $(flake8_rc_file)
+$(sdist_file): $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(dist_dependent_files) $(version_file)
+	@echo "Makefile: Building the source distribution archive: $(sdist_file)"
+	$(PYTHON_CMD) -m build --sdist --outdir $(dist_dir) .
+	@echo "Makefile: Done building the source distribution archive: $(sdist_file)"
+
+$(bdist_file) $(version_file): $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(dist_dependent_files)
+	@echo "Makefile: Building the wheel distribution archive: $(bdist_file)"
+	$(PYTHON_CMD) -m build --wheel --outdir $(dist_dir) -C--universal .
+	@echo "Makefile: Done building the wheel distribution archive: $(bdist_file)"
+
+$(done_dir)/flake8_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done $(flake8_rc_file) $(check_py_files)
 	@echo "Makefile: Performing flake8 checks with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
 	flake8 --config $(flake8_rc_file) $(check_py_files)
 	echo "done" >$@
@@ -411,12 +412,12 @@ $(done_dir)/pylint_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(
 	echo "done" >$@
 	@echo "Makefile: Done performing pylint checks"
 
-$(done_dir)/safety_all_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_all_policy_file) minimum-constraints.txt minimum-constraints-install.txt
-	@echo "Makefile: Running Safety for all packages (and tolerate safety issues when RUN_TYPE is normal or scheduled)"
+$(done_dir)/safety_develop_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_develop_policy_file) minimum-constraints-develop.txt
+	@echo "Makefile: Running Safety for development packages (and tolerate safety issues when RUN_TYPE is normal or scheduled)"
 	-$(call RM_FUNC,$@)
-	bash -c "safety check --policy-file $(safety_all_policy_file) -r minimum-constraints.txt --full-report || test '$(RUN_TYPE)' == 'normal' || test '$(RUN_TYPE)' == 'scheduled' || exit 1"
+	bash -c "safety check --policy-file $(safety_develop_policy_file) -r minimum-constraints-develop.txt --full-report || test '$(RUN_TYPE)' == 'normal' || test '$(RUN_TYPE)' == 'scheduled' || exit 1"
 	echo "done" >$@
-	@echo "Makefile: Done running Safety for all packages"
+	@echo "Makefile: Done running Safety for development packages"
 
 $(done_dir)/safety_install_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Makefile $(safety_install_policy_file) minimum-constraints-install.txt
 	@echo "Makefile: Running Safety for install packages (and tolerate safety issues when RUN_TYPE is normal)"
@@ -432,9 +433,11 @@ $(done_dir)/bandit_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(
 	echo "done" >$@
 	@echo "Makefile: Done running Bandit"
 
-$(done_dir)/docker_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Dockerfile .dockerignore Makefile MANIFEST.in $(dist_included_files)
-	@echo "Makefile: Building Docker image $(docker_registry):latest"
+$(done_dir)/docker_$(pymn)_$(PACKAGE_LEVEL).done: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done Dockerfile .dockerignore $(bdist_file)
+	@echo "Makefile: Building Docker image $(docker_image_name):$(docker_image_tag)"
 	-$(call RM_FUNC,$@)
-	docker build -t $(docker_registry):latest .
+	docker build --tag $(docker_image_name):$(docker_image_tag) --build-arg bdist_file=$(bdist_file) --build-arg package_version=$(subst +,.,$(package_version)) --build-arg build_date="$(shell date -Iseconds)" --build-arg git_commit="$(shell git rev-parse HEAD)" .
+	docker run --rm $(docker_image_name):$(docker_image_tag) --version
+	docker image list --filter reference=$(docker_image_name)
 	@echo "Makefile: Done building Docker image"
 	echo "done" >$@

changes/100.feature.rst

@@ -0,0 +1 @@
+Improved building of the Docker container to reduce its size.

changes/100.incompatible.rst

@@ -0,0 +1,2 @@
+Changed mage name of Docker container image from 'zhmcosforwarder' to
+'zhmc_os_forwarder' to match the command name.

changes/80.feature.rst

@@ -0,0 +1 @@
+Migrated to pyproject.toml.