From f9cfba2d350294327cd3ac070bbdb1eaeae2f9c7 Mon Sep 17 00:00:00 2001 From: Shubh Bapna Date: Thu, 26 Sep 2024 17:42:06 -0400 Subject: [PATCH] modularize handle_requirement Signed-off-by: Shubh Bapna --- src/fromager/bootstrapper.py | 557 +++++++++++++++++++++++++++++ src/fromager/commands/bootstrap.py | 12 +- src/fromager/context.py | 67 ---- src/fromager/requirements_file.py | 6 + src/fromager/sdist.py | 548 ---------------------------- src/fromager/sources.py | 7 +- src/fromager/wheels.py | 2 +- tests/test_bootstrapper.py | 363 +++++++++++++++++++ tests/test_context.py | 134 ------- tests/test_sdist.py | 245 ------------- 10 files changed, 935 insertions(+), 1006 deletions(-) create mode 100644 src/fromager/bootstrapper.py delete mode 100644 src/fromager/sdist.py create mode 100644 tests/test_bootstrapper.py delete mode 100644 tests/test_sdist.py diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py new file mode 100644 index 00000000..1863b452 --- /dev/null +++ b/src/fromager/bootstrapper.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +import json +import logging +import operator +import pathlib +import shutil +import typing + +from packaging.requirements import Requirement +from packaging.utils import NormalizedName, canonicalize_name +from packaging.version import Version + +from . import ( + build_environment, + dependencies, + finders, + progress, + resolver, + server, + sources, + wheels, +) +from .dependency_graph import DependencyGraph +from .requirements_file import RequirementType, SourceType + +if typing.TYPE_CHECKING: + from . import context + +logger = logging.getLogger(__name__) + + +class Bootstrapper: + def __init__( + self, + ctx: context.WorkContext, + progressbar: progress.Progressbar | None = None, + prev_graph: DependencyGraph | None = None, + ) -> None: + self.ctx = ctx + self.progressbar = progressbar or progress.Progressbar(None) + self.prev_graph = prev_graph + self.why: list[tuple[RequirementType, Requirement, Version]] = [] + # Push items onto the stack as we start to resolve their + # dependencies so at the end we have a list of items that need to + # be built in order. + self._build_stack: list[typing.Any] = [] + self._build_requirements: set[tuple[NormalizedName, str]] = set() + + # Track requirements we've seen before so we don't resolve the + # same dependencies over and over and so we can break cycles in + # the dependency list. The key is the requirements spec, rather + # than the package, in case we do have multiple rules for the same + # package. + self._seen_requirements: set[tuple[NormalizedName, tuple[str, ...], str]] = ( + set() + ) + + self._build_order_filename = self.ctx.work_dir / "build-order.json" + + def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: + constraint = self.ctx.constraints.get_constraint(req.name) + if constraint: + logger.info( + f"{req.name}: incoming requirement {req} matches constraint {constraint}. Will apply both." + ) + + pbi = self.ctx.package_build_info(req) + if pbi.pre_built: + resolved_version, wheel_url, wheel_filename, unpack_dir = ( + self._handle_prebuilt_sources(req, req_type) + ) + # Remember that this is a prebuilt wheel, and where we got it. + source_url = wheel_url + source_url_type = str(SourceType.PREBUILT) + else: + resolved_version, source_url, source_filename, source_url_type = ( + self._handle_sources(req, req_type) + ) + + self._add_to_graph(req, req_type, resolved_version, source_url) + + # Avoid cyclic dependencies and redundant processing. + if self._has_been_seen(req, resolved_version): + logger.debug( + f"{req.name}: redundant {req_type} requirement {self.why} -> {req} resolves to {resolved_version}" + ) + return resolved_version + self._mark_as_seen(req, resolved_version) + + logger.info( + f"{req.name}: new {req_type} dependency {req} resolves to {resolved_version}" + ) + + # Build the dependency chain up to the point of this new + # requirement using a new list so we can avoid modifying the list + # we're given. + self.why.append((req_type, req, resolved_version)) + + # for cleanup + build_env = None + sdist_root_dir = None + if not pbi.pre_built: + ( + sdist_root_dir, + unpack_dir, + build_dependencies, + ) = self._prepare_source(req, source_filename, resolved_version) + + found_wheel_filename = self._is_wheel_built(req, resolved_version) + if not found_wheel_filename: + wheel_filename, build_env = self._build( + req, resolved_version, sdist_root_dir, build_dependencies + ) + else: + wheel_filename = found_wheel_filename + + self._add_to_build_order( + req=req, + version=resolved_version, + source_url=source_url, + source_url_type=source_url_type, + prebuilt=pbi.pre_built, + constraint=constraint, + ) + + # Process installation dependencies for all wheels. + install_dependencies = dependencies.get_install_dependencies_of_wheel( + req, wheel_filename, unpack_dir + ) + self.progressbar.update_total(len(install_dependencies)) + for dep in self._sort_requirements(install_dependencies): + try: + self.bootstrap(dep, RequirementType.INSTALL) + except Exception as err: + raise ValueError( + f"could not handle {RequirementType.INSTALL} dependency {dep} for {self.why}" + ) from err + self.progressbar.update() + + # we are done processing this req, so lets remove it from the why chain + self.why.pop() + self._cleanup(req, sdist_root_dir, build_env) + return resolved_version + + def _is_wheel_built( + self, req: Requirement, resolved_version: Version + ) -> pathlib.Path | None: + pbi = self.ctx.package_build_info(req) + + # FIXME: This is a bit naive, but works for most wheels, including + # our more expensive ones, and there's not a way to know the + # actual name without doing most of the work to build the wheel. + wheel_filename = finders.find_wheel( + downloads_dir=self.ctx.wheels_downloads, + req=req, + dist_version=str(resolved_version), + build_tag=pbi.build_tag(resolved_version), + ) + if wheel_filename: + logger.info( + f"{req.name}: have wheel version {resolved_version}: {wheel_filename}" + ) + return wheel_filename + + def _build( + self, + req: Requirement, + resolved_version: Version, + sdist_root_dir: pathlib.Path, + build_dependencies: set[Requirement], + ) -> tuple[pathlib.Path, build_environment.BuildEnvironment]: + build_env = build_environment.BuildEnvironment( + self.ctx, + sdist_root_dir.parent, + build_dependencies, + ) + try: + find_sdist_result = finders.find_sdist( + self.ctx, self.ctx.sdists_builds, req, str(resolved_version) + ) + if not find_sdist_result: + sources.build_sdist( + ctx=self.ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + else: + logger.info( + f"{req.name} have sdist version {resolved_version}: {find_sdist_result}" + ) + except Exception as err: + logger.warning(f"{req.name}: failed to build source distribution: {err}") + + built_filename = wheels.build_wheel( + ctx=self.ctx, + req=req, + sdist_root_dir=sdist_root_dir, + version=resolved_version, + build_env=build_env, + ) + server.update_wheel_mirror(self.ctx) + # When we update the mirror, the built file moves to the + # downloads directory. + wheel_filename = self.ctx.wheels_downloads / built_filename.name + logger.info( + f"{req.name}: built wheel for version {resolved_version}: {wheel_filename}" + ) + return wheel_filename, build_env + + def _prepare_source( + self, req: Requirement, source_filename: pathlib.Path, resolved_version: Version + ) -> tuple[pathlib.Path, pathlib.Path, set[Requirement]]: + sdist_root_dir = sources.prepare_source( + ctx=self.ctx, + req=req, + source_filename=source_filename, + version=resolved_version, + ) + unpack_dir = sdist_root_dir.parent + + build_system_dependencies = dependencies.get_build_system_dependencies( + ctx=self.ctx, req=req, sdist_root_dir=sdist_root_dir + ) + self._handle_build_requirements( + RequirementType.BUILD_SYSTEM, + build_system_dependencies, + ) + + build_backend_dependencies = dependencies.get_build_backend_dependencies( + ctx=self.ctx, req=req, sdist_root_dir=sdist_root_dir + ) + self._handle_build_requirements( + RequirementType.BUILD_BACKEND, + build_backend_dependencies, + ) + + build_sdist_dependencies = dependencies.get_build_sdist_dependencies( + ctx=self.ctx, req=req, sdist_root_dir=sdist_root_dir + ) + self._handle_build_requirements( + RequirementType.BUILD_SDIST, + build_sdist_dependencies, + ) + + return ( + sdist_root_dir, + unpack_dir, + build_system_dependencies + | build_backend_dependencies + | build_sdist_dependencies, + ) + + def _handle_build_requirements( + self, build_type: RequirementType, build_dependencies: set[Requirement] + ) -> None: + self.progressbar.update_total(len(build_dependencies)) + + for dep in self._sort_requirements(build_dependencies): + try: + resolved = self.bootstrap(req=dep, req_type=build_type) + except Exception as err: + raise ValueError( + f"could not handle {build_type} dependency {dep} for {self.why}" + ) from err + # We may need these dependencies installed in order to run build hooks + # Example: frozenlist build-system.requires includes expandvars because + # it is used by the packaging/pep517_backend/ build backend + build_environment.maybe_install(self.ctx, dep, build_type, str(resolved)) + self.progressbar.update() + + def _handle_sources( + self, req: Requirement, req_type: RequirementType + ) -> tuple[Version, str, pathlib.Path, str]: + source_url, resolved_version = self._resolve_source_with_history( + req=req, + req_type=req_type, + ) + + source_filename = sources.download_source( + ctx=self.ctx, + req=req, + version=resolved_version, + download_url=source_url, + ) + source_url_type = sources.get_source_type(self.ctx, req) + + return (resolved_version, source_url, source_filename, source_url_type) + + def _handle_prebuilt_sources( + self, req: Requirement, req_type: RequirementType + ) -> tuple[Version, str, pathlib.Path, pathlib.Path]: + logger.info(f"{req.name}: {req_type} requirement {req} uses a pre-built wheel") + + wheel_url, resolved_version = self._resolve_prebuilt_with_history( + req=req, + req_type=req_type, + ) + + wheel_filename = wheels.download_wheel(req, wheel_url, self.ctx.wheels_prebuilt) + + # Add the wheel to the mirror so it is available to anything + # that needs to install it. We leave a copy in the prebuilt + # directory to make it easy to remove the wheel from the + # downloads directory before uploading to a proper package + # index. + dest_name = self.ctx.wheels_downloads / wheel_filename.name + if not dest_name.exists(): + logger.info(f"{req.name}: updating temporary mirror with pre-built wheel") + shutil.copy(wheel_filename, dest_name) + server.update_wheel_mirror(self.ctx) + unpack_dir = self.ctx.work_dir / f"{req.name}-{resolved_version}" + if not unpack_dir.exists(): + unpack_dir.mkdir() + return (resolved_version, wheel_url, wheel_filename, unpack_dir) + + def _resolve_source_with_history( + self, + req: Requirement, + req_type: RequirementType, + ) -> tuple[str, Version]: + cached_resolution = self._resolve_from_graph( + req=req, + req_type=req_type, + pre_built=False, + ) + if cached_resolution: + source_url, resolved_version = cached_resolution + logger.debug( + f"{req.name}: resolved from previous bootstrap to {resolved_version}" + ) + else: + source_url, resolved_version = sources.resolve_source( + ctx=self.ctx, req=req, sdist_server_url=resolver.PYPI_SERVER_URL + ) + return (source_url, resolved_version) + + def _resolve_prebuilt_with_history( + self, + req: Requirement, + req_type: RequirementType, + ) -> tuple[str, Version]: + cached_resolution = self._resolve_from_graph( + req=req, + req_type=req_type, + pre_built=True, + ) + + if cached_resolution: + wheel_url, resolved_version = cached_resolution + logger.debug( + f"{req.name}: resolved from previous bootstrap to {resolved_version}" + ) + else: + servers = wheels.get_wheel_server_urls(self.ctx, req) + wheel_url, resolved_version = wheels.resolve_prebuilt_wheel( + ctx=self.ctx, req=req, wheel_server_urls=servers + ) + return (wheel_url, resolved_version) + + def _resolve_from_graph( + self, + req: Requirement, + req_type: RequirementType, + pre_built: bool, + ) -> tuple[str, Version] | None: + _, parent_req, _ = self.why[-1] if self.why else (None, None, None) + + # we have already resolved top level reqs before bootstrapping + # so they should already be in the root node + if req_type == RequirementType.TOP_LEVEL: + for edge in self.ctx.dependency_graph.get_root_node().get_outgoing_edges( + req.name, RequirementType.TOP_LEVEL + ): + if edge.req == req: + return ( + edge.destination_node.download_url, + edge.destination_node.version, + ) + # this should never happen since we already resolved top level reqs and their + # resolution should be in the root nodes + raise ValueError( + f"{req.name}: {req} appears as a toplevel requirement but it's resolution does not exist in the root node of the graph" + ) + + if not self.prev_graph: + return None + + seen_version: set[str] = set() + + # first perform resolution using the top level reqs before looking at history + possible_versions_in_top_level: list[tuple[str, Version]] = [] + for ( + top_level_edge + ) in self.ctx.dependency_graph.get_root_node().get_outgoing_edges( + req.name, RequirementType.TOP_LEVEL + ): + possible_versions_in_top_level.append( + ( + top_level_edge.destination_node.download_url, + top_level_edge.destination_node.version, + ) + ) + seen_version.add(str(top_level_edge.destination_node.version)) + + resolver_result = self._resolve_from_version_source( + possible_versions_in_top_level, req + ) + if resolver_result: + return resolver_result + + # only if there is nothing in top level reqs, resolve using history + possible_versions_from_graph: list[tuple[str, Version]] = [] + # check all nodes which have the same parent name irrespective of the parent's version + for parent_node in self.prev_graph.get_nodes_by_name( + parent_req.name if parent_req else None + ): + # if the edge matches the current req and type then it is a possible candidate + # filtering on type might not be necessary, but we are being safe here. This will + # for sure ensure that bootstrap takes the same route as it did in the previous one. + # If we don't filter by type then it might pick up a different version from a different + # type that should have appeared much later in the resolution process. + for edge in parent_node.get_outgoing_edges(req.name, req_type): + if ( + edge.destination_node.pre_built == pre_built + and str(edge.destination_node.version) not in seen_version + ): + possible_versions_from_graph.append( + ( + edge.destination_node.download_url, + edge.destination_node.version, + ) + ) + seen_version.add(str(edge.destination_node.version)) + + return self._resolve_from_version_source(possible_versions_from_graph, req) + + def _resolve_from_version_source( + self, + version_source: list[tuple[str, Version]], + req: Requirement, + ) -> tuple[str, Version] | None: + if not version_source: + return None + try: + provider = resolver.GenericProvider( + version_source=lambda x, y, z: version_source, + constraints=self.ctx.constraints, + ) + return resolver.resolve_from_provider(provider, req) + except Exception as err: + logger.debug( + f"{req.name}: could not resolve {req} from {version_source}: {err}" + ) + return None + + def _cleanup( + self, + req: Requirement, + sdist_root_dir: pathlib.Path | None, + build_env: build_environment.BuildEnvironment | None, + ) -> None: + if not self.ctx.cleanup: + return + + # Cleanup the source tree and build environment, leaving any other + # artifacts that were created. + if sdist_root_dir: + logger.debug(f"{req.name}: cleaning up source tree {sdist_root_dir}") + shutil.rmtree(sdist_root_dir) + logger.debug(f"{req.name}: cleaned up source tree {sdist_root_dir}") + if build_env: + logger.debug(f"{req.name}: cleaning up build environment {build_env.path}") + shutil.rmtree(build_env.path) + logger.debug(f"{req.name}: cleaned up build environment {build_env.path}") + + def _add_to_graph( + self, + req: Requirement, + req_type: RequirementType, + req_version: Version, + download_url: str, + ) -> None: + if req_type == RequirementType.TOP_LEVEL: + return + + _, parent_req, parent_version = self.why[-1] if self.why else (None, None, None) + pbi = self.ctx.package_build_info(req) + # Update the dependency graph after we determine that this requirement is + # useful but before we determine if it is redundant so that we capture all + # edges to use for building a valid constraints file. + self.ctx.dependency_graph.add_dependency( + parent_name=canonicalize_name(parent_req.name) if parent_req else None, + parent_version=parent_version, + req_type=req_type, + req=req, + req_version=req_version, + download_url=download_url, + pre_built=pbi.pre_built, + ) + self.ctx.write_to_graph_to_file() + + def _sort_requirements( + self, + requirements: typing.Iterable[Requirement], + ) -> typing.Iterable[Requirement]: + return sorted(requirements, key=operator.attrgetter("name")) + + def _resolved_key( + self, req: Requirement, version: Version + ) -> tuple[NormalizedName, tuple[str, ...], str]: + return (canonicalize_name(req.name), tuple(sorted(req.extras)), str(version)) + + def _mark_as_seen(self, req: Requirement, version: Version) -> None: + key = self._resolved_key(req, version) + logger.debug(f"{req.name}: remembering seen sdist {key}") + self._seen_requirements.add(key) + + def _has_been_seen(self, req: Requirement, version: Version) -> bool: + return self._resolved_key(req, version) in self._seen_requirements + + def _add_to_build_order( + self, + req: Requirement, + version: Version, + source_url: str, + source_url_type: str, + prebuilt: bool = False, + constraint: Requirement | None = None, + ) -> None: + # We only care if this version of this package has been built, + # and don't want to trigger building it twice. The "extras" + # value, included in the _resolved_key() output, can confuse + # that so we ignore itand build our own key using just the + # name and version. + key = (canonicalize_name(req.name), str(version)) + if key in self._build_requirements: + return + logger.info(f"{req.name}: adding {key} to build order") + self._build_requirements.add(key) + info = { + "req": str(req), + "constraint": str(constraint) if constraint else "", + "dist": canonicalize_name(req.name), + "version": str(version), + "prebuilt": prebuilt, + "source_url": source_url, + "source_url_type": source_url_type, + } + self._build_stack.append(info) + with open(self._build_order_filename, "w") as f: + # Set default=str because the why value includes + # Requirement and Version instances that can't be + # converted to JSON without help. + json.dump(self._build_stack, f, indent=2, default=str) diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index 4cf4c460..677eea7e 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -6,13 +6,13 @@ from packaging.requirements import Requirement from .. import ( + bootstrapper, clickext, context, dependency_graph, progress, requirements_file, resolver, - sdist, server, sources, wheels, @@ -133,14 +133,10 @@ def bootstrap( ) with progress.progress_context(total=len(to_build)) as progressbar: + bt = bootstrapper.Bootstrapper(wkctx, progressbar, prev_graph) + for req in to_build: - sdist.handle_requirement( - wkctx, - req, - req_type=requirements_file.RequirementType.TOP_LEVEL, - progressbar=progressbar, - prev_graph=prev_graph, - ) + bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL) progressbar.update() # If we put pre-built wheels in the downloads directory, we should diff --git a/src/fromager/context.py b/src/fromager/context.py index e98e6200..e19cb29c 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -1,9 +1,6 @@ -import collections -import json import logging import os import pathlib -import typing from urllib.parse import urlparse from packaging.requirements import Requirement @@ -71,26 +68,10 @@ def __init__( self.network_isolation = network_isolation self.settings_dir = settings_dir - self._build_order_filename = self.work_dir / "build-order.json" self._constraints_filename = self.work_dir / "constraints.txt" - # Push items onto the stack as we start to resolve their - # dependencies so at the end we have a list of items that need to - # be built in order. - self._build_stack: list[typing.Any] = [] - self._build_requirements: set[tuple[NormalizedName, str]] = set() - self.all_edges: BuildRequirements = collections.defaultdict(list) self.dependency_graph = dependency_graph.DependencyGraph() - # Track requirements we've seen before so we don't resolve the - # same dependencies over and over and so we can break cycles in - # the dependency list. The key is the requirements spec, rather - # than the package, in case we do have multiple rules for the same - # package. - self._seen_requirements: set[tuple[NormalizedName, tuple[str, ...], str]] = ( - set() - ) - @property def pip_wheel_server_args(self) -> list[str]: args = ["--index-url", self.wheel_server_url] @@ -105,54 +86,6 @@ def pip_constraint_args(self) -> list[str]: return [] return ["--constraint", os.fspath(self.input_constraints_file)] - def _resolved_key( - self, req: Requirement, version: Version - ) -> tuple[NormalizedName, tuple[str, ...], str]: - return (canonicalize_name(req.name), tuple(sorted(req.extras)), str(version)) - - def mark_as_seen(self, req: Requirement, version: Version) -> None: - key = self._resolved_key(req, version) - logger.debug(f"{req.name}: remembering seen sdist {key}") - self._seen_requirements.add(key) - - def has_been_seen(self, req: Requirement, version: Version) -> bool: - return self._resolved_key(req, version) in self._seen_requirements - - def add_to_build_order( - self, - req: Requirement, - version: Version, - source_url: str, - source_url_type: str, - prebuilt: bool = False, - constraint: Requirement | None = None, - ) -> None: - # We only care if this version of this package has been built, - # and don't want to trigger building it twice. The "extras" - # value, included in the _resolved_key() output, can confuse - # that so we ignore itand build our own key using just the - # name and version. - key = (canonicalize_name(req.name), str(version)) - if key in self._build_requirements: - return - logger.info(f"{req.name}: adding {key} to build order") - self._build_requirements.add(key) - info = { - "req": str(req), - "constraint": str(constraint) if constraint else "", - "dist": canonicalize_name(req.name), - "version": str(version), - "prebuilt": prebuilt, - "source_url": source_url, - "source_url_type": source_url_type, - } - self._build_stack.append(info) - with open(self._build_order_filename, "w") as f: - # Set default=str because the why value includes - # Requirement and Version instances that can't be - # converted to JSON without help. - json.dump(self._build_stack, f, indent=2, default=str) - def write_to_graph_to_file(self): with open(self.work_dir / "graph.json", "w") as f: self.dependency_graph.serialize(f) diff --git a/src/fromager/requirements_file.py b/src/fromager/requirements_file.py index f901f05b..afdc1d19 100644 --- a/src/fromager/requirements_file.py +++ b/src/fromager/requirements_file.py @@ -18,6 +18,12 @@ class RequirementType(StrEnum): BUILD_SDIST = "build-sdist" +class SourceType(StrEnum): + PREBUILT = "prebuilt" + SDIST = "sdist" + OVERRIDE = "override" + + def parse_requirements_file( req_file: pathlib.Path, ) -> typing.Iterable[str]: diff --git a/src/fromager/sdist.py b/src/fromager/sdist.py deleted file mode 100644 index 362f8568..00000000 --- a/src/fromager/sdist.py +++ /dev/null @@ -1,548 +0,0 @@ -from __future__ import annotations - -import logging -import operator -import pathlib -import shutil -import typing - -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name -from packaging.version import Version - -from . import ( - build_environment, - dependencies, - finders, - progress, - resolver, - server, - sources, - wheels, -) -from .dependency_graph import DependencyGraph -from .requirements_file import RequirementType - -if typing.TYPE_CHECKING: - from . import context - -logger = logging.getLogger(__name__) - - -def handle_requirement( - ctx: context.WorkContext, - req: Requirement, - req_type: RequirementType = RequirementType.TOP_LEVEL, - why: list[tuple[RequirementType, Requirement, Version]] | None = None, - progressbar: progress.Progressbar | None = None, - prev_graph: DependencyGraph | None = None, -) -> str: - if why is None: - why = [] - if progressbar is None: - progressbar = progress.Progressbar(None) - - logger.info( - f'{req.name}: {"*" * (len(why) + 1)} handling {req_type} requirement {req} {why}' - ) - - constraint = ctx.constraints.get_constraint(req.name) - if constraint: - logger.info( - f"{req.name}: incoming requirement {req} matches constraint {constraint}. Will apply both." - ) - - pbi = ctx.package_build_info(req) - pre_built = pbi.pre_built - _, parent_req, parent_version = why[-1] if why else (None, None, None) - - # Resolve the dependency and get either the pre-built wheel our - # the source code. - if not pre_built: - source_url, resolved_version = _resolve_source_with_history( - ctx, - prev_graph=prev_graph, - req=req, - req_type=req_type, - parent_req=parent_req, - ) - - source_filename = sources.download_source( - ctx=ctx, - req=req, - version=resolved_version, - download_url=source_url, - ) - source_url_type = sources.get_source_type(ctx, req) - else: - logger.info(f"{req.name}: {req_type} requirement {req} uses a pre-built wheel") - wheel_url, resolved_version = _resolve_prebuilt_with_history( - ctx, - prev_graph=prev_graph, - req=req, - req_type=req_type, - parent_req=parent_req, - ) - - wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) - # Remember that this is a prebuilt wheel, and where we got it. - source_url = wheel_url - source_url_type = "prebuilt" - # Add the wheel to the mirror so it is available to anything - # that needs to install it. We leave a copy in the prebuilt - # directory to make it easy to remove the wheel from the - # downloads directory before uploading to a proper package - # index. - dest_name = ctx.wheels_downloads / wheel_filename.name - if not dest_name.exists(): - logger.info(f"{req.name}: updating temporary mirror with pre-built wheel") - shutil.copy(wheel_filename, dest_name) - server.update_wheel_mirror(ctx) - unpack_dir = ctx.work_dir / f"{req.name}-{resolved_version}" - if not unpack_dir.exists(): - unpack_dir.mkdir() - - # we resolve top level requirements outside this function, so they already exist in the graph - if req_type != RequirementType.TOP_LEVEL: - # Update the dependency graph after we determine that this requirement is - # useful but before we determine if it is redundant so that we capture all - # edges to use for building a valid constraints file. - ctx.dependency_graph.add_dependency( - parent_name=canonicalize_name(parent_req.name) if parent_req else None, - parent_version=parent_version, - req_type=req_type, - req=req, - req_version=resolved_version, - download_url=source_url, - pre_built=pre_built, - ) - ctx.write_to_graph_to_file() - - # Avoid cyclic dependencies and redundant processing. - if ctx.has_been_seen(req, resolved_version): - logger.debug( - f"{req.name}: redundant {req_type} requirement {why} -> {req} resolves to {resolved_version}" - ) - return resolved_version - ctx.mark_as_seen(req, resolved_version) - - logger.info( - f"{req.name}: new {req_type} dependency {req} resolves to {resolved_version}" - ) - - # Build the dependency chain up to the point of this new - # requirement using a new list so we can avoid modifying the list - # we're given. - why = why[:] + [(req_type, req, resolved_version)] - - # for cleanup - build_env = None - sdist_root_dir = None - - if not pre_built: - sdist_root_dir = sources.prepare_source( - ctx=ctx, - req=req, - source_filename=source_filename, - version=resolved_version, - ) - unpack_dir = sdist_root_dir.parent - - build_system_dependencies = _handle_build_system_requirements( - ctx, - req, - why, - sdist_root_dir, - progressbar=progressbar, - prev_graph=prev_graph, - ) - - build_backend_dependencies = _handle_build_backend_requirements( - ctx, - req, - why, - sdist_root_dir, - progressbar=progressbar, - prev_graph=prev_graph, - ) - - build_sdist_dependencies = _handle_build_sdist_requirements( - ctx, - req, - why, - sdist_root_dir, - progressbar=progressbar, - prev_graph=prev_graph, - ) - - # Add the new package to the build order list before trying to - # build it so we have a record of the dependency even if the build - # fails. - ctx.add_to_build_order( - req=req, - version=resolved_version, - source_url=source_url, - source_url_type=source_url_type, - prebuilt=pre_built, - constraint=constraint, - ) - - if not pre_built: - # FIXME: This is a bit naive, but works for most wheels, including - # our more expensive ones, and there's not a way to know the - # actual name without doing most of the work to build the wheel. - wheel_filename = finders.find_wheel( - downloads_dir=ctx.wheels_downloads, - req=req, - dist_version=resolved_version, - build_tag=pbi.build_tag(resolved_version), - ) - if wheel_filename: - logger.info( - f"{req.name}: have wheel version {resolved_version}: {wheel_filename}" - ) - else: - logger.info( - f"{req.name}: preparing to build wheel for version {resolved_version}" - ) - build_env = build_environment.BuildEnvironment( - ctx, - sdist_root_dir.parent, - build_system_dependencies - | build_backend_dependencies - | build_sdist_dependencies, - ) - try: - find_sdist_result = finders.find_sdist( - ctx, ctx.sdists_builds, req, resolved_version - ) - if not find_sdist_result: - sources.build_sdist( - ctx=ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - else: - logger.info( - f"{req.name} have sdist version {resolved_version}: {find_sdist_result}" - ) - except Exception as err: - logger.warning( - f"{req.name}: failed to build source distribution: {err}" - ) - built_filename = wheels.build_wheel( - ctx=ctx, - req=req, - sdist_root_dir=sdist_root_dir, - version=resolved_version, - build_env=build_env, - ) - server.update_wheel_mirror(ctx) - # When we update the mirror, the built file moves to the - # downloads directory. - wheel_filename = ctx.wheels_downloads / built_filename.name - logger.info( - f"{req.name}: built wheel for version {resolved_version}: {wheel_filename}" - ) - - # Process installation dependencies for all wheels. - next_req_type = RequirementType.INSTALL - install_dependencies = dependencies.get_install_dependencies_of_wheel( - req, wheel_filename, unpack_dir - ) - progressbar.update_total(len(install_dependencies)) - - for dep in _sort_requirements(install_dependencies): - try: - handle_requirement( - ctx, - req=dep, - req_type=next_req_type, - why=why, - progressbar=progressbar, - prev_graph=prev_graph, - ) - except Exception as err: - raise ValueError( - f"could not handle {next_req_type} dependency {dep} for {why}" - ) from err - progressbar.update() - - # Cleanup the source tree and build environment, leaving any other - # artifacts that were created. - if ctx.cleanup: - if sdist_root_dir: - logger.debug(f"{req.name}: cleaning up source tree {sdist_root_dir}") - shutil.rmtree(sdist_root_dir) - logger.debug(f"{req.name}: cleaned up source tree {sdist_root_dir}") - if build_env: - logger.debug(f"{req.name}: cleaning up build environment {build_env.path}") - shutil.rmtree(build_env.path) - logger.debug(f"{req.name}: cleaned up build environment {build_env.path}") - - return resolved_version - - -def _resolve_from_graph( - ctx: context.WorkContext, - prev_graph: DependencyGraph | None, - parent_req: Requirement | None, - req: Requirement, - req_type: RequirementType, - pre_built: bool, -) -> tuple[str, Version] | None: - # we have already resolved top level reqs before bootstrapping - # so they should already be in the root node - if req_type == RequirementType.TOP_LEVEL: - for edge in ctx.dependency_graph.get_root_node().get_outgoing_edges( - req.name, RequirementType.TOP_LEVEL - ): - if edge.req == req: - return ( - edge.destination_node.download_url, - edge.destination_node.version, - ) - # this should never happen since we already resolved top level reqs and their - # resolution should be in the root nodes - raise ValueError( - f"{req.name}: {req} appears as a toplevel requirement but it's resolution does not exist in the root node of the graph" - ) - - if not prev_graph: - return None - - seen_version: set[str] = set() - - # first perform resolution using the top level reqs before looking at history - possible_versions_in_top_level: list[tuple[str, Version]] = [] - for top_level_edge in ctx.dependency_graph.get_root_node().get_outgoing_edges( - req.name, RequirementType.TOP_LEVEL - ): - possible_versions_in_top_level.append( - ( - top_level_edge.destination_node.download_url, - top_level_edge.destination_node.version, - ) - ) - seen_version.add(str(top_level_edge.destination_node.version)) - - resolver_result = _resolve_from_version_source( - ctx, possible_versions_in_top_level, req - ) - if resolver_result: - return resolver_result - - # only if there is nothing in top level reqs, resolve using history - possible_versions_from_graph: list[tuple[str, Version]] = [] - # check all nodes which have the same parent name irrespective of the parent's version - for parent_node in prev_graph.get_nodes_by_name( - parent_req.name if parent_req else None - ): - # if the edge matches the current req and type then it is a possible candidate - # filtering on type might not be necessary, but we are being safe here. This will - # for sure ensure that bootstrap takes the same route as it did in the previous one. - # If we don't filter by type then it might pick up a different version from a different - # type that should have appeared much later in the resolution process. - for edge in parent_node.get_outgoing_edges(req.name, req_type): - if ( - edge.destination_node.pre_built == pre_built - and str(edge.destination_node.version) not in seen_version - ): - possible_versions_from_graph.append( - ( - edge.destination_node.download_url, - edge.destination_node.version, - ) - ) - seen_version.add(str(edge.destination_node.version)) - - return _resolve_from_version_source(ctx, possible_versions_from_graph, req) - - -def _resolve_from_version_source( - ctx: context.WorkContext, - version_source: list[tuple[str, Version]], - req: Requirement, -) -> tuple[str, Version] | None: - if not version_source: - return None - try: - provider = resolver.GenericProvider( - version_source=lambda x, y, z: version_source, constraints=ctx.constraints - ) - return resolver.resolve_from_provider(provider, req) - except Exception as err: - logger.debug( - f"{req.name}: could not resolve {req} from {version_source}: {err}" - ) - return None - - -def _resolve_source_with_history( - ctx: context.WorkContext, - prev_graph: DependencyGraph | None, - req: Requirement, - req_type: RequirementType, - parent_req: Requirement | None, -) -> tuple[str, Version]: - cached_resolution = _resolve_from_graph( - ctx, - prev_graph, - parent_req=parent_req, - req=req, - req_type=req_type, - pre_built=False, - ) - if cached_resolution: - source_url, resolved_version = cached_resolution - logger.debug( - f"{req.name}: resolved from previous bootstrap to {resolved_version}" - ) - else: - source_url, resolved_version = sources.resolve_source( - ctx=ctx, req=req, sdist_server_url=resolver.PYPI_SERVER_URL - ) - return (source_url, resolved_version) - - -def _resolve_prebuilt_with_history( - ctx: context.WorkContext, - prev_graph: DependencyGraph | None, - req: Requirement, - req_type: RequirementType, - parent_req: Requirement | None, -) -> tuple[str, Version]: - cached_resolution = _resolve_from_graph( - ctx, - prev_graph, - parent_req=parent_req, - req=req, - req_type=req_type, - pre_built=True, - ) - - if cached_resolution: - wheel_url, resolved_version = cached_resolution - logger.debug( - f"{req.name}: resolved from previous bootstrap to {resolved_version}" - ) - else: - servers = wheels.get_wheel_server_urls(ctx, req) - wheel_url, resolved_version = wheels.resolve_prebuilt_wheel( - ctx=ctx, req=req, wheel_server_urls=servers - ) - return (wheel_url, resolved_version) - - -def _sort_requirements( - requirements: typing.Iterable[Requirement], -) -> typing.Iterable[Requirement]: - return sorted(requirements, key=operator.attrgetter("name")) - - -def _handle_build_system_requirements( - ctx: context.WorkContext, - req: Requirement, - why: list | None, - sdist_root_dir: pathlib.Path, - progressbar: progress.Progressbar, - prev_graph: DependencyGraph | None, -) -> set[Requirement]: - build_system_dependencies = dependencies.get_build_system_dependencies( - ctx=ctx, req=req, sdist_root_dir=sdist_root_dir - ) - progressbar.update_total(len(build_system_dependencies)) - - for dep in _sort_requirements(build_system_dependencies): - try: - resolved = handle_requirement( - ctx, - req=dep, - req_type=RequirementType.BUILD_SYSTEM, - why=why, - progressbar=progressbar, - prev_graph=prev_graph, - ) - except Exception as err: - raise ValueError( - f"could not handle build-system dependency {dep} for {why}" - ) from err - # We may need these dependencies installed in order to run build hooks - # Example: frozenlist build-system.requires includes expandvars because - # it is used by the packaging/pep517_backend/ build backend - build_environment.maybe_install( - ctx, dep, RequirementType.BUILD_SYSTEM, resolved - ) - progressbar.update() - return build_system_dependencies - - -def _handle_build_backend_requirements( - ctx: context.WorkContext, - req: Requirement, - why: list, - sdist_root_dir: pathlib.Path, - progressbar: progress.Progressbar, - prev_graph: DependencyGraph | None, -) -> set[Requirement]: - build_backend_dependencies = dependencies.get_build_backend_dependencies( - ctx=ctx, req=req, sdist_root_dir=sdist_root_dir - ) - progressbar.update_total(len(build_backend_dependencies)) - - for dep in _sort_requirements(build_backend_dependencies): - try: - resolved = handle_requirement( - ctx, - req=dep, - req_type=RequirementType.BUILD_BACKEND, - why=why, - progressbar=progressbar, - prev_graph=prev_graph, - ) - except Exception as err: - raise ValueError( - f"could not handle build-backend dependency {dep} for {why}" - ) from err - # Build backends are often used to package themselves, so in - # order to determine their dependencies they may need to be - # installed. - build_environment.maybe_install( - ctx, dep, RequirementType.BUILD_BACKEND, resolved - ) - progressbar.update() - return build_backend_dependencies - - -def _handle_build_sdist_requirements( - ctx: context.WorkContext, - req: Requirement, - why: list | None, - sdist_root_dir: pathlib.Path, - progressbar: progress.Progressbar, - prev_graph: DependencyGraph | None, -) -> set[Requirement]: - build_sdist_dependencies = dependencies.get_build_sdist_dependencies( - ctx=ctx, req=req, sdist_root_dir=sdist_root_dir - ) - progressbar.update_total(len(build_sdist_dependencies)) - - for dep in _sort_requirements(build_sdist_dependencies): - try: - resolved = handle_requirement( - ctx, - req=dep, - req_type=RequirementType.BUILD_SDIST, - why=why, - progressbar=progressbar, - prev_graph=prev_graph, - ) - except Exception as err: - raise ValueError( - f"could not handle build-sdist dependency {dep} for {why}" - ) from err - build_environment.maybe_install(ctx, dep, RequirementType.BUILD_SDIST, resolved) - progressbar.update() - return build_sdist_dependencies diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 3698b987..447868f4 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -20,6 +20,7 @@ external_commands, overrides, pyproject, + requirements_file, resolver, tarballs, vendor_rust, @@ -33,7 +34,7 @@ def get_source_type(ctx: context.WorkContext, req: Requirement) -> str: - source_type = "sdist" + source_type = requirements_file.SourceType.SDIST pbi = ctx.package_build_info(req) if ( overrides.find_override_method(req.name, "download_source") @@ -41,8 +42,8 @@ def get_source_type(ctx: context.WorkContext, req: Requirement) -> str: or overrides.find_override_method(req.name, "get_resolver_provider") or pbi.download_source_url(resolve_template=False) ): - source_type = "override" - return source_type + source_type = requirements_file.SourceType.OVERRIDE + return str(source_type) def download_source( diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index d2803d26..a4e12d02 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -242,7 +242,7 @@ def build_wheel( sdist_root_dir: pathlib.Path, version: Version, build_env: build_environment.BuildEnvironment, -) -> pathlib.Path | None: +) -> pathlib.Path: pbi = ctx.package_build_info(req) logger.info( f"{req.name}: building wheel for {req} in {sdist_root_dir} writing to {ctx.wheels_build}" diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py new file mode 100644 index 00000000..3f4ae56c --- /dev/null +++ b/tests/test_bootstrapper.py @@ -0,0 +1,363 @@ +import json + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +from fromager import bootstrapper +from fromager.context import WorkContext +from fromager.dependency_graph import DependencyGraph +from fromager.requirements_file import RequirementType + +old_graph = DependencyGraph() + +old_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("foo"), + req_version=Version("1.0.0"), +) + +old_graph.add_dependency( + parent_name=canonicalize_name("foo"), + parent_version=Version("1.0.0"), + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5"), + req_version=Version("7"), +) + +old_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("bar"), + req_version=Version("1.0.0"), +) + +old_graph.add_dependency( + parent_name=canonicalize_name("bar"), + parent_version=Version("1.0.0"), + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5,<7"), + req_version=Version("6"), +) + +old_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("blah"), + req_version=Version("1.0.0"), +) + +old_graph.add_dependency( + parent_name=canonicalize_name("blah"), + parent_version=Version("1.0.0"), + req_type=RequirementType.INSTALL, + req=Requirement("pbr==5"), + req_version=Version("5"), +) + + +def test_resolve_from_graph_no_changes(tmp_context: WorkContext): + bt = bootstrapper.Bootstrapper(tmp_context, None, old_graph) + bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] + + # resolving new dependency that doesn't exist in graph + assert ( + bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("xyz"), + pre_built=False, + ) + is None + ) + + # resolving pbr dependency of foo + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5"), + pre_built=False, + ) == ("", Version("7")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("bar"), Version("1.0.0"))] + # resolving pbr dependency of bar + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5,<7"), + pre_built=False, + ) == ("", Version("6")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("blah"), Version("1.0.0"))] + # resolving pbr dependency of blah + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr==5"), + pre_built=False, + ) == ("", Version("5")) + + +def test_resolve_from_graph_install_dep_upgrade(tmp_context: WorkContext): + bt = bootstrapper.Bootstrapper(tmp_context, None, old_graph) + + # simulating new bootstrap with a toplevel requirement of pbr==8 + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("pbr==8"), + req_version=Version("8"), + ) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] + # resolving pbr dependency of foo + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5"), + pre_built=False, + ) == ("", Version("8")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("bar"), Version("1.0.0"))] + # resolving pbr dependency of bar + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5,<7"), + pre_built=False, + ) == ("", Version("6")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("blah"), Version("1.0.0"))] + # resolving pbr dependency of blah + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr==5"), + pre_built=False, + ) == ("", Version("5")) + + +def test_resolve_from_graph_install_dep_downgrade(tmp_context: WorkContext): + bt = bootstrapper.Bootstrapper(tmp_context, None, old_graph) + + # simulating new bootstrap with a toplevel requirement of pbr<=6 + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("pbr<=6"), + req_version=Version("6"), + ) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] + # resolving pbr dependency of foo + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5"), + pre_built=False, + ) == ("", Version("6")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("bar"), Version("1.0.0"))] + # resolving pbr dependency of bar + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5,<7"), + pre_built=False, + ) == ("", Version("6")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("blah"), Version("1.0.0"))] + # resolving pbr dependency of blah + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr==5"), + pre_built=False, + ) == ("", Version("5")) + + +def test_resolve_from_graph_toplevel_dep(tmp_context: WorkContext): + bt = bootstrapper.Bootstrapper(tmp_context, None, old_graph) + + # simulating new bootstrap with a toplevel requirement for foo + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("foo==2"), + req_version=Version("2"), + ) + + # simulating new bootstrap with a toplevel requirement of bar (no change) + tmp_context.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=Requirement("bar"), + req_version=Version("1.0.0"), + ) + + bt.why = [] + # resolving foo + assert bt._resolve_from_graph( + req_type=RequirementType.TOP_LEVEL, + req=Requirement("foo==2"), + pre_built=False, + ) == ("", Version("2")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("2"))] + # resolving pbr dependency of foo even if foo version changed + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5"), + pre_built=False, + ) == ("", Version("7")) + + bt.why = [] + # resolving bar + assert bt._resolve_from_graph( + req_type=RequirementType.TOP_LEVEL, + req=Requirement("bar"), + pre_built=False, + ) == ("", Version("1.0.0")) + + bt.why = [(RequirementType.TOP_LEVEL, Requirement("bar"), Version("1.0.0"))] + # resolving pbr dependency of bar + assert bt._resolve_from_graph( + req_type=RequirementType.INSTALL, + req=Requirement("pbr>=5,<7"), + pre_built=False, + ) == ("", Version("6")) + + +def test_seen(tmp_context): + bt = bootstrapper.Bootstrapper(tmp_context) + req = Requirement("testdist") + version = "1.2" + assert not bt._has_been_seen(req, version) + bt._mark_as_seen(req, version) + assert bt._has_been_seen(req, version) + + +def test_seen_extras(tmp_context): + req1 = Requirement("testdist") + req2 = Requirement("testdist[extra]") + version = "1.2" + bt = bootstrapper.Bootstrapper(tmp_context) + assert not bt._has_been_seen(req1, version) + bt._mark_as_seen(req1, version) + assert bt._has_been_seen(req1, version) + assert not bt._has_been_seen(req2, version) + bt._mark_as_seen(req2, version) + assert bt._has_been_seen(req1, version) + assert bt._has_been_seen(req2, version) + + +def test_seen_name_canonicalization(tmp_context): + req = Requirement("flit_core") + version = "1.2" + bt = bootstrapper.Bootstrapper(tmp_context) + assert not bt._has_been_seen(req, version) + bt._mark_as_seen(req, version) + assert bt._has_been_seen(req, version) + + +def test_build_order(tmp_context): + bt = bootstrapper.Bootstrapper(tmp_context) + bt._add_to_build_order( + req=Requirement("buildme>1.0"), + version="6.0", + source_url="url", + source_url_type="sdist", + ) + bt._add_to_build_order( + req=Requirement("testdist>1.0"), + version="1.2", + source_url="url", + source_url_type="sdist", + ) + contents_str = bt._build_order_filename.read_text() + contents = json.loads(contents_str) + expected = [ + { + "req": "buildme>1.0", + "dist": "buildme", + "version": "6.0", + "prebuilt": False, + "source_url": "url", + "source_url_type": "sdist", + "constraint": "", + }, + { + "req": "testdist>1.0", + "dist": "testdist", + "version": "1.2", + "prebuilt": False, + "source_url": "url", + "source_url_type": "sdist", + "constraint": "", + }, + ] + assert expected == contents + + +def test_build_order_repeats(tmp_context): + bt = bootstrapper.Bootstrapper(tmp_context) + bt._add_to_build_order( + Requirement("buildme>1.0"), + "6.0", + "url", + "sdist", + ) + bt._add_to_build_order( + Requirement("buildme>1.0"), + "6.0", + "url", + "sdist", + ) + bt._add_to_build_order( + Requirement("buildme[extra]>1.0"), + "6.0", + "url", + "sdist", + ) + contents_str = bt._build_order_filename.read_text() + contents = json.loads(contents_str) + expected = [ + { + "req": "buildme>1.0", + "dist": "buildme", + "version": "6.0", + "prebuilt": False, + "source_url": "url", + "source_url_type": "sdist", + "constraint": "", + }, + ] + assert expected == contents + + +def test_build_order_name_canonicalization(tmp_context): + bt = bootstrapper.Bootstrapper(tmp_context) + bt._add_to_build_order( + Requirement("flit-core>1.0"), + "3.9.0", + "url", + "sdist", + ) + bt._add_to_build_order( + Requirement("flit_core>1.0"), + "3.9.0", + "url", + "sdist", + ) + contents_str = bt._build_order_filename.read_text() + contents = json.loads(contents_str) + expected = [ + { + "req": "flit-core>1.0", + "dist": "flit-core", + "version": "3.9.0", + "prebuilt": False, + "source_url": "url", + "source_url_type": "sdist", + "constraint": "", + }, + ] + assert expected == contents diff --git a/tests/test_context.py b/tests/test_context.py index 435ed31e..f9b037b1 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,142 +1,8 @@ -import json import os -from packaging.requirements import Requirement - from fromager import context -def test_seen(tmp_context): - req = Requirement("testdist") - version = "1.2" - assert not tmp_context.has_been_seen(req, version) - tmp_context.mark_as_seen(req, version) - assert tmp_context.has_been_seen(req, version) - - -def test_seen_extras(tmp_context): - req1 = Requirement("testdist") - req2 = Requirement("testdist[extra]") - version = "1.2" - assert not tmp_context.has_been_seen(req1, version) - tmp_context.mark_as_seen(req1, version) - assert tmp_context.has_been_seen(req1, version) - assert not tmp_context.has_been_seen(req2, version) - tmp_context.mark_as_seen(req2, version) - assert tmp_context.has_been_seen(req1, version) - assert tmp_context.has_been_seen(req2, version) - - -def test_seen_name_canonicalization(tmp_context): - req = Requirement("flit_core") - version = "1.2" - assert not tmp_context.has_been_seen(req, version) - tmp_context.mark_as_seen(req, version) - assert tmp_context.has_been_seen(req, version) - - -def test_build_order(tmp_context): - tmp_context.add_to_build_order( - req=Requirement("buildme>1.0"), - version="6.0", - source_url="url", - source_url_type="sdist", - ) - tmp_context.add_to_build_order( - req=Requirement("testdist>1.0"), - version="1.2", - source_url="url", - source_url_type="sdist", - ) - contents_str = tmp_context._build_order_filename.read_text() - contents = json.loads(contents_str) - expected = [ - { - "req": "buildme>1.0", - "dist": "buildme", - "version": "6.0", - "prebuilt": False, - "source_url": "url", - "source_url_type": "sdist", - "constraint": "", - }, - { - "req": "testdist>1.0", - "dist": "testdist", - "version": "1.2", - "prebuilt": False, - "source_url": "url", - "source_url_type": "sdist", - "constraint": "", - }, - ] - assert expected == contents - - -def test_build_order_repeats(tmp_context): - tmp_context.add_to_build_order( - Requirement("buildme>1.0"), - "6.0", - "url", - "sdist", - ) - tmp_context.add_to_build_order( - Requirement("buildme>1.0"), - "6.0", - "url", - "sdist", - ) - tmp_context.add_to_build_order( - Requirement("buildme[extra]>1.0"), - "6.0", - "url", - "sdist", - ) - contents_str = tmp_context._build_order_filename.read_text() - contents = json.loads(contents_str) - expected = [ - { - "req": "buildme>1.0", - "dist": "buildme", - "version": "6.0", - "prebuilt": False, - "source_url": "url", - "source_url_type": "sdist", - "constraint": "", - }, - ] - assert expected == contents - - -def test_build_order_name_canonicalization(tmp_context): - tmp_context.add_to_build_order( - Requirement("flit-core>1.0"), - "3.9.0", - "url", - "sdist", - ) - tmp_context.add_to_build_order( - Requirement("flit_core>1.0"), - "3.9.0", - "url", - "sdist", - ) - contents_str = tmp_context._build_order_filename.read_text() - contents = json.loads(contents_str) - expected = [ - { - "req": "flit-core>1.0", - "dist": "flit-core", - "version": "3.9.0", - "prebuilt": False, - "source_url": "url", - "source_url_type": "sdist", - "constraint": "", - }, - ] - assert expected == contents - - def test_pip_constraints_args(tmp_path): constraints_file = tmp_path / "constraints.txt" constraints_file.write_text("\n") # the file has to exist diff --git a/tests/test_sdist.py b/tests/test_sdist.py deleted file mode 100644 index 9ae23283..00000000 --- a/tests/test_sdist.py +++ /dev/null @@ -1,245 +0,0 @@ -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name -from packaging.version import Version - -from fromager import sdist -from fromager.context import WorkContext -from fromager.dependency_graph import DependencyGraph -from fromager.requirements_file import RequirementType - -old_graph = DependencyGraph() - -old_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("foo"), - req_version=Version("1.0.0"), -) - -old_graph.add_dependency( - parent_name=canonicalize_name("foo"), - parent_version=Version("1.0.0"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5"), - req_version=Version("7"), -) - -old_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("bar"), - req_version=Version("1.0.0"), -) - -old_graph.add_dependency( - parent_name=canonicalize_name("bar"), - parent_version=Version("1.0.0"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5,<7"), - req_version=Version("6"), -) - -old_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("blah"), - req_version=Version("1.0.0"), -) - -old_graph.add_dependency( - parent_name=canonicalize_name("blah"), - parent_version=Version("1.0.0"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr==5"), - req_version=Version("5"), -) - - -def test_resolve_from_graph_no_changes(tmp_context: WorkContext): - # resolving new dependency that doesn't exist in graph - assert ( - sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("foo"), - req_type=RequirementType.INSTALL, - req=Requirement("xyz"), - pre_built=False, - ) - is None - ) - - # resolving pbr dependency of foo - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("foo"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5"), - pre_built=False, - ) == ("", Version("7")) - - # resolving pbr dependency of bar - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("bar"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5,<7"), - pre_built=False, - ) == ("", Version("6")) - - # resolving pbr dependency of blah - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("blah"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr==5"), - pre_built=False, - ) == ("", Version("5")) - - -def test_resolve_from_graph_install_dep_upgrade(tmp_context: WorkContext): - # simulating new bootstrap with a toplevel requirement of pbr==8 - tmp_context.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("pbr==8"), - req_version=Version("8"), - ) - - # resolving pbr dependency of foo - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("foo"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5"), - pre_built=False, - ) == ("", Version("8")) - - # resolving pbr dependency of bar - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("bar"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5,<7"), - pre_built=False, - ) == ("", Version("6")) - - # resolving pbr dependency of blah - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("blah"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr==5"), - pre_built=False, - ) == ("", Version("5")) - - -def test_resolve_from_graph_install_dep_downgrade(tmp_context: WorkContext): - # simulating new bootstrap with a toplevel requirement of pbr<=6 - tmp_context.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("pbr<=6"), - req_version=Version("6"), - ) - - # resolving pbr dependency of foo - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("foo"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5"), - pre_built=False, - ) == ("", Version("6")) - - # resolving pbr dependency of bar - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("bar"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5,<7"), - pre_built=False, - ) == ("", Version("6")) - - # resolving pbr dependency of blah - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("blah"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr==5"), - pre_built=False, - ) == ("", Version("5")) - - -def test_resolve_from_graph_toplevel_dep(tmp_context: WorkContext): - # simulating new bootstrap with a toplevel requirement for foo - tmp_context.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("foo==2"), - req_version=Version("2"), - ) - - # simulating new bootstrap with a toplevel requirement of bar (no change) - tmp_context.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("bar"), - req_version=Version("1.0.0"), - ) - - # resolving foo - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("foo==2"), - pre_built=False, - ) == ("", Version("2")) - - # resolving pbr dependency of foo even if foo version changed - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("foo"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5"), - pre_built=False, - ) == ("", Version("7")) - - # resolving bar - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=None, - req_type=RequirementType.TOP_LEVEL, - req=Requirement("bar"), - pre_built=False, - ) == ("", Version("1.0.0")) - - # resolving pbr dependency of bar - assert sdist._resolve_from_graph( - tmp_context, - prev_graph=old_graph, - parent_req=Requirement("bar"), - req_type=RequirementType.INSTALL, - req=Requirement("pbr>=5,<7"), - pre_built=False, - ) == ("", Version("6"))