Initial cargo support

.github/workflows/gating.yaml

@@ -27,7 +27,7 @@ jobs:
         apt-get update && apt-get install --no-install-recommends --no-install-suggests -y git
         python3 -m pip install --upgrade pip
         pip install nox
-        pip install tomli
+        pip install tomlkit
 
     - uses: actions/checkout@v4
       with:
@@ -69,7 +69,7 @@ jobs:
         apt-get update && apt-get install --no-install-recommends --no-install-suggests -y git
         python3 -m pip install --upgrade pip
         pip install nox
-        pip install tomli
+        pip install tomlkit
 
     - uses: actions/checkout@v4
       with:
@@ -105,7 +105,7 @@ jobs:
           python3 -m venv /var/tmp/venv
           /var/tmp/venv/bin/pip3 install --upgrade pip
           /var/tmp/venv/bin/pip3 install nox
-          /var/tmp/venv/bin/pip3 install tomli
+          /var/tmp/venv/bin/pip3 install tomlkit
 
       - name: add checkout action...
         uses: actions/checkout@v4

Dockerfile

@@ -28,6 +28,8 @@ RUN dnf -y install \
     --setopt install_weak_deps=0 \
     --nodocs \
     gcc \
+    # not a build dependency, but we copy the binary to the final image
+    cargo \
     python3-devel \
     python3-pip \
     python3-setuptools \
@@ -49,6 +51,7 @@ COPY --from=golang_120 /usr/local/go /usr/local/go/go1.20
 COPY --from=golang_121 /usr/local/go /usr/local/go/go1.21
 COPY --from=node /usr/local/lib/node_modules/corepack /usr/local/lib/corepack
 COPY --from=node /usr/local/bin/node /usr/local/bin/node
+COPY --from=builder /usr/bin/cargo /usr/bin/cargo
 COPY --from=builder /venv /venv
 
 # link corepack, yarn, and go to standard PATH location

cachi2/core/models/input.py

@@ -52,7 +52,7 @@ def show_error(error: "ErrorDict") -> str:
 
 
 # Supported package managers
-PackageManagerType = Literal["bundler", "generic", "gomod", "npm", "pip", "rpm", "yarn"]
+PackageManagerType = Literal["bundler", "cargo", "generic", "gomod", "npm", "pip", "rpm", "yarn"]
 
 Flag = Literal[
     "cgo-disable", "dev-package-managers", "force-gomod-tidy", "gomod-vendor", "gomod-vendor-check"
@@ -116,6 +116,12 @@ class BundlerPackageInput(_PackageInputBase):
     allow_binary: bool = False
 
 
+class CargoPackageInput(_PackageInputBase):
+    """Accepted input for a cargo package."""
+
+    type: Literal["cargo"]
+
+
 class GenericPackageInput(_PackageInputBase):
     """Accepted input for generic package."""
 
@@ -227,6 +233,7 @@ class YarnPackageInput(_PackageInputBase):
 PackageInput = Annotated[
     Union[
         BundlerPackageInput,
+        CargoPackageInput,
         GenericPackageInput,
         GomodPackageInput,
         NpmPackageInput,
@@ -303,6 +310,11 @@ def bundler_packages(self) -> list[BundlerPackageInput]:
         """Get the bundler packages specified for this request."""
         return self._packages_by_type(BundlerPackageInput)
 
+    @property
+    def cargo_packages(self) -> list[CargoPackageInput]:
+        """Get the cargo packages specified for this request."""
+        return self._packages_by_type(CargoPackageInput)
+
     @property
     def generic_packages(self) -> list[GenericPackageInput]:
         """Get the generic packages specified for this request."""

cachi2/core/package_managers/cargo/__init__.py

@@ -0,0 +1,3 @@
+from cachi2.core.package_managers.cargo.main import fetch_cargo_source
+
+__all__ = ["fetch_cargo_source"]

cachi2/core/package_managers/cargo/main.py

@@ -0,0 +1,59 @@
+import logging
+import textwrap
+
+import tomlkit
+from tomlkit.toml_file import TOMLFile
+
+from cachi2.core.models.input import Request
+from cachi2.core.models.output import Component, EnvironmentVariable, ProjectFile, RequestOutput
+from cachi2.core.rooted_path import RootedPath
+from cachi2.core.utils import run_cmd
+
+log = logging.getLogger(__name__)
+
+
+def fetch_cargo_source(request: Request) -> RequestOutput:
+    """Fetch the source code for all cargo packages specified in a request."""
+    components: list[Component] = []
+    environment_variables: list[EnvironmentVariable] = []
+    project_files: list[ProjectFile] = []
+
+    for package in request.cargo_packages:
+        package_dir = request.source_dir.join_within_root(package.path)
+        components.extend(_resolve_cargo_package(package_dir, request.output_dir))
+        # cargo allows to specify configuration per-package
+        # https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure
+        project_files.append(_use_vendored_sources(package_dir))
+
+    return RequestOutput.from_obj_list(components, environment_variables, project_files)
+
+
+def _resolve_cargo_package(package_dir: RootedPath, output_dir: RootedPath) -> list[Component]:
+    """Resolve a single cargo package."""
+    vendor_dir = output_dir.join_within_root("deps/cargo")
+    cmd = ["cargo", "vendor", "--locked", str(vendor_dir)]
+    log.info("Fetching cargo dependencies at %s", package_dir)
+    run_cmd(cmd=cmd, params={"cwd": package_dir})
+    return []
+
+
+def _use_vendored_sources(package_dir: RootedPath) -> ProjectFile:
+    """Make sure cargo will use the vendored sources when building the project."""
+    cargo_config = package_dir.join_within_root(".cargo/config.toml")
+    cargo_config.path.parent.mkdir(parents=True, exist_ok=True)
+    cargo_config.path.touch(exist_ok=True)
+
+    template = """
+    [source.crates-io]
+    replace-with = "vendored-sources"
+
+    [source.vendored-sources]
+    directory = "${output_dir}/deps/cargo"
+    """
+
+    toml_file = TOMLFile(cargo_config)
+    original_content = toml_file.read()
+    original_content.update(tomlkit.parse(textwrap.dedent(template)))
+    toml_file.write(original_content)
+
+    return ProjectFile(abspath=cargo_config.path, template=template)

cachi2/core/package_managers/pip.py

@@ -27,7 +27,7 @@
     cast,
 )
 
-import tomli
+import tomlkit
 from packageurl import PackageURL
 
 from cachi2.core.rooted_path import RootedPath
@@ -462,11 +462,11 @@ def get_version(self) -> Optional[str]:
     @functools.cached_property
     def _parsed_toml(self) -> dict[str, Any]:
         try:
-            with open(self._setup_file, "rb") as f:
-                log.debug("Parsing pyproject.toml at %r", str(self._setup_file))
-                data = tomli.load(f)
-                return data
-        except tomli.TOMLDecodeError as e:
+            log.debug("Parsing pyproject.toml at %r", str(self._setup_file))
+            return tomlkit.parse(self._setup_file.path.read_text())
+        # no exceptions are exposed in the public API reference for tomlkit
+        # check https://github.com/python-poetry/tomlkit/issues/399
+        except Exception as e:
             log.error("Failed to parse pyproject.toml: %s", e)
             return {}
 

cachi2/core/resolver.py

@@ -5,7 +5,7 @@
 from cachi2.core.errors import UnsupportedFeature
 from cachi2.core.models.input import PackageManagerType, Request
 from cachi2.core.models.output import RequestOutput
-from cachi2.core.package_managers import bundler, generic, gomod, metayarn, npm, pip, rpm
+from cachi2.core.package_managers import bundler, cargo, generic, gomod, metayarn, npm, pip, rpm
 from cachi2.core.package_managers.utils import merge_outputs
 from cachi2.core.rooted_path import RootedPath
 from cachi2.core.utils import copy_directory
@@ -24,6 +24,7 @@
 # This is where we put package managers currently under development in order to
 # invoke them via CLI
 _dev_package_managers: dict[PackageManagerType, Handler] = {
+    "cargo": cargo.fetch_cargo_source,
     "rpm": rpm.fetch_rpm_source,
 }
 

noxfile.py

@@ -16,9 +16,10 @@
 
 import os
 from pathlib import Path
+from typing import Any
 
 import nox
-import tomli
+import tomlkit
 from nox.sessions import Session
 
 # default sessions to run (sorted alphabetically)
@@ -39,7 +40,7 @@ def install_requirements(session: Session) -> None:
 def parse_supported_python_versions() -> list[str]:
     """Parse supported Python versions from pyproject.toml."""
     pyproject_path = Path("pyproject.toml")
-    pyproject = tomli.loads(pyproject_path.read_text())
+    pyproject: dict[str, Any] = tomlkit.parse(pyproject_path.read_text())
     classifiers: list[str] = pyproject["project"]["classifiers"]
 
     result = []

pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
   "pyyaml",
   "requests",
   "semver",
-  "tomli",
+  "tomlkit",
   "typer",
   "createrepo-c",
   "packageurl-python"

requirements-extras.txt

@@ -1170,13 +1170,16 @@ tomli==2.2.1 \
     --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7
     # via
     #   black
-    #   cachi2 (pyproject.toml)
     #   coverage
     #   flake8-pyproject
     #   mypy
     #   nox
     #   pytest
     #   pytest-env
+tomlkit==0.13.2 \
+    --hash=sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde \
+    --hash=sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79
+    # via cachi2 (pyproject.toml)
 typer==0.15.1 \
     --hash=sha256:7994fb7b8155b64d3402518560648446072864beefd44aa2dc36972a5972e847 \
     --hash=sha256:a0588c0a7fa68a1978a069818657778f86abe6ff5ea6abf472f940a08bfe4f0a

requirements.txt

@@ -752,39 +752,9 @@ soupsieve==2.6 \
     --hash=sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb \
     --hash=sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9
     # via beautifulsoup4
-tomli==2.2.1 \
-    --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \
-    --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \
-    --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \
-    --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \
-    --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \
-    --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \
-    --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \
-    --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \
-    --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \
-    --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \
-    --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \
-    --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \
-    --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \
-    --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \
-    --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \
-    --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \
-    --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \
-    --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \
-    --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \
-    --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \
-    --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \
-    --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \
-    --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \
-    --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \
-    --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \
-    --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \
-    --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \
-    --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \
-    --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \
-    --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \
-    --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \
-    --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7
+tomlkit==0.13.2 \
+    --hash=sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde \
+    --hash=sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79
     # via cachi2 (pyproject.toml)
 typer==0.15.1 \
     --hash=sha256:7994fb7b8155b64d3402518560648446072864beefd44aa2dc36972a5972e847 \

tests/unit/models/test_input.py

@@ -160,7 +160,7 @@ def test_valid_packages(self, input_data: dict[str, Any], expect_data: dict[str,
             ),
             pytest.param(
                 {"type": "go-package"},
-                r"Input tag 'go-package' found using 'type' does not match any of the expected tags: 'bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
+                r"Input tag 'go-package' found using 'type' does not match any of the expected tags: 'bundler', 'cargo', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
                 id="incorrect_type_tag",
             ),
             pytest.param(

tests/unit/package_managers/cargo/test_main.py

@@ -0,0 +1,42 @@
+import tomlkit
+
+from cachi2.core.package_managers.cargo.main import _use_vendored_sources
+from cachi2.core.rooted_path import RootedPath
+
+
+def test_use_vendored_sources_creates_config_file(rooted_tmp_path: RootedPath) -> None:
+    config_path = rooted_tmp_path.join_within_root(".cargo/config.toml")
+    config_path.path.parent.mkdir(parents=True)
+
+    project_file = _use_vendored_sources(rooted_tmp_path)
+
+    assert config_path.path.exists()
+    assert project_file.abspath == config_path.path
+
+    config_content = tomlkit.parse(config_path.path.read_text())
+
+    assert config_content.get("source") == {
+        "crates-io": {"replace-with": "vendored-sources"},
+        "vendored-sources": {"directory": "${output_dir}/deps/cargo"},
+    }
+
+
+def test_use_vendored_sources_preserves_existing_config(rooted_tmp_path: RootedPath) -> None:
+    config_path = rooted_tmp_path.join_within_root(".cargo/config.toml")
+    config_path.path.parent.mkdir(parents=True)
+
+    some_toml = """
+    [build]
+    rustflags = ["-C", "target-cpu=native"]
+    """
+    config_path.path.write_text(some_toml)
+
+    _use_vendored_sources(rooted_tmp_path)
+
+    config_content = tomlkit.parse(config_path.path.read_text())
+
+    assert config_content.get("build") == {"rustflags": ["-C", "target-cpu=native"]}
+    assert config_content.get("source") == {
+        "crates-io": {"replace-with": "vendored-sources"},
+        "vendored-sources": {"directory": "${output_dir}/deps/cargo"},
+    }

tests/unit/test_cli.py

@@ -360,23 +360,23 @@ def test_specify_packages(
                 [
                     "Error: InvalidInput: 1 validation error for user input",
                     "packages -> 0",
-                    "Input tag 'idk' found using 'type' does not match any of the expected tags: 'bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
+                    "Input tag 'idk' found using 'type' does not match any of the expected tags: 'bundler', 'cargo', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
                 ],
             ),
             (
                 '[{"type": "idk"}]',
                 [
                     "Error: InvalidInput: 1 validation error for user input",
                     "packages -> 0",
-                    "Input tag 'idk' found using 'type' does not match any of the expected tags: 'bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
+                    "Input tag 'idk' found using 'type' does not match any of the expected tags: 'bundler', 'cargo', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
                 ],
             ),
             (
                 '{"packages": [{"type": "idk"}]}',
                 [
                     "Error: InvalidInput: 1 validation error for user input",
                     "packages -> 0",
-                    "Input tag 'idk' found using 'type' does not match any of the expected tags: 'bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
+                    "Input tag 'idk' found using 'type' does not match any of the expected tags: 'bundler', 'cargo', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn'",
                 ],
             ),
             # Missing package type