Allow running doctests via rust.doctest()

docs/markdown/Rust-module.md

@@ -24,6 +24,8 @@ like Meson, rather than Meson work more like rust.
 rustmod.test(name, target, ...)
 ```
 
+*Since 1.8.0*
+
 This function creates a new rust unittest target from an existing rust
 based target, which may be a library or executable. It does this by
 copying the sources and arguments passed to the original target and
@@ -36,11 +38,38 @@ It also takes the following keyword arguments:
 
 - `dependencies`: a list of test-only Dependencies
 - `link_with`: a list of additional build Targets to link with (*since 1.2.0*)
+- `link_whole`: a list of additional build Targets to link with in their entirety (*since 1.8.0*)
 - `rust_args`: a list of extra arguments passed to the Rust compiler (*since 1.2.0*)
 
 This function  also accepts all of the keyword arguments accepted by the
 [[test]] function except `protocol`, it will set that automatically.
 
+### doctest()
+
+```meson
+rustmod.doctest(name, target, ...)
+```
+
+This function creates a new `test()` target from an existing rust
+based library target. The test will use `rustdoc` to extract and run
+the doctests that are included in `target`'s sources.
+
+This function takes two positional arguments, the first is the name of the
+test and the second is the library or executable that is the rust based target.
+It also takes the following keyword arguments:
+
+- `dependencies`: a list of test-only Dependencies
+- `link_with`: a list of additional build Targets to link with
+- `link_whole`: a list of additional build Targets to link with in their entirety
+- `rust_args`: a list of extra arguments passed to the Rust compiler
+
+The target is linked automatically into the doctests.
+
+This function  also accepts all of the keyword arguments accepted by the
+[[test]] function except `protocol`, it will set that automatically.
+However, arguments are limited to strings that do not contain spaces
+due to limitations of `rustdoc`.
+
 ### bindgen()
 
 This function wraps bindgen to simplify creating rust bindings around C

docs/markdown/snippets/rust-test-link-whole.md

@@ -0,0 +1,4 @@
+## `rust.test` now supports `link_whole`
+
+The `test` function in the `rust` module now supports the `link_whole`
+keyword argument in addition to `link_with` and `dependencies`.

mesonbuild/backend/backends.py

@@ -2079,9 +2079,8 @@ def compiler_to_generator(self, target: build.BuildTarget,
         Some backends don't support custom compilers. This is a convenience
         method to convert a Compiler to a Generator.
         '''
-        exelist = compiler.get_exelist()
-        exe = programs.ExternalProgram(exelist[0])
-        args = exelist[1:]
+        exe = programs.ExternalProgram(compiler.get_exe())
+        args = compiler.get_exe_args()
         commands = self.compiler_to_generator_args(target, compiler)
         generator = build.Generator(exe, args + commands.to_native(),
                                     [output_templ], depfile='@[email protected]',

mesonbuild/backend/ninjabackend.py

@@ -45,6 +45,7 @@
     from ..linkers.linkers import DynamicLinker, StaticLinker
     from ..compilers.cs import CsCompiler
     from ..compilers.fortran import FortranCompiler
+    from ..compilers.rust import RustCompiler
     from ..mesonlib import FileOrString
     from .backends import TargetIntrospectionData
 
@@ -1928,25 +1929,12 @@ def _get_rust_dependency_name(self, target: build.BuildTarget, dependency: LibTy
         # in Rust
         return target.rust_dependency_map.get(dependency.name, dependency.name).replace('-', '_')
 
-    def generate_rust_target(self, target: build.BuildTarget) -> None:
-        rustc = target.compilers['rust']
+    def generate_rust_sources(self, target: build.BuildTarget) -> T.Tuple[T.List[str], str]:
+        orderdeps: T.List[str] = []
+
         # Rust compiler takes only the main file as input and
         # figures out what other files are needed via import
         # statements and magic.
-        base_proxy = target.get_options()
-        args = rustc.compiler_args()
-        # Compiler args for compiling this target
-        args += compilers.get_base_compile_args(base_proxy, rustc, self.environment)
-        self.generate_generator_list_rules(target)
-
-        # dependencies need to cause a relink, they're not just for ordering
-        deps: T.List[str] = []
-
-        # Dependencies for rust-project.json
-        project_deps: T.List[RustDep] = []
-
-        orderdeps: T.List[str] = []
-
         main_rust_file = None
         if target.structured_sources:
             if target.structured_sources.needs_copy():
@@ -1976,41 +1964,47 @@ def generate_rust_target(self, target: build.BuildTarget) -> None:
             orderdeps.extend(_ods)
 
         for i in target.get_sources():
-            if not rustc.can_compile(i):
-                raise InvalidArguments(f'Rust target {target.get_basename()} contains a non-rust source file.')
             if main_rust_file is None:
                 main_rust_file = i.rel_to_builddir(self.build_to_src)
         for g in target.get_generated_sources():
             for i in g.get_outputs():
-                if not rustc.can_compile(i):
-                    raise InvalidArguments(f'Rust target {target.get_basename()} contains a non-rust source file.')
                 if isinstance(g, GeneratedList):
                     fname = os.path.join(self.get_target_private_dir(target), i)
                 else:
                     fname = os.path.join(g.get_subdir(), i)
                 if main_rust_file is None:
                     main_rust_file = fname
                 orderdeps.append(fname)
-        if main_rust_file is None:
-            raise RuntimeError('A Rust target has no Rust sources. This is weird. Also a bug. Please report')
+
+        return orderdeps, main_rust_file
+
+    def get_rust_compiler_args(self, target: build.BuildTarget, rustc: Compiler, src_crate_type: str,
+                               depfile: T.Optional[str] = None) -> T.List[str]:
+        base_proxy = target.get_options()
+        # Compiler args for compiling this target
+        args = compilers.get_base_compile_args(base_proxy, rustc, self.environment)
+
         target_name = self.get_target_filename(target)
-        args.extend(['--crate-type', target.rust_crate_type])
+        args.extend(['--crate-type', src_crate_type])
 
         # If we're dynamically linking, add those arguments
-        #
-        # Rust is super annoying, calling -C link-arg foo does not work, it has
-        # to be -C link-arg=foo
         if target.rust_crate_type in {'bin', 'dylib'}:
             args.extend(rustc.get_linker_always_args())
 
         args += self.generate_basic_compiler_args(target, rustc)
         # Rustc replaces - with _. spaces or dots are not allowed, so we replace them with underscores
         args += ['--crate-name', target.name.replace('-', '_').replace(' ', '_').replace('.', '_')]
-        depfile = os.path.join(self.get_target_private_dir(target), target.name + '.d')
-        args += rustc.get_dependency_gen_args(target_name, depfile)
+        if depfile:
+            args += rustc.get_dependency_gen_args(target_name, depfile)
         args += rustc.get_output_args(target_name)
         args += ['-C', 'metadata=' + target.get_id()]
         args += target.get_extra_args('rust')
+        return args
+
+    def get_rust_compiler_deps_and_args(self, target: build.BuildTarget, rustc: Compiler) -> T.Tuple[T.List[str], T.List[RustDep], T.List[str]]:
+        deps: T.List[str] = []
+        project_deps: T.List[RustDep] = []
+        args: T.List[str] = []
 
         # Rustc always use non-debug Windows runtime. Inject the one selected
         # by Meson options instead.
@@ -2124,6 +2118,33 @@ def _link_library(libname: str, static: bool, bundle: bool = False):
         if isinstance(target, build.SharedLibrary) or has_shared_deps:
             args += self.get_build_rpath_args(target, rustc)
 
+        return deps, project_deps, args
+
+    def generate_rust_target(self, target: build.BuildTarget) -> None:
+        rustc = T.cast('RustCompiler', target.compilers['rust'])
+        self.generate_generator_list_rules(target)
+
+        for i in target.get_sources():
+            if not rustc.can_compile(i):
+                raise InvalidArguments(f'Rust target {target.get_basename()} contains a non-rust source file.')
+        for g in target.get_generated_sources():
+            for i in g.get_outputs():
+                if not rustc.can_compile(i):
+                    raise InvalidArguments(f'Rust target {target.get_basename()} contains a non-rust source file.')
+
+        orderdeps, main_rust_file = self.generate_rust_sources(target)
+        target_name = self.get_target_filename(target)
+        if main_rust_file is None:
+            raise RuntimeError('A Rust target has no Rust sources. This is weird. Also a bug. Please report')
+
+        args = rustc.compiler_args()
+
+        depfile = os.path.join(self.get_target_private_dir(target), target.name + '.d')
+        args += self.get_rust_compiler_args(target, rustc, target.rust_crate_type, depfile)
+
+        deps, project_deps, deps_args = self.get_rust_compiler_deps_and_args(target, rustc)
+        args += deps_args
+
         proc_macro_dylib_path = None
         if target.rust_crate_type == 'proc-macro':
             proc_macro_dylib_path = self.get_target_filename_abs(target)
@@ -2140,6 +2161,7 @@ def _link_library(libname: str, static: bool, bundle: bool = False):
         if orderdeps:
             element.add_orderdep(orderdeps)
         if deps:
+            # dependencies need to cause a relink, they're not just for ordering
             element.add_dep(deps)
         element.add_item('ARGS', args)
         element.add_item('targetdep', depfile)
@@ -2148,6 +2170,15 @@ def _link_library(libname: str, static: bool, bundle: bool = False):
             self.generate_shsym(target)
         self.create_target_source_introspection(target, rustc, args, [main_rust_file], [])
 
+        if target.doctests:
+            assert target.doctests.target is not None
+            rustdoc = rustc.get_rustdoc(self.environment)
+            args = rustdoc.get_exe_args()
+            args += self.get_rust_compiler_args(target.doctests.target, rustdoc, target.rust_crate_type)
+            _, _, deps_args = self.get_rust_compiler_deps_and_args(target.doctests.target, rustdoc)
+            args += deps_args
+            target.doctests.cmd_args = args.to_native() + [main_rust_file] + target.doctests.cmd_args
+
     @staticmethod
     def get_rule_suffix(for_machine: MachineChoice) -> str:
         return PerMachine('_FOR_BUILD', '')[for_machine]

mesonbuild/build.py

@@ -43,7 +43,7 @@
     from .backend.backends import Backend
     from .compilers import Compiler
     from .interpreter.interpreter import SourceOutputs, Interpreter
-    from .interpreter.interpreterobjects import Test
+    from .interpreter.interpreterobjects import Test, Doctest
     from .interpreterbase import SubProject
     from .linkers.linkers import StaticLinker
     from .mesonlib import ExecutableSerialisation, FileMode, FileOrString
@@ -755,6 +755,7 @@ def __init__(
         self.name_prefix_set = False
         self.name_suffix_set = False
         self.filename = 'no_name'
+        self.doctests: T.Optional[Doctest] = None
         # The debugging information file this target will generate
         self.debug_filename = None
         # The list of all files outputted by this target. Useful in cases such

mesonbuild/cmake/toolchain.py

@@ -9,6 +9,7 @@
 from .common import language_map, cmake_get_generator_args
 from .. import mlog
 
+import os.path
 import shutil
 import typing as T
 from enum import Enum
@@ -198,7 +199,7 @@ def is_cmdline_option(compiler: 'Compiler', arg: str) -> bool:
         if compiler.get_argument_syntax() == 'msvc':
             return arg.startswith('/')
         else:
-            if compiler.exelist[0] == 'zig' and arg in {'ar', 'cc', 'c++', 'dlltool', 'lib', 'ranlib', 'objcopy', 'rc'}:
+            if os.path.basename(compiler.get_exe()) == 'zig' and arg in {'ar', 'cc', 'c++', 'dlltool', 'lib', 'ranlib', 'objcopy', 'rc'}:
                 return True
             return arg.startswith('-')
 

mesonbuild/compilers/compilers.py

@@ -488,6 +488,12 @@ def get_id(self) -> str:
     def get_modes(self) -> T.List[Compiler]:
         return self.modes
 
+    def get_exe(self) -> str:
+        return self.exelist[0]
+
+    def get_exe_args(self) -> T.List[str]:
+        return self.exelist[1:]
+
     def get_linker_id(self) -> str:
         # There is not guarantee that we have a dynamic linker instance, as
         # some languages don't have separate linkers and compilers. In those

mesonbuild/compilers/detect.py

@@ -1094,7 +1094,7 @@ def detect_rust_compiler(env: 'Environment', for_machine: MachineChoice) -> Rust
                 extra_args: T.Dict[str, T.Union[str, bool]] = {}
                 always_args: T.List[str] = []
                 if is_link_exe:
-                    compiler.extend(cls.use_linker_args(cc.linker.exelist[0], ''))
+                    compiler.extend(cls.use_linker_args(cc.linker.get_exe(), ''))
                     extra_args['direct'] = True
                     extra_args['machine'] = cc.linker.machine
                 else:
@@ -1126,7 +1126,7 @@ def detect_rust_compiler(env: 'Environment', for_machine: MachineChoice) -> Rust
                 # inserts the correct prefix itself.
                 assert isinstance(linker, linkers.VisualStudioLikeLinkerMixin)
                 linker.direct = True
-                compiler.extend(cls.use_linker_args(linker.exelist[0], ''))
+                compiler.extend(cls.use_linker_args(linker.get_exe(), ''))
             else:
                 # On linux and macos rust will invoke the c compiler for
                 # linking, on windows it will use lld-link or link.exe.

mesonbuild/compilers/rust.py

@@ -103,7 +103,7 @@ def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoic
     def needs_static_linker(self) -> bool:
         return False
 
-    def sanity_check(self, work_dir: str, environment: 'Environment') -> None:
+    def sanity_check(self, work_dir: str, environment: Environment) -> None:
         source_name = os.path.join(work_dir, 'sanity.rs')
         output_name = os.path.join(work_dir, 'rusttest')
         cmdlist = self.exelist.copy()
@@ -270,6 +270,8 @@ def get_colorout_args(self, colortype: str) -> T.List[str]:
 
     def get_linker_always_args(self) -> T.List[str]:
         args: T.List[str] = []
+        # Rust is super annoying, calling -C link-arg foo does not work, it has
+        # to be -C link-arg=foo
         for a in super().get_linker_always_args():
             args.extend(['-C', f'link-arg={a}'])
         return args
@@ -303,7 +305,7 @@ def get_rust_tool(self, name: str, env: Environment) -> T.List[str]:
             exelist = rustup_exelist + [name]
         else:
             exelist = [name]
-            args = self.exelist[1:]
+            args = self.get_exe_args()
 
         from ..programs import find_external_program
         for prog in find_external_program(env, self.for_machine, exelist[0], exelist[0],
@@ -315,6 +317,14 @@ def get_rust_tool(self, name: str, env: Environment) -> T.List[str]:
 
         return exelist + args
 
+    @functools.lru_cache(maxsize=None)
+    def get_rustdoc(self, env: 'Environment') -> T.Optional[RustdocTestCompiler]:
+        exelist = self.get_rust_tool('rustdoc', env)
+        if not exelist:
+            return None
+
+        return RustdocTestCompiler(exelist, self.version, self.for_machine,
+                                   self.is_cross, self.info, linker=self.linker)
 
 class ClippyRustCompiler(RustCompiler):
 

mesonbuild/dependencies/dev.py

@@ -591,7 +591,7 @@ def __init__(self, environment: 'Environment', kwargs: JNISystemDependencyKW):
 
         self.java_home = environment.properties[self.for_machine].get_java_home()
         if not self.java_home:
-            self.java_home = pathlib.Path(shutil.which(self.javac.exelist[0])).resolve().parents[1]
+            self.java_home = pathlib.Path(shutil.which(self.javac.get_exe())).resolve().parents[1]
             if m.is_darwin():
                 problem_java_prefix = pathlib.Path('/System/Library/Frameworks/JavaVM.framework/Versions')
                 if problem_java_prefix in self.java_home.parents: