[DNM] Add documentation for hypothetical pre-timeout scripts

.config/nextest.toml

@@ -62,7 +62,7 @@ max-threads = 8
 [test-groups.unused-group]
 max-threads = 8
 
-[script.build-seed-archive]
+[script.setup.build-seed-archive]
 # This command builds a seed archive that's used by the integration-tests
 # package. This archive is not currently used by the older nextest-runner
 # integration framework, but that should really go away at some point.

fixtures/nextest-tests/.config/nextest.toml

@@ -91,8 +91,8 @@ max-threads = 4
 [test-groups.unused]
 max-threads = 20
 
-[script.my-script-unix]
+[script.setup.my-script-unix]
 command = './scripts/my-script.sh'
 
-[script.my-script-windows]
+[script.setup.my-script-windows]
 command = 'cmd /c "scripts\\my-script.bat"'

nextest-runner/src/config/config_impl.rs

@@ -4,16 +4,17 @@
 use super::{
     ArchiveConfig, CompiledByProfile, CompiledData, CompiledDefaultFilter, ConfigExperimental,
     CustomTestGroup, DefaultJunitImpl, DeserializedOverride, DeserializedProfileScriptConfig,
-    JunitConfig, JunitImpl, NextestVersionDeserialize, RetryPolicy, ScriptConfig, ScriptId,
-    SettingSource, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings,
+    JunitConfig, JunitImpl, NextestVersionDeserialize, RetryPolicy, ScriptId, SettingSource,
+    SetupScriptConfig, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings,
     TestThreads, ThreadsRequired, ToolConfigFile,
 };
 use crate::{
+    config::{PreTimeoutScript, PreTimeoutScriptConfig, ScriptType},
     errors::{
         provided_by_tool, ConfigParseError, ConfigParseErrorKind, ProfileNotFound,
         UnknownConfigScriptError, UnknownTestGroupError,
     },
-    list::TestList,
+    list::{TestInstance, TestList},
     platform::BuildPlatforms,
     reporter::{FinalStatusLevel, StatusLevel, TestOutputDisplay},
 };
@@ -217,7 +218,7 @@ impl NextestConfig {
         let mut compiled = CompiledByProfile::for_default_config();
 
         let mut known_groups = BTreeSet::new();
-        let mut known_scripts = BTreeSet::new();
+        let mut known_scripts = BTreeMap::new();
 
         // Next, merge in tool configs.
         for ToolConfigFile { config_file, tool } in tool_config_files_rev {
@@ -289,7 +290,7 @@ impl NextestConfig {
         experimental: &BTreeSet<ConfigExperimental>,
         unknown_callback: &mut impl FnMut(&Utf8Path, Option<&str>, &BTreeSet<String>),
         known_groups: &mut BTreeSet<CustomTestGroup>,
-        known_scripts: &mut BTreeSet<ScriptId>,
+        known_scripts: &mut BTreeMap<ScriptId, ScriptType>,
     ) -> Result<(), ConfigParseError> {
         // Try building default builder + this file to get good error attribution and handle
         // overrides additively.
@@ -328,8 +329,8 @@ impl NextestConfig {
 
         known_groups.extend(valid_groups);
 
-        // If scripts are present, check that the experimental feature is enabled.
-        if !this_config.scripts.is_empty()
+        // If setup scripts are present, check that the experimental feature is enabled.
+        if !this_config.scripts.setup.is_empty()
             && !experimental.contains(&ConfigExperimental::SetupScripts)
         {
             return Err(ConfigParseError::new(
@@ -341,9 +342,33 @@ impl NextestConfig {
             ));
         }
 
-        // Check that setup scripts are named as expected.
-        let (valid_scripts, invalid_scripts): (BTreeSet<_>, _) =
-            this_config.scripts.keys().cloned().partition(|script| {
+        // If pre-timeout scripts are present, check that the experimental feature is enabled.
+        if !this_config.scripts.pre_timeout.is_empty()
+            && !experimental.contains(&ConfigExperimental::PreTimeoutScripts)
+        {
+            return Err(ConfigParseError::new(
+                config_file,
+                tool,
+                ConfigParseErrorKind::ExperimentalFeatureNotEnabled {
+                    feature: ConfigExperimental::PreTimeoutScripts,
+                },
+            ));
+        }
+
+        if let Some(dup) = this_config.scripts.duplicate_ids().next() {
+            return Err(ConfigParseError::new(
+                config_file,
+                tool,
+                ConfigParseErrorKind::DuplicateConfigScriptName(dup.clone()),
+            ));
+        }
+
+        // Check that scripts are named as expected.
+        let (valid_scripts, invalid_scripts): (BTreeSet<_>, _) = this_config
+            .scripts
+            .all_script_ids()
+            .cloned()
+            .partition(|script| {
                 if let Some(tool) = tool {
                     // The first component must be the tool name.
                     script
@@ -365,7 +390,10 @@ impl NextestConfig {
             return Err(ConfigParseError::new(config_file, tool, kind));
         }
 
-        known_scripts.extend(valid_scripts);
+        known_scripts.extend(valid_scripts.into_iter().map(|id| {
+            let script_type = this_config.scripts.script_type(&id);
+            (id, script_type)
+        }));
 
         let this_config = this_config.into_config_impl();
 
@@ -434,45 +462,76 @@ impl NextestConfig {
 
         // Check that scripts are known.
         let mut unknown_script_errors = Vec::new();
-        let mut check_script_ids = |profile_name: &str, scripts: &[ScriptId]| {
-            if !scripts.is_empty() && !experimental.contains(&ConfigExperimental::SetupScripts) {
-                return Err(ConfigParseError::new(
-                    config_file,
-                    tool,
-                    ConfigParseErrorKind::ExperimentalFeatureNotEnabled {
-                        feature: ConfigExperimental::SetupScripts,
-                    },
-                ));
-            }
-            for script in scripts {
-                if !known_scripts.contains(script) {
-                    unknown_script_errors.push(UnknownConfigScriptError {
-                        profile_name: profile_name.to_owned(),
-                        name: script.clone(),
-                    });
+        let mut check_script_ids =
+            |profile_name: &str, script_type: ScriptType, scripts: &[ScriptId]| {
+                let config_feature = match script_type {
+                    ScriptType::Setup => ConfigExperimental::SetupScripts,
+                    ScriptType::PreTimeout => ConfigExperimental::PreTimeoutScripts,
+                };
+                if !scripts.is_empty() && !experimental.contains(&config_feature) {
+                    return Err(ConfigParseError::new(
+                        config_file,
+                        tool,
+                        ConfigParseErrorKind::ExperimentalFeatureNotEnabled {
+                            feature: config_feature,
+                        },
+                    ));
+                }
+                for script in scripts {
+                    match known_scripts.get(script) {
+                        None => {
+                            unknown_script_errors.push(UnknownConfigScriptError {
+                                profile_name: profile_name.to_owned(),
+                                name: script.clone(),
+                            });
+                        }
+                        Some(actual_script_type) if *actual_script_type != script_type => {
+                            return Err(ConfigParseError::new(
+                                config_file,
+                                tool,
+                                ConfigParseErrorKind::WrongConfigScriptType {
+                                    script: script.clone(),
+                                    attempted: script_type,
+                                    actual: actual_script_type.clone(),
+                                },
+                            ));
+                        }
+                        Some(_) => (),
+                    }
                 }
-            }
 
-            Ok(())
-        };
+                Ok(())
+            };
 
         this_compiled
             .default
             .scripts
             .iter()
-            .try_for_each(|scripts| check_script_ids("default", &scripts.setup))?;
+            .try_for_each(|scripts| {
+                check_script_ids("default", ScriptType::Setup, &scripts.setup)?;
+                check_script_ids(
+                    "default",
+                    ScriptType::PreTimeout,
+                    scripts.pre_timeout.as_slice(),
+                )
+            })?;
         this_compiled
             .other
             .iter()
             .try_for_each(|(profile_name, data)| {
-                data.scripts
-                    .iter()
-                    .try_for_each(|scripts| check_script_ids(profile_name, &scripts.setup))
+                data.scripts.iter().try_for_each(|scripts| {
+                    check_script_ids(profile_name, ScriptType::Setup, &scripts.setup)?;
+                    check_script_ids(
+                        profile_name,
+                        ScriptType::PreTimeout,
+                        scripts.pre_timeout.as_slice(),
+                    )
+                })
             })?;
 
         // If there were any unknown scripts, error out.
         if !unknown_script_errors.is_empty() {
-            let known_scripts = known_scripts.iter().cloned().collect();
+            let known_scripts = known_scripts.keys().cloned().collect();
             return Err(ConfigParseError::new(
                 config_file,
                 tool,
@@ -578,8 +637,7 @@ pub struct EarlyProfile<'cfg> {
     default_profile: &'cfg DefaultProfileImpl,
     custom_profile: Option<&'cfg CustomProfileImpl>,
     test_groups: &'cfg BTreeMap<CustomTestGroup, TestGroupConfig>,
-    // This is ordered because the scripts are used in the order they're defined.
-    scripts: &'cfg IndexMap<ScriptId, ScriptConfig>,
+    scripts: &'cfg Scripts,
     // Invariant: `compiled_data.default_filter` is always present.
     pub(super) compiled_data: CompiledData<PreBuildPlatform>,
 }
@@ -646,7 +704,7 @@ pub struct EvaluatableProfile<'cfg> {
     custom_profile: Option<&'cfg CustomProfileImpl>,
     test_groups: &'cfg BTreeMap<CustomTestGroup, TestGroupConfig>,
     // This is ordered because the scripts are used in the order they're defined.
-    scripts: &'cfg IndexMap<ScriptId, ScriptConfig>,
+    scripts: &'cfg Scripts,
     // Invariant: `compiled_data.default_filter` is always present.
     pub(super) compiled_data: CompiledData<FinalConfig>,
     // The default filter that's been resolved after considering overrides (i.e.
@@ -683,8 +741,8 @@ impl<'cfg> EvaluatableProfile<'cfg> {
     }
 
     /// Returns the global script configuration.
-    pub fn script_config(&self) -> &'cfg IndexMap<ScriptId, ScriptConfig> {
-        self.scripts
+    pub fn script_config(&self) -> &'cfg Scripts {
+        &self.scripts
     }
 
     /// Returns the retry count for this profile.
@@ -777,6 +835,11 @@ impl<'cfg> EvaluatableProfile<'cfg> {
         SetupScripts::new(self, test_list)
     }
 
+    /// Returns the pre-timeout script for the specified test, if it exists.
+    pub fn pre_timeout_script(&self, test_instance: &TestInstance<'_>) -> Option<PreTimeoutScript> {
+        PreTimeoutScript::new(self, test_instance)
+    }
+
     /// Returns settings for individual tests.
     pub fn settings_for(&self, query: &TestQuery<'_>) -> TestSettings {
         TestSettings::new(self, query)
@@ -809,7 +872,7 @@ impl<'cfg> EvaluatableProfile<'cfg> {
 pub(super) struct NextestConfigImpl {
     store: StoreConfigImpl,
     test_groups: BTreeMap<CustomTestGroup, TestGroupConfig>,
-    scripts: IndexMap<ScriptId, ScriptConfig>,
+    scripts: Scripts,
     default_profile: DefaultProfileImpl,
     other_profiles: HashMap<String, CustomProfileImpl>,
 }
@@ -863,7 +926,7 @@ struct NextestConfigDeserialize {
     #[serde(default)]
     test_groups: BTreeMap<CustomTestGroup, TestGroupConfig>,
     #[serde(default, rename = "script")]
-    scripts: IndexMap<ScriptId, ScriptConfig>,
+    scripts: Scripts,
     #[serde(rename = "profile")]
     profiles: HashMap<String, CustomProfileImpl>,
 }
@@ -962,7 +1025,7 @@ impl DefaultProfileImpl {
         &self.overrides
     }
 
-    pub(super) fn setup_scripts(&self) -> &[DeserializedProfileScriptConfig] {
+    pub(super) fn scripts(&self) -> &[DeserializedProfileScriptConfig] {
         &self.scripts
     }
 }
@@ -1024,6 +1087,47 @@ impl CustomProfileImpl {
     }
 }
 
+/// The scripts defined in a profile.
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Scripts {
+    // These maps are ordered because scripts are used in the order they're defined.
+    /// The setup scripts defined in a profile.
+    #[serde(default)]
+    pub setup: IndexMap<ScriptId, SetupScriptConfig>,
+    /// The pre-timeout scripts defined in a profile.
+    #[serde(default)]
+    pub pre_timeout: IndexMap<ScriptId, PreTimeoutScriptConfig>,
+}
+
+impl Scripts {
+    /// Determines the type of the script with the given ID.
+    ///
+    /// Panics if the ID is invalid.
+    fn script_type(&self, id: &ScriptId) -> ScriptType {
+        if self.setup.contains_key(id) {
+            ScriptType::Setup
+        } else if self.pre_timeout.contains_key(id) {
+            ScriptType::PreTimeout
+        } else {
+            panic!("Scripts::script_type called with invalid script ID: {id}")
+        }
+    }
+
+    /// Returns an iterator over the names of all scripts of all types.
+    fn all_script_ids(&self) -> impl Iterator<Item = &ScriptId> {
+        self.setup.keys().chain(self.pre_timeout.keys())
+    }
+
+    /// Returns an iterator over names that are used by more than one type of
+    /// script.
+    fn duplicate_ids(&self) -> impl Iterator<Item = &ScriptId> {
+        self.pre_timeout
+            .keys()
+            .filter(|k| self.setup.contains_key(*k))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

nextest-runner/src/config/nextest_version.rs

@@ -232,11 +232,13 @@ impl NextestVersionConfig {
 pub enum ConfigExperimental {
     /// Enable support for setup scripts.
     SetupScripts,
+    /// Enable support for pre-timeout scripts.
+    PreTimeoutScripts,
 }
 
 impl ConfigExperimental {
     fn known() -> impl Iterator<Item = Self> {
-        vec![Self::SetupScripts].into_iter()
+        vec![Self::SetupScripts, Self::PreTimeoutScripts].into_iter()
     }
 }
 
@@ -246,6 +248,7 @@ impl FromStr for ConfigExperimental {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "setup-scripts" => Ok(Self::SetupScripts),
+            "pre-timeout-scripts" => Ok(Self::PreTimeoutScripts),
             _ => Err(()),
         }
     }
@@ -255,6 +258,7 @@ impl fmt::Display for ConfigExperimental {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::SetupScripts => write!(f, "setup-scripts"),
+            Self::PreTimeoutScripts => write!(f, "pre-timeout-scripts"),
         }
     }
 }