Add proper integration tests that are separate from the workspace.

While this does not catch RA errors it should simulate bevy users better.

.gitignore

@@ -4,6 +4,7 @@ crates/**/target
 benches/**/target
 tools/**/target
 **/*.rs.bk
+rustc-ice-*.txt
 
 # Cargo
 Cargo.lock

Cargo.toml

@@ -29,6 +29,10 @@ members = [
   # Bevy's error codes. This is a crate so we can automatically check all of the code blocks.
   "errors",
 ]
+exclude = [
+  # Integration tests are not part of the workspace
+  "tests-integration",
+]
 
 [workspace.lints.clippy]
 doc_markdown = "warn"

tests-integration/simple-ecs-test/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "simple-ecs-test"
+edition = "2021"
+
+[dependencies]
+bevy = { path = "../../" }

tests-integration/simple-ecs-test/src/lib.rs

@@ -0,0 +1,21 @@
+#![allow(dead_code)]
+use bevy::prelude::*;
+
+#[derive(Component)]
+struct _Component {
+    _value: f32,
+}
+
+#[derive(Resource)]
+struct _Resource {
+    _value: f32,
+}
+
+fn hello_world() {
+    println!("hello world!");
+}
+
+#[test]
+fn simple_ecs_test() {
+    App::new().add_systems(Update, hello_world).run();
+}

tools/ci/src/ci.rs

@@ -74,6 +74,13 @@ impl CI {
                 cmds.append(&mut commands::ClippyCommand::default().prepare(sh, flags));
                 cmds.append(&mut commands::TestCommand::default().prepare(sh, flags));
                 cmds.append(&mut commands::TestCheckCommand::default().prepare(sh, flags));
+                cmds.append(&mut commands::IntegrationTestCommand::default().prepare(sh, flags));
+                cmds.append(
+                    &mut commands::IntegrationTestCheckCommand::default().prepare(sh, flags),
+                );
+                cmds.append(
+                    &mut commands::IntegrationTestCleanCommand::default().prepare(sh, flags),
+                );
                 cmds.append(&mut commands::DocCheckCommand::default().prepare(sh, flags));
                 cmds.append(&mut commands::DocTestCommand::default().prepare(sh, flags));
                 cmds.append(&mut commands::CompileCheckCommand::default().prepare(sh, flags));
@@ -100,6 +107,9 @@ enum Commands {
     Clippy(commands::ClippyCommand),
     Test(commands::TestCommand),
     TestCheck(commands::TestCheckCommand),
+    IntegrationTest(commands::IntegrationTestCommand),
+    IntegrationTestCheck(commands::IntegrationTestCheckCommand),
+    IntegrationTestClean(commands::IntegrationTestCleanCommand),
     DocCheck(commands::DocCheckCommand),
     DocTest(commands::DocTestCommand),
     CompileCheck(commands::CompileCheckCommand),
@@ -120,6 +130,9 @@ impl Prepare for Commands {
             Commands::Clippy(subcommand) => subcommand.prepare(sh, flags),
             Commands::Test(subcommand) => subcommand.prepare(sh, flags),
             Commands::TestCheck(subcommand) => subcommand.prepare(sh, flags),
+            Commands::IntegrationTest(subcommand) => subcommand.prepare(sh, flags),
+            Commands::IntegrationTestCheck(subcommand) => subcommand.prepare(sh, flags),
+            Commands::IntegrationTestClean(subcommand) => subcommand.prepare(sh, flags),
             Commands::DocCheck(subcommand) => subcommand.prepare(sh, flags),
             Commands::DocTest(subcommand) => subcommand.prepare(sh, flags),
             Commands::CompileCheck(subcommand) => subcommand.prepare(sh, flags),

tools/ci/src/commands/compile.rs

@@ -1,13 +1,13 @@
 use crate::{
     commands::{
         BenchCheckCommand, CompileCheckCommand, CompileFailCommand, ExampleCheckCommand,
-        TestCheckCommand,
+        IntegrationTestCheckCommand, TestCheckCommand,
     },
     Flag, Prepare, PreparedCommand,
 };
 use argh::FromArgs;
 
-/// Alias for running the `compile-fail`, `bench-check`, `example-check`, `compile-check`, and `test-check` subcommands.
+/// Alias for running the `compile-fail`, `bench-check`, `example-check`, `compile-check`, `test-check` and `test-integration-check` subcommands.
 #[derive(FromArgs, Default)]
 #[argh(subcommand, name = "compile")]
 pub struct CompileCommand {}
@@ -20,6 +20,7 @@ impl Prepare for CompileCommand {
         commands.append(&mut ExampleCheckCommand::default().prepare(sh, flags));
         commands.append(&mut CompileCheckCommand::default().prepare(sh, flags));
         commands.append(&mut TestCheckCommand::default().prepare(sh, flags));
+        commands.append(&mut IntegrationTestCheckCommand::default().prepare(sh, flags));
         commands
     }
 }

tools/ci/src/commands/integration_test.rs

@@ -0,0 +1,30 @@
+use crate::{commands::get_integration_tests, Flag, Prepare, PreparedCommand};
+use argh::FromArgs;
+use xshell::cmd;
+
+/// Runs all integration tests (except for doc tests).
+#[derive(FromArgs, Default)]
+#[argh(subcommand, name = "integration-test")]
+pub struct IntegrationTestCommand {}
+
+impl Prepare for IntegrationTestCommand {
+    fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec<PreparedCommand<'a>> {
+        let no_fail_fast = flags
+            .contains(Flag::KEEP_GOING)
+            .then_some("--no-fail-fast")
+            .unwrap_or_default();
+
+        get_integration_tests(sh)
+            .into_iter()
+            .map(|path| {
+                PreparedCommand::new::<Self>(
+                    cmd!(
+                        sh,
+                        "cargo test --manifest-path {path}/Cargo.toml --tests {no_fail_fast}"
+                    ),
+                    "Please fix failing integration tests in output above.",
+                )
+            })
+            .collect()
+    }
+}

tools/ci/src/commands/integration_test_check.rs

@@ -0,0 +1,35 @@
+use std::path::Path;
+
+use crate::{Flag, Prepare, PreparedCommand};
+use argh::FromArgs;
+use xshell::cmd;
+
+pub fn get_integration_tests(sh: &xshell::Shell) -> Vec<String> {
+    let integration_test_paths = sh.read_dir(Path::new("./tests-integration")).unwrap();
+
+    // Filter out non-directories
+    integration_test_paths
+        .into_iter()
+        .filter(|path| path.is_dir())
+        .map(|path| path.to_string_lossy().to_string())
+        .collect()
+}
+
+/// Checks that all integration tests compile.
+#[derive(FromArgs, Default)]
+#[argh(subcommand, name = "integration-test-check")]
+pub struct IntegrationTestCheckCommand {}
+
+impl Prepare for IntegrationTestCheckCommand {
+    fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec<PreparedCommand<'a>> {
+        get_integration_tests(sh)
+            .into_iter()
+            .map(|path| {
+                PreparedCommand::new::<Self>(
+                    cmd!(sh, "cargo check --manifest-path {path}/Cargo.toml --tests"),
+                    "Please fix compiler errors for tests in output above.",
+                )
+            })
+            .collect()
+    }
+}

tools/ci/src/commands/integration_test_clean.rs

@@ -0,0 +1,24 @@
+use crate::{Flag, Prepare, PreparedCommand};
+use argh::FromArgs;
+use xshell::cmd;
+
+use super::get_integration_tests;
+
+/// Cleans the build artifacts for all integration tests.
+#[derive(FromArgs, Default)]
+#[argh(subcommand, name = "integration-test-clean")]
+pub struct IntegrationTestCleanCommand {}
+
+impl Prepare for IntegrationTestCleanCommand {
+    fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec<PreparedCommand<'a>> {
+        get_integration_tests(sh)
+            .into_iter()
+            .map(|path| {
+                PreparedCommand::new::<Self>(
+                    cmd!(sh, "cargo clean --manifest-path {path}/Cargo.toml"),
+                    "Failed to clean integration test.",
+                )
+            })
+            .collect()
+    }
+}

tools/ci/src/commands/mod.rs

@@ -9,6 +9,9 @@ pub use doc_check::*;
 pub use doc_test::*;
 pub use example_check::*;
 pub use format::*;
+pub use integration_test::*;
+pub use integration_test_check::*;
+pub use integration_test_clean::*;
 pub use lints::*;
 pub use test::*;
 pub use test_check::*;
@@ -24,6 +27,9 @@ mod doc_check;
 mod doc_test;
 mod example_check;
 mod format;
+mod integration_test;
+mod integration_test_check;
+mod integration_test_clean;
 mod lints;
 mod test;
 mod test_check;

tools/ci/src/commands/test_check.rs

@@ -11,7 +11,7 @@ impl Prepare for TestCheckCommand {
     fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec<PreparedCommand<'a>> {
         vec![PreparedCommand::new::<Self>(
             cmd!(sh, "cargo check --workspace --tests"),
-            "Please fix compiler examples for tests in output above.",
+            "Please fix compiler errors for tests in output above.",
         )]
     }
 }