replace with_iterations with with_test_time

bin/cargo-bolero/src/random.rs

@@ -17,8 +17,10 @@ pub(crate) fn test(selection: &Selection, test_args: &test::Args) -> Result<()>
     }
 
     optional_arg!(seed, "BOLERO_RANDOM_SEED");
-    optional_arg!(runs, "BOLERO_RANDOM_ITERATIONS");
     optional_arg!(max_input_length, "BOLERO_RANDOM_MAX_LEN");
+    if let Some(t) = test_args.time {
+        cmd.env("BOLERO_RANDOM_TEST_TIME_MS", t.as_millis().to_string());
+    }
 
     // TODO implement other options
     /*

lib/bolero-engine/src/rng.rs

@@ -1,29 +1,29 @@
 use crate::{driver, panic, ByteSliceTestInput, Engine, TargetLocation, Test};
-use core::fmt::Debug;
+use core::{fmt::Debug, time::{Duration}};
 use rand::{rngs::StdRng, Rng, RngCore, SeedableRng};
+use std::time::Instant;
 
 #[derive(Clone, Copy, Debug)]
 pub struct Options {
-    pub iterations: Option<usize>,
+    pub test_time: Option<Duration>,
     pub max_len: Option<usize>,
     pub seed: Option<u64>,
 }
 
 impl Default for Options {
     fn default() -> Self {
         Self {
-            iterations: get_var("BOLERO_RANDOM_ITERATIONS"),
+            test_time: get_var("BOLERO_RANDOM_TEST_TIME_MS").map(Duration::from_millis),
             max_len: get_var("BOLERO_RANDOM_MAX_LEN"),
             seed: get_var("BOLERO_RANDOM_SEED"),
         }
     }
 }
 
 impl Options {
-    pub fn iterations_or_default(&self) -> usize {
-        // RNG tests are really slow with miri so we limit the number of iterations
-        self.iterations
-            .unwrap_or(if cfg!(miri) { 25 } else { 1000 })
+    pub fn test_time_or_default(&self) -> Duration {
+        self.test_time
+            .unwrap_or_else(|| Duration::from_secs(1))
     }
 
     pub fn max_len_or_default(&self) -> usize {
@@ -42,7 +42,7 @@ impl Options {
 /// enough to find edge cases.
 #[derive(Clone)]
 pub struct RngEngine {
-    pub iterations: usize,
+    pub test_time: Duration,
     pub max_len: usize,
     pub seed: u64,
 }
@@ -56,7 +56,7 @@ impl Default for RngEngine {
 impl From<Options> for RngEngine {
     fn from(options: Options) -> Self {
         Self {
-            iterations: options.iterations_or_default(),
+            test_time: options.test_time_or_default(),
             max_len: options.max_len_or_default(),
             seed: options.seed_or_rand(),
         }
@@ -70,9 +70,9 @@ impl RngEngine {
         Self::default()
     }
 
-    /// Set the number of test iterations
-    pub fn with_iterations(self, iterations: usize) -> Self {
-        Self { iterations, ..self }
+    /// Set the test time
+    pub fn with_test_time(self, test_time: Duration) -> Self {
+        Self { test_time, ..self }
     }
 
     /// Set the maximum length of a test input
@@ -97,32 +97,11 @@ where
 
         let mut state = RngState::new(self.seed, self.max_len, options);
 
-        let mut valid = 0;
-        let mut invalid = 0;
-        while valid < self.iterations {
+        let start_time = Instant::now();
+        while start_time.elapsed() < self.test_time {
+            eprintln!("running, elapsed is {:?}", start_time.elapsed());
             match test.test(&mut state.next_input()) {
-                Ok(true) => {
-                    valid += 1;
-                    continue;
-                }
-                Ok(false) => {
-                    invalid += 1;
-                    if invalid > self.iterations * 2 {
-                        panic!(
-                            concat!(
-                                "Test input could not be satisfied after {} iterations:\n",
-                                "         valid: {}\n",
-                                "       invalid: {}\n",
-                                "  target count: {}\n",
-                                "\n",
-                                "Try reconfiguring the input generator to produce more valid inputs",
-                            ),
-                            valid + invalid,
-                            valid,
-                            invalid,
-                            self.iterations
-                        );
-                    }
+                Ok(_) => {
                     continue;
                 }
                 #[cfg(not(miri))]

lib/bolero/src/lib.rs

@@ -359,8 +359,8 @@ cfg_if::cfg_if! {
     if #[cfg(any(fuzzing, kani))] {
         impl<G, Engine, InputOwnership> TestTarget<G, Engine, InputOwnership> {
             /// Set the number of iterations executed
-            pub fn with_iterations(self, iterations: usize) -> Self {
-                let _ = iterations;
+            pub fn with_test_time(self, test_time: Duration) -> Self {
+                let _ = test_time;
                 self
             }
 
@@ -373,8 +373,8 @@ cfg_if::cfg_if! {
     } else {
         impl<G, InputOwnership> TestTarget<G, crate::test::TestEngine, InputOwnership> {
             /// Set the number of iterations executed
-            pub fn with_iterations(mut self, iterations: usize) -> Self {
-                self.engine.with_iterations(iterations);
+            pub fn with_test_time(mut self, test_time: Duration) -> Self {
+                self.engine.with_test_time(test_time);
                 self
             }
 

lib/bolero/src/test/mod.rs

@@ -1,8 +1,8 @@
 #![cfg_attr(fuzzing_random, allow(dead_code))]
 
 use bolero_engine::{driver, rng, test_failure::TestFailure, Engine, TargetLocation, Test};
-use core::iter::empty;
-use std::path::PathBuf;
+use core::{iter::empty, time::Duration};
+use std::{path::PathBuf, time::Instant};
 
 mod input;
 use input::*;
@@ -33,8 +33,8 @@ impl TestEngine {
         }
     }
 
-    pub fn with_iterations(&mut self, iterations: usize) -> &mut Self {
-        self.rng_cfg.iterations = self.rng_cfg.iterations.or(Some(iterations));
+    pub fn with_test_time(&mut self, test_time: Duration) -> &mut Self {
+        self.rng_cfg.test_time = self.rng_cfg.test_time.or(Some(test_time));
         self
     }
 
@@ -76,12 +76,11 @@ impl TestEngine {
     fn rng_tests(&self) -> impl Iterator<Item = RngTest> {
         use rand::{rngs::StdRng, RngCore, SeedableRng};
 
-        let iterations = self.rng_cfg.iterations_or_default();
         let max_len = self.rng_cfg.max_len_or_default();
         let seed = self.rng_cfg.seed_or_rand();
         let mut seed_rng = StdRng::seed_from_u64(seed);
 
-        (0..iterations)
+        (0..)
             .scan(seed, move |state, _index| {
                 let seed = *state;
                 *state = seed_rng.next_u64();
@@ -90,7 +89,7 @@ impl TestEngine {
             .map(move |seed| input::RngTest { seed, max_len })
     }
 
-    fn tests(&self) -> Vec<NamedTest> {
+    fn tests(&self) -> impl Iterator<Item = NamedTest> {
         let rng_tests = self.rng_tests().map(move |test| NamedTest {
             name: format!("[BOLERO_RANDOM_SEED={}]", test.seed),
             data: TestInput::RngTest(test),
@@ -103,7 +102,6 @@ impl TestEngine {
             .chain(self.file_tests(["corpus"].iter().cloned()))
             .chain(self.file_tests(["afl_state", "queue"].iter().cloned()))
             .chain(rng_tests)
-            .collect()
     }
 
     #[cfg(any(fuzzing_random, test))]
@@ -144,8 +142,8 @@ impl TestEngine {
         report.spawn_timer();
 
         let inputs = {
-            if self.rng_cfg.iterations.is_none() {
-                self.rng_cfg.iterations = Some(usize::MAX);
+            if self.rng_cfg.test_time.is_none() {
+                self.rng_cfg.test_time = Some(Duration::from_secs(3600 * 24 * 365 * 100));
             }
             self.rng_tests()
         };
@@ -246,7 +244,13 @@ impl TestEngine {
         bolero_engine::panic::set_hook();
         bolero_engine::panic::forward_panic(false);
 
+        let start_time = Instant::now();
+        let test_time = self.rng_cfg.test_time_or_default();
         for test in tests {
+            if start_time.elapsed() > test_time {
+                break;
+            }
+
             progress();
 
             if let Err(err) = testfn(&test.data) {

lib/bolero/src/tests.rs

@@ -53,17 +53,17 @@ fn range_generator_cloned_test() {
 #[test]
 fn nested_test() {
     check!().with_generator(0..=5).for_each(|_input: &u8| {
-        // println!("{:?}", input);
+        // println!("{:?}", _input);
     });
 }
 
 #[test]
-fn iteration_number() {
+fn with_test_time() {
     // Atomic to avoid having to think about unwind safety
     use std::sync::atomic::Ordering;
     let num_iters = std::sync::atomic::AtomicUsize::new(0);
-    check!().with_iterations(5).for_each(|_| {
+    check!().with_test_time(core::time::Duration::from_millis(5)).for_each(|_| {
         num_iters.fetch_add(1, Ordering::Relaxed);
     });
-    assert_eq!(num_iters.load(Ordering::Relaxed), 5);
+    assert!(num_iters.load(Ordering::Relaxed) > 10);
 }