Added BENCH/TEST_PRNG, replacing other ad-hoc sources of randomness

When you add a function to every benchmark suite, you know if should probably be provided by the benchmark runner itself. That being said, randomness in tests/benchmarks is a bit tricky because it needs to be strictly controlled and reproducible. No global state is used, allowing tests/benches to maintain multiple randomness stream which can be useful for checking results during a run. There's an argument for having global prng state in that the prng could be preserved across power-loss, but I have yet to see a use for this, and it would add a significant requirement to any future test/bench runner.
littlefs-project · Dec 7, 2022 · b0382fa · b0382fa
1 parent d8e7ffb
commit b0382fa
Show file tree

Hide file tree

Showing 10 changed files with 134 additions and 124 deletions.
diff --git a/benches/bench_dir.toml b/benches/bench_dir.toml
@@ -1,17 +1,3 @@
-
-
-# deterministic prng
-code = '''
-static uint32_t xorshift32(uint32_t *state) {
-    uint32_t x = *state;
-    x ^= x << 13;
-    x ^= x >> 17;
-    x ^= x << 5;
-    *state = x;
-    return x;
-}
-'''
-
 [cases.bench_dir_open]
 # 0 = in-order
 # 1 = reversed-order
@@ -37,7 +23,7 @@ code = '''
         uint32_t file_prng = i;
         for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
             for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
-                buffer[k] = xorshift32(&file_prng);
+                buffer[k] = BENCH_PRNG(&file_prng);
             }
             lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
         }
@@ -52,7 +38,7 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (N-1-i)
-            : xorshift32(&prng) % N;
+            : BENCH_PRNG(&prng) % N;
         sprintf(name, "file%08x", i_);
         lfs_file_t file;
         lfs_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0;
@@ -61,7 +47,7 @@ code = '''
         for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
             lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
             for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
-                assert(buffer[k] == xorshift32(&file_prng));
+                assert(buffer[k] == BENCH_PRNG(&file_prng));
             }
         }
 
@@ -93,7 +79,7 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (N-1-i)
-            : xorshift32(&prng) % N;
+            : BENCH_PRNG(&prng) % N;
         sprintf(name, "file%08x", i_);
         lfs_file_t file;
         lfs_file_open(&lfs, &file, name,
@@ -102,7 +88,7 @@ code = '''
         uint32_t file_prng = i_;
         for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
             for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
-                buffer[k] = xorshift32(&file_prng);
+                buffer[k] = BENCH_PRNG(&file_prng);
             }
             lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
         }
@@ -139,7 +125,7 @@ code = '''
         uint32_t file_prng = i;
         for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
             for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
-                buffer[k] = xorshift32(&file_prng);
+                buffer[k] = BENCH_PRNG(&file_prng);
             }
             lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
         }
@@ -154,7 +140,7 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (N-1-i)
-            : xorshift32(&prng) % N;
+            : BENCH_PRNG(&prng) % N;
         sprintf(name, "file%08x", i_);
         int err = lfs_remove(&lfs, name);
         assert(!err || err == LFS_ERR_NOENT);
@@ -185,7 +171,7 @@ code = '''
         uint32_t file_prng = i;
         for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
             for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
-                buffer[k] = xorshift32(&file_prng);
+                buffer[k] = BENCH_PRNG(&file_prng);
             }
             lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
         }
@@ -235,7 +221,7 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (N-1-i)
-            : xorshift32(&prng) % N;
+            : BENCH_PRNG(&prng) % N;
         printf("hm %d\n", i);
         sprintf(name, "dir%08x", i_);
         int err = lfs_mkdir(&lfs, name);
@@ -271,7 +257,7 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (N-1-i)
-            : xorshift32(&prng) % N;
+            : BENCH_PRNG(&prng) % N;
         sprintf(name, "dir%08x", i_);
         int err = lfs_remove(&lfs, name);
         assert(!err || err == LFS_ERR_NOENT);

diff --git a/benches/bench_file.toml b/benches/bench_file.toml
@@ -1,17 +1,3 @@
-
-
-# deterministic prng
-code = '''
-static uint32_t xorshift32(uint32_t *state) {
-    uint32_t x = *state;
-    x ^= x << 13;
-    x ^= x >> 17;
-    x ^= x << 5;
-    *state = x;
-    return x;
-}
-'''
-
 [cases.bench_file_read]
 # 0 = in-order
 # 1 = reversed-order
@@ -33,7 +19,7 @@ code = '''
     for (lfs_size_t i = 0; i < chunks; i++) {
         uint32_t chunk_prng = i;
         for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
-            buffer[j] = xorshift32(&chunk_prng);
+            buffer[j] = BENCH_PRNG(&chunk_prng);
         }
 
         lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
@@ -50,14 +36,14 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (chunks-1-i)
-            : xorshift32(&prng) % chunks;
+            : BENCH_PRNG(&prng) % chunks;
         lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET)
                 => i_*CHUNK_SIZE;
         lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
 
         uint32_t chunk_prng = i_;
         for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
-            assert(buffer[j] == xorshift32(&chunk_prng));
+            assert(buffer[j] == BENCH_PRNG(&chunk_prng));
         }
     }
 
@@ -91,10 +77,10 @@ code = '''
         lfs_off_t i_
             = (ORDER == 0) ? i
             : (ORDER == 1) ? (chunks-1-i)
-            : xorshift32(&prng) % chunks;
+            : BENCH_PRNG(&prng) % chunks;
         uint32_t chunk_prng = i_;
         for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
-            buffer[j] = xorshift32(&chunk_prng);
+            buffer[j] = BENCH_PRNG(&chunk_prng);
         }
 
         lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET)

diff --git a/runners/bench_runner.c b/runners/bench_runner.c
@@ -529,6 +529,19 @@ void bench_trace(const char *fmt, ...) {
 }
 
 
+// bench prng
+uint32_t bench_prng(uint32_t *state) {
+    // A simple xorshift32 generator, easily reproducible. Keep in mind
+    // determinism is much more important than actual randomness here.
+    uint32_t x = *state;
+    x ^= x << 13;
+    x ^= x >> 17;
+    x ^= x << 5;
+    *state = x;
+    return x;
+}
+
+
 // bench recording state
 static struct lfs_config *bench_cfg = NULL;
 static lfs_emubd_io_t bench_last_readed = 0;

diff --git a/runners/bench_runner.h b/runners/bench_runner.h
@@ -74,6 +74,12 @@ struct bench_suite {
 };
 
 
+// deterministic prng for pseudo-randomness in benches
+uint32_t bench_prng(uint32_t *state);
+
+#define BENCH_PRNG(state) bench_prng(state)
+
+
 // access generated bench defines
 intmax_t bench_define(size_t define);
 

diff --git a/runners/test_runner.c b/runners/test_runner.c
@@ -545,6 +545,19 @@ void test_trace(const char *fmt, ...) {
 }
 
 
+// test prng
+uint32_t test_prng(uint32_t *state) {
+    // A simple xorshift32 generator, easily reproducible. Keep in mind
+    // determinism is much more important than actual randomness here.
+    uint32_t x = *state;
+    x ^= x << 13;
+    x ^= x >> 17;
+    x ^= x << 5;
+    *state = x;
+    return x;
+}
+
+
 // encode our permutation into a reusable id
 static void perm_printid(
         const struct test_suite *suite,

diff --git a/runners/test_runner.h b/runners/test_runner.h
@@ -67,6 +67,12 @@ struct test_suite {
 };
 
 
+// deterministic prng for pseudo-randomness in testes
+uint32_t test_prng(uint32_t *state);
+
+#define TEST_PRNG(state) test_prng(state)
+
+
 // access generated test defines
 intmax_t test_define(size_t define);