When a test fails suggest adding the failing scenario as a custom one

build.gradle.kts

@@ -16,6 +16,7 @@ plugins {
     kotlin("multiplatform")
     id("maven-publish")
     id("kotlinx.team.infra") version "0.3.0-dev-64"
+    kotlin("plugin.serialization") version "1.6.0"
 }
 
 repositories {
@@ -44,6 +45,7 @@ kotlin {
             val kotlinxCoroutinesVersion: String by project
             val asmVersion: String by project
             val reflectionsVersion: String by project
+            val kotlinxSerialisationJsonVersion: String by project
             dependencies {
                 api("org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion")
                 api("org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion")
@@ -52,6 +54,7 @@ kotlin {
                 api("org.ow2.asm:asm-commons:$asmVersion")
                 api("org.ow2.asm:asm-util:$asmVersion")
                 api("org.reflections:reflections:$reflectionsVersion")
+                api("org.jetbrains.kotlinx:kotlinx-serialization-json:$kotlinxSerialisationJsonVersion")
             }
         }
 

gradle.properties

@@ -15,6 +15,7 @@ version=2.22-SNAPSHOT
 inceptionYear=2019
 lastCopyrightYear=2023
 
+kotlinxSerialisationJsonVersion=1.3.0
 kotlinVersion=1.6.21
 kotlinxCoroutinesVersion=1.6.4
 asmVersion=9.4

src/jvm/main/org/jetbrains/kotlinx/lincheck/CTestConfiguration.kt

@@ -38,8 +38,12 @@ abstract class CTestConfiguration(
     val customScenarios: List<ExecutionScenario>
 ) {
     abstract fun createStrategy(
-        testClass: Class<*>, scenario: ExecutionScenario, validationFunctions: List<Method>,
-        stateRepresentationMethod: Method?, verifier: Verifier
+        testClass: Class<*>,
+        scenario: ExecutionScenario,
+        validationFunctions: List<Method>,
+        stateRepresentationMethod: Method?,
+        verifier: Verifier,
+        reproduceSettingsFactory: ReproduceSettingsFactory
     ): Strategy
 
     companion object {

src/jvm/main/org/jetbrains/kotlinx/lincheck/CTestStructure.java

@@ -10,6 +10,8 @@
 
 package org.jetbrains.kotlinx.lincheck;
 
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.jetbrains.kotlinx.lincheck.annotations.*;
 import org.jetbrains.kotlinx.lincheck.execution.*;
 import org.jetbrains.kotlinx.lincheck.paramgen.*;
@@ -48,13 +50,13 @@ private CTestStructure(List<ActorGenerator> actorGenerators, List<ParameterGener
     /**
      * Constructs {@link CTestStructure} for the specified test class.
      */
-    public static CTestStructure getFromTestClass(Class<?> testClass) {
+    public static CTestStructure getFromTestClass(Class<?> testClass, Options<?, ?> options) {
         Map<String, OperationGroup> groupConfigs = new HashMap<>();
         List<ActorGenerator> actorGenerators = new ArrayList<>();
         List<Method> validationFunctions = new ArrayList<>();
         List<Method> stateRepresentations = new ArrayList<>();
         Class<?> clazz = testClass;
-        RandomProvider randomProvider = new RandomProvider();
+        RandomProvider randomProvider = createRandomProvider(options);
         Map<Class<?>, ParameterGenerator<?>> parameterGeneratorsMap = new HashMap<>();
 
         while (clazz != null) {
@@ -267,6 +269,15 @@ private static ParameterGenerator<?> getOrCreateGenerator(Method m,
         }
     }
 
+    @NotNull
+    private static RandomProvider createRandomProvider(@Nullable Options<?, ?> options) {
+        if (options == null || options.getReproduceSettings() == null) {
+            return new RandomProvider();
+        }
+
+        return new RandomProvider(options.getReproduceSettings().getRandomSeedGeneratorSeed());
+    }
+
     private static ParameterGenerator<?> createGenerator(Param paramAnn, RandomProvider randomProvider) {
         try {
             return paramAnn.gen().getConstructor(RandomProvider.class, String.class).newInstance(randomProvider, paramAnn.conf());

src/jvm/main/org/jetbrains/kotlinx/lincheck/LinChecker.kt

@@ -19,9 +19,10 @@ import kotlin.reflect.*
  * This class runs concurrent tests.
  */
 class LinChecker (private val testClass: Class<*>, options: Options<*, *>?) {
-    private val testStructure = CTestStructure.getFromTestClass(testClass)
+    private val testStructure = CTestStructure.getFromTestClass(testClass, options)
     private val testConfigurations: List<CTestConfiguration>
     private val reporter: Reporter
+    private val reproduceSettingsFactory = ReproduceSettingsFactory(testStructure.randomProvider.seedGeneratorSeed)
 
     init {
         val logLevel = options?.logLevel ?: testClass.getAnnotation(LogLevel::class.java)?.value ?: DEFAULT_LOG_LEVEL
@@ -117,7 +118,8 @@ class LinChecker (private val testClass: Class<*>, options: Options<*, *>?) {
             scenario = this,
             validationFunctions = testStructure.validationFunctions,
             stateRepresentationMethod = testStructure.stateRepresentation,
-            verifier = verifier
+            verifier = verifier,
+            reproduceSettingsFactory = reproduceSettingsFactory
         ).run()
 
     private fun CTestConfiguration.createVerifier() =

src/jvm/main/org/jetbrains/kotlinx/lincheck/Options.kt

@@ -29,6 +29,8 @@ abstract class Options<OPT : Options<OPT, CTEST>, CTEST : CTestConfiguration> {
     protected var sequentialSpecification: Class<*>? = null
     protected var timeoutMs: Long = CTestConfiguration.DEFAULT_TIMEOUT_MS
     protected var customScenarios: MutableList<ExecutionScenario> = mutableListOf()
+    var reproduceSettings: ReproduceSettings? = null
+        private set
 
     /**
      * Number of different test scenarios to be executed
@@ -152,6 +154,13 @@ abstract class Options<OPT : Options<OPT, CTEST>, CTEST : CTestConfiguration> {
     fun addCustomScenario(scenarioBuilder: DSLScenarioBuilder.() -> Unit) =
         addCustomScenario(scenario { scenarioBuilder() })
 
+    /**
+     * Used to fixate run options to reproduce certain scenario results
+     */
+    fun withReproduceSettings(configuration: String): OPT = applyAndCast {
+        reproduceSettings = ConfigurationStringEncoder.decodeReproduceSettings(configuration)
+    }
+
     /**
      * Internal, DO NOT USE.
      */

src/jvm/main/org/jetbrains/kotlinx/lincheck/RandomProvider.kt

@@ -12,15 +12,25 @@ package org.jetbrains.kotlinx.lincheck
 
 import java.util.Random
 
-private const val SEED_GENERATOR_SEED = 0L
-
 /**
  * Used to provide [Random] with different seeds to parameters generators and method generator
  * Is being created every time on each test to make an execution deterministic.
  */
 class RandomProvider {
+    /**
+     * This field is exposed to provide it as information to reproduce the failure
+     */
+    val seedGeneratorSeed: Long
+    private val seedGenerator: Random = Random()
 
-    private val seedGenerator = Random(SEED_GENERATOR_SEED)
+    constructor() {
+        seedGeneratorSeed = seedGenerator.nextLong()
+        seedGenerator.setSeed(seedGeneratorSeed)
+    }
 
+    constructor(seedGeneratorSeed: Long) {
+        this.seedGeneratorSeed = seedGeneratorSeed
+        seedGenerator.setSeed(seedGeneratorSeed)
+    }
     fun createRandom(): Random = Random(seedGenerator.nextLong())
 }

src/jvm/main/org/jetbrains/kotlinx/lincheck/Reporter.kt

@@ -10,6 +10,7 @@
 
 package org.jetbrains.kotlinx.lincheck
 
+import org.jetbrains.kotlinx.lincheck.ConfigurationStringEncoder.encodeToConfigurationString
 import org.jetbrains.kotlinx.lincheck.LoggingLevel.*
 import org.jetbrains.kotlinx.lincheck.execution.*
 import org.jetbrains.kotlinx.lincheck.runner.*
@@ -355,7 +356,7 @@ internal fun StringBuilder.appendFailure(failure: LincheckFailure): StringBuilde
         when (val exceptionsProcessingResult = collectExceptionStackTraces(results)) {
             // If some exception was thrown from the Lincheck itself, we ask for bug reporting
             is InternalLincheckBugResult -> {
-                appendInternalLincheckBugFailure(exceptionsProcessingResult.exception)
+                appendInternalLincheckBugFailure(exceptionsProcessingResult.exception, failure.reproduceSettings)
                 return this
             }
 
@@ -368,8 +369,12 @@ internal fun StringBuilder.appendFailure(failure: LincheckFailure): StringBuilde
         is DeadlockWithDumpFailure -> appendDeadlockWithDumpFailure(failure)
         is UnexpectedExceptionFailure -> appendUnexpectedExceptionFailure(failure)
         is ValidationFailure -> when (failure.exception) {
-            is LincheckInternalBugException -> appendInternalLincheckBugFailure(failure.exception)
-            else ->  appendValidationFailure(failure)
+            is LincheckInternalBugException -> appendInternalLincheckBugFailure(
+                failure.exception,
+                failure.reproduceSettings
+            )
+
+            else -> appendValidationFailure(failure)
         }
         is ObstructionFreedomViolationFailure -> appendObstructionFreedomViolationFailure(failure)
     }
@@ -379,6 +384,16 @@ internal fun StringBuilder.appendFailure(failure: LincheckFailure): StringBuilde
     } else {
         appendExceptionsStackTracesBlock(exceptionStackTraces)
     }
+    appendLine()
+    appendLine()
+    appendLine("= To reproduce exactly this test execution with the same scenarios, you can add this setting in your testing options configuration =")
+    append(".withReproduceSettings(\"${encodeToConfigurationString(failure.reproduceSettings)}\")")
+    determineTestLanguage(Thread.currentThread().stackTrace)?.let { testLanguage ->
+        appendLine()
+        appendLine()
+        appendLine("= You can add this scenario as a custom test. Insert this code in your testing options configuration =")
+        append(generateAddCustomScenarioBlock(failure.scenario, testLanguage))
+    }
     return this
 }
 
@@ -403,12 +418,14 @@ internal fun StringBuilder.appendExceptionsStackTraces(exceptionStackTraces: Map
     return this
 }
 
-fun StringBuilder.appendInternalLincheckBugFailure(exception: Throwable) {
+fun StringBuilder.appendInternalLincheckBugFailure(exception: Throwable, reproduceSettings: ReproduceSettings) {
     appendLine(
         """
         Wow! You've caught a bug in Lincheck.
         We kindly ask to provide an issue here: https://github.com/JetBrains/lincheck/issues,
-        attaching a stack trace printed below and the code that causes the error.
+        attaching a stack trace printed below, reproduce settings line and the code that causes the error.
+
+        Reproduce settings line: ${encodeToConfigurationString(reproduceSettings)}
 
         Exception stacktrace:
     """.trimIndent()