Execution results are presented in case of any failure (#314)

Execution results are presented in case of any failure

src/jvm/main/org/jetbrains/kotlinx/lincheck/execution/ExecutionResult.kt

@@ -20,7 +20,7 @@ data class ExecutionResult(
      * Results of the initial sequential part of the execution.
      * @see ExecutionScenario.initExecution
      */
-    val initResults: List<Result>,
+    val initResults: List<Result?>,
     /**
      * State representation at the end of the init part.
      */
@@ -38,13 +38,13 @@ data class ExecutionResult(
      * Results of the last sequential part of the execution.
      * @see ExecutionScenario.postExecution
      */
-    val postResults: List<Result>,
+    val postResults: List<Result?>,
     /**
      * State representation at the end of the scenario.
      */
     val afterPostStateRepresentation: String?
 ) {
-    constructor(initResults: List<Result>, parallelResultsWithClock: List<List<ResultWithClock>>, postResults: List<Result>) :
+    constructor(initResults: List<Result?>, parallelResultsWithClock: List<List<ResultWithClock>>, postResults: List<Result?>) :
         this(initResults, null, parallelResultsWithClock, null, postResults, null)
 
     /**
@@ -139,10 +139,10 @@ val ExecutionResult.withEmptyClocks: ExecutionResult get() = ExecutionResult(
     this.afterPostStateRepresentation
 )
 
-val ExecutionResult.parallelResults: List<List<Result>> get() =
+val ExecutionResult.parallelResults: List<List<Result?>> get() =
     parallelResultsWithClock.map { it.map { r -> r.result } }
 
-val ExecutionResult.threadsResults: List<List<Result>> get() =
+val ExecutionResult.threadsResults: List<List<Result?>> get() =
     threadsResultsWithClock.map { it.map { r -> r.result } }
 
 // for tests

src/jvm/main/org/jetbrains/kotlinx/lincheck/execution/HBClock.kt

@@ -31,8 +31,8 @@ data class HBClock(val clock: IntArray) {
 fun emptyClock(size: Int) = HBClock(emptyClockArray(size))
 fun emptyClockArray(size: Int) = IntArray(size) { 0 }
 
-data class ResultWithClock(val result: Result, val clockOnStart: HBClock)
+data class ResultWithClock(val result: Result?, val clockOnStart: HBClock)
 
 fun Result.withEmptyClock(threads: Int) = ResultWithClock(this, emptyClock(threads))
 fun List<Result>.withEmptyClock(threads: Int): List<ResultWithClock> = map { it.withEmptyClock(threads) }
-fun List<ResultWithClock>.withEmptyClock() = map { it.result.withEmptyClock(it.clockOnStart.threads) }
+fun List<ResultWithClock>.withEmptyClock() = mapNotNull { it.result?.withEmptyClock(it.clockOnStart.threads) }

src/jvm/main/org/jetbrains/kotlinx/lincheck/runner/InvocationResult.kt

@@ -27,20 +27,22 @@ class CompletedInvocationResult(
 /**
  * Indicates that the invocation has run into deadlock or livelock found by [ManagedStrategy].
  */
-data object ManagedDeadlockInvocationResult : InvocationResult()
+class ManagedDeadlockInvocationResult(val results: ExecutionResult) : InvocationResult()
 
 /**
  * The invocation was not completed after timeout and runner halted the execution.
  */
 class RunnerTimeoutInvocationResult(
     val threadDump: Map<Thread, Array<StackTraceElement>>,
+    val results: ExecutionResult
 ): InvocationResult()
 
 /**
  * The invocation has completed with an unexpected exception.
  */
 class UnexpectedExceptionInvocationResult(
-    val exception: Throwable
+    val exception: Throwable,
+    val results: ExecutionResult
 ) : InvocationResult()
 
 /**
@@ -50,18 +52,20 @@ class UnexpectedExceptionInvocationResult(
  */
 class ValidationFailureInvocationResult(
     val scenario: ExecutionScenario,
-    val exception: Throwable
+    val exception: Throwable,
+    val results: ExecutionResult
 ) : InvocationResult()
 
 /**
  * Obstruction freedom check is requested,
  * but an invocation that hangs has been found.
  */
 class ObstructionFreedomViolationInvocationResult(
-    val reason: String
+    val reason: String,
+    val results: ExecutionResult
 ) : InvocationResult()
 
 /**
  * Indicates that spin-cycle has been found for the first time and replay of current interleaving is required.
  */
-object SpinCycleFoundAndReplayRequired: InvocationResult()
+data object SpinCycleFoundAndReplayRequired: InvocationResult()

src/jvm/main/org/jetbrains/kotlinx/lincheck/runner/ParallelThreadsRunner.kt

@@ -308,35 +308,46 @@ internal open class ParallelThreadsRunner(
                 executor.submitAndAwait(arrayOf(validationPart), timeout)
                 val validationResult = validationPart.results.single()
                 if (validationResult is ExceptionResult) {
-                    return ValidationFailureInvocationResult(scenario, validationResult.throwable)
+                    return ValidationFailureInvocationResult(scenario, validationResult.throwable, collectExecutionResults())
                 }
             }
             // Combine the results and convert them for the standard class loader (if they are of non-primitive types).
             // We do not want the transformed code to be reachable outside of the runner and strategy classes.
-            return CompletedInvocationResult(
-                ExecutionResult(
-                    initResults = initialPartExecution?.results?.toList().orEmpty(),
-                    parallelResultsWithClock = parallelPartExecutions.map { execution ->
-                        execution.results.zip(execution.clocks).map {
-                            ResultWithClock(it.first, HBClock(it.second.clone()))
-                        }
-                    },
-                    postResults = postPartExecution?.results?.toList().orEmpty(),
-                    afterInitStateRepresentation = afterInitStateRepresentation,
-                    afterParallelStateRepresentation = afterParallelStateRepresentation,
-                    afterPostStateRepresentation = afterPostStateRepresentation
-                )
-            )
+            return CompletedInvocationResult(collectExecutionResults(afterInitStateRepresentation, afterParallelStateRepresentation, afterPostStateRepresentation))
         } catch (e: TimeoutException) {
             val threadDump = collectThreadDump(this)
-            return RunnerTimeoutInvocationResult(threadDump)
+            return RunnerTimeoutInvocationResult(threadDump, collectExecutionResults())
         } catch (e: ExecutionException) {
-            return UnexpectedExceptionInvocationResult(e.cause!!)
+            return UnexpectedExceptionInvocationResult(e.cause!!, collectExecutionResults())
         } finally {
             resetState()
         }
     }
 
+    /**
+     * This method is called when we have some execution result other than [CompletedInvocationResult].
+     */
+    fun collectExecutionResults(): ExecutionResult {
+        return collectExecutionResults(null, null, null)
+    }
+
+    private fun collectExecutionResults(
+        afterInitStateRepresentation: String?,
+        afterParallelStateRepresentation: String?,
+        afterPostStateRepresentation: String?
+    ) = ExecutionResult(
+        initResults = initialPartExecution?.results?.toList().orEmpty(),
+        parallelResultsWithClock = parallelPartExecutions.map { execution ->
+            execution.results.zip(execution.clocks).map {
+                ResultWithClock(it.first, HBClock(it.second.clone()))
+            }
+        },
+        postResults = postPartExecution?.results?.toList().orEmpty(),
+        afterInitStateRepresentation = afterInitStateRepresentation,
+        afterParallelStateRepresentation = afterParallelStateRepresentation,
+        afterPostStateRepresentation = afterPostStateRepresentation
+    )
+
 
     private fun createInitialPartExecution() =
         if (scenario.initExecution.isNotEmpty()) {

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/LincheckFailure.kt

@@ -16,50 +16,59 @@ import org.jetbrains.kotlinx.lincheck.strategy.managed.*
 
 sealed class LincheckFailure(
     val scenario: ExecutionScenario,
+    val results: ExecutionResult,
     val trace: Trace?
 ) {
     override fun toString() = StringBuilder().appendFailure(this).toString()
 }
 
 internal class IncorrectResultsFailure(
     scenario: ExecutionScenario,
-    val results: ExecutionResult,
+    results: ExecutionResult,
     trace: Trace? = null
-) : LincheckFailure(scenario, trace)
+) : LincheckFailure(scenario, results, trace)
 
-internal class DeadlockOrLivelockFailure(
+internal class ManagedDeadlockFailure(
     scenario: ExecutionScenario,
-    // Thread dump is not present in case of model checking
-    val threadDump: Map<Thread, Array<StackTraceElement>>?,
+    results: ExecutionResult,
     trace: Trace? = null
-) : LincheckFailure(scenario, trace)
+) : LincheckFailure(scenario,results, trace)
+
+internal class TimeoutFailure(
+    scenario: ExecutionScenario,
+    results: ExecutionResult,
+    val threadDump: Map<Thread, Array<StackTraceElement>>,
+) : LincheckFailure(scenario,results, null)
 
 internal class UnexpectedExceptionFailure(
     scenario: ExecutionScenario,
+    results: ExecutionResult,
     val exception: Throwable,
     trace: Trace? = null
-) : LincheckFailure(scenario, trace)
+) : LincheckFailure(scenario,results, trace)
 
 internal class ValidationFailure(
     scenario: ExecutionScenario,
+    results: ExecutionResult,
     val exception: Throwable,
     trace: Trace? = null
-) : LincheckFailure(scenario, trace) {
+) : LincheckFailure(scenario,results, trace) {
     val validationFunctionName: String = scenario.validationFunction!!.method.name
 }
 
 internal class ObstructionFreedomViolationFailure(
     scenario: ExecutionScenario,
+    results: ExecutionResult,
     val reason: String,
     trace: Trace? = null
-) : LincheckFailure(scenario, trace)
+) : LincheckFailure(scenario, results, trace)
 
 internal fun InvocationResult.toLincheckFailure(scenario: ExecutionScenario, trace: Trace? = null) = when (this) {
-    is ManagedDeadlockInvocationResult -> DeadlockOrLivelockFailure(scenario, threadDump = null, trace)
-    is RunnerTimeoutInvocationResult -> DeadlockOrLivelockFailure(scenario, threadDump, trace = null)
-    is UnexpectedExceptionInvocationResult -> UnexpectedExceptionFailure(scenario, exception, trace)
-    is ValidationFailureInvocationResult -> ValidationFailure(scenario, exception, trace)
-    is ObstructionFreedomViolationInvocationResult -> ObstructionFreedomViolationFailure(scenario, reason, trace)
+    is ManagedDeadlockInvocationResult -> ManagedDeadlockFailure(scenario, results, trace)
+    is RunnerTimeoutInvocationResult -> TimeoutFailure(scenario, results, threadDump)
+    is UnexpectedExceptionInvocationResult -> UnexpectedExceptionFailure(scenario, results, exception, trace)
+    is ValidationFailureInvocationResult -> ValidationFailure(scenario, results, exception, trace)
+    is ObstructionFreedomViolationInvocationResult -> ObstructionFreedomViolationFailure(scenario, results, reason, trace)
     is CompletedInvocationResult -> IncorrectResultsFailure(scenario, results, trace)
     else -> error("Unexpected invocation result type: ${this.javaClass.simpleName}")
 }

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/ManagedStrategy.kt

@@ -272,13 +272,13 @@ abstract class ManagedStrategy(
     }
 
     private fun failDueToDeadlock(): Nothing {
-        suddenInvocationResult = ManagedDeadlockInvocationResult
+        suddenInvocationResult = ManagedDeadlockInvocationResult(runner.collectExecutionResults())
         // Forcibly finish the current execution by throwing an exception.
         throw ForcibleExecutionFinishError
     }
 
     private fun failDueToLivelock(lazyMessage: () -> String): Nothing {
-        suddenInvocationResult = ObstructionFreedomViolationInvocationResult(lazyMessage())
+        suddenInvocationResult = ObstructionFreedomViolationInvocationResult(lazyMessage(), runner.collectExecutionResults())
         // Forcibly finish the current execution by throwing an exception.
         throw ForcibleExecutionFinishError
     }
@@ -440,7 +440,7 @@ abstract class ManagedStrategy(
         // the managed strategy can construct a trace to reproduce this failure.
         // Let's then store the corresponding failing result and construct the trace.
         if (exception === ForcibleExecutionFinishError) return // not a forcible execution finish
-        suddenInvocationResult = UnexpectedExceptionInvocationResult(exception)
+        suddenInvocationResult = UnexpectedExceptionInvocationResult(exception, runner.collectExecutionResults())
     }
 
     override fun onActorStart(iThread: Int) = runInIgnoredSection {
@@ -499,7 +499,7 @@ abstract class ManagedStrategy(
                 val nextThread = (0 until nThreads).firstOrNull { !finished[it] && isSuspended[it] }
                 if (nextThread == null) {
                     // must switch not to get into a deadlock, but there are no threads to switch.
-                    suddenInvocationResult = ManagedDeadlockInvocationResult
+                    suddenInvocationResult = ManagedDeadlockInvocationResult(runner.collectExecutionResults())
                     // forcibly finish execution by throwing an exception.
                     throw ForcibleExecutionFinishError
                 }

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/TraceReporter.kt

@@ -12,16 +12,18 @@ package org.jetbrains.kotlinx.lincheck.strategy.managed
 import org.jetbrains.kotlinx.lincheck.*
 import org.jetbrains.kotlinx.lincheck.execution.*
 import org.jetbrains.kotlinx.lincheck.runner.ExecutionPart
-import org.jetbrains.kotlinx.lincheck.strategy.DeadlockOrLivelockFailure
-import org.jetbrains.kotlinx.lincheck.strategy.LincheckFailure
+import org.jetbrains.kotlinx.lincheck.strategy.*
+import org.jetbrains.kotlinx.lincheck.strategy.ManagedDeadlockFailure
+import org.jetbrains.kotlinx.lincheck.strategy.ObstructionFreedomViolationFailure
+import org.jetbrains.kotlinx.lincheck.strategy.TimeoutFailure
 import org.jetbrains.kotlinx.lincheck.strategy.ValidationFailure
 import java.util.*
 import kotlin.math.*
 
 @Synchronized // we should avoid concurrent executions to keep `objectNumeration` consistent
 internal fun StringBuilder.appendTrace(
     failure: LincheckFailure,
-    results: ExecutionResult?,
+    results: ExecutionResult,
     trace: Trace,
     exceptionStackTraces: Map<Throwable, ExceptionNumberAndStacktrace>
 ) {
@@ -48,7 +50,7 @@ private fun StringBuilder.appendShortTrace(
     val traceRepresentation = traceGraphToRepresentationList(sectionsFirstNodes, false)
     appendLine(TRACE_TITLE)
     appendTraceRepresentation(failure.scenario, traceRepresentation)
-    if (failure is DeadlockOrLivelockFailure) {
+    if (failure is ManagedDeadlockFailure || failure is TimeoutFailure) {
         appendLine(ALL_UNFINISHED_THREADS_IN_DEADLOCK_MESSAGE)
     }
     appendLine()
@@ -64,7 +66,7 @@ private fun StringBuilder.appendDetailedTrace(
     appendLine(DETAILED_TRACE_TITLE)
     val traceRepresentationVerbose = traceGraphToRepresentationList(sectionsFirstNodes, true)
     appendTraceRepresentation(failure.scenario, traceRepresentationVerbose)
-    if (failure is DeadlockOrLivelockFailure) {
+    if (failure is ManagedDeadlockFailure || failure is TimeoutFailure) {
         appendLine(ALL_UNFINISHED_THREADS_IN_DEADLOCK_MESSAGE)
     }
 }
@@ -127,7 +129,7 @@ class TableSectionColumnsRepresentation(
  */
 internal fun constructTraceGraph(
     failure: LincheckFailure,
-    results: ExecutionResult?,
+    results: ExecutionResult,
     trace: Trace,
     exceptionStackTraces: Map<Throwable, ExceptionNumberAndStacktrace>
 ): List<TraceNode> {
@@ -177,8 +179,7 @@ internal fun constructTraceGraph(
                     last = lastNode,
                     callDepth = 0,
                     actorRepresentation = actorRepresentations[iThread][nextActor],
-                    resultRepresentation = resultProvider[iThread, nextActor]
-                        ?.let { actorNodeResultRepresentation(it, exceptionStackTraces) }
+                    resultRepresentation = actorNodeResultRepresentation(resultProvider[iThread, nextActor], failure, exceptionStackTraces)
                 )
             }
             actorNodes[iThread][nextActor] = actorNode
@@ -220,7 +221,7 @@ internal fun constructTraceGraph(
                     last = lastNode,
                     callDepth = 0,
                     actorRepresentation = actorRepresentations[iThread][actorId],
-                    resultRepresentation = actorNodeResultRepresentation(actorResult, exceptionStackTraces)
+                    resultRepresentation = actorNodeResultRepresentation(actorResult, failure, exceptionStackTraces)
                 )
                 actorNodes[iThread][actorId] = actorNode
                 traceGraphNodes += actorNode
@@ -251,6 +252,20 @@ internal fun constructTraceGraph(
     return traceGraphNodesSections.map { it.first() }
 }
 
+private fun actorNodeResultRepresentation(result: Result?, failure: LincheckFailure, exceptionStackTraces: Map<Throwable, ExceptionNumberAndStacktrace>): String? {
+    // We don't mark actors that violated obstruction freedom as hung.
+    if (result == null && failure is ObstructionFreedomViolationFailure) return null
+    return when (result) {
+        null -> "<hung>"
+        is ExceptionResult -> {
+            val exceptionNumberRepresentation = exceptionStackTraces[result.throwable]?.let { " #${it.number}" } ?: ""
+            "$result$exceptionNumberRepresentation"
+        }
+        is VoidResult -> null // don't print
+        else -> result.toString()
+    }
+}
+
 /**
  * Helper class to provider execution results, including a validation function result
  */
@@ -259,24 +274,21 @@ private class ExecutionResultsProvider(result: ExecutionResult?, failure: Linche
     /**
      * A map of type Map<(threadId, actorId) -> Result>
      */
-    private val threadNumberToActorResultMap: Map<Pair<Int, Int>, Result> = when {
-        // If the results of the failure are present, then just collect them to a map.
-        // In that case, we know that the failure reason is not validation function, so we ignore it.
-        (result != null) -> {
-            result.threadsResults
+    private val threadNumberToActorResultMap: Map<Pair<Int, Int>, Result?>
+
+    init {
+        val results = hashMapOf<Pair<Int, Int>, Result?>()
+        if (result != null) {
+            results += result.threadsResults
                 .flatMapIndexed { tId, actors -> actors.flatMapIndexed { actorId, result ->
                     listOf((tId to actorId) to result)
                 }}
                 .toMap()
         }
-
-        // If validation function is the reason if the failure then the only result we're interested in
-        // is the validation function exception.
-        failure is ValidationFailure -> {
-            mapOf((0 to firstThreadActorCount(failure)) to ExceptionResult.create(failure.exception, false))
+        if (failure is ValidationFailure) {
+            results[0 to firstThreadActorCount(failure)] = ExceptionResult.create(failure.exception, false)
         }
-
-        else -> emptyMap()
+        threadNumberToActorResultMap = results
     }
 
     operator fun get(iThread: Int, actorId: Int): Result? {