don't block unless shutdown is allowed

src/libponyrt/sched/scheduler.c

@@ -45,6 +45,7 @@ static PONY_ATOMIC(uint32_t) active_scheduler_count_check;
 static scheduler_t* scheduler;
 static PONY_ATOMIC(bool) temporarily_disable_scheduler_scaling;
 static PONY_ATOMIC(bool) detect_quiescence;
+static PONY_ATOMIC(bool) shutdown_allowed;
 static bool use_yield;
 static mpmcq_t inject;
 static __pony_thread_local scheduler_t* this_scheduler;
@@ -887,7 +888,10 @@ static pony_actor_t* steal(scheduler_t* sched)
         steal_attempts++;
       }
       else if ((clocks_elapsed > PONY_SCHED_BLOCK_THRESHOLD) &&
-        (ponyint_mutemap_size(&sched->mute_mapping) == 0))
+        (ponyint_mutemap_size(&sched->mute_mapping) == 0) &&
+        // only if shutdown is allowed because there's a race between startup
+        // and termination signalling to wake sleeping threads otherwise
+        atomic_load_explicit(&shutdown_allowed, memory_order_relaxed))
       {
         // only considered blocked if we're scheduler > 0 or if we're scheduler
         // 0 and there are no noiisy actors registered
@@ -1325,6 +1329,9 @@ bool ponyint_sched_start(bool library)
 
   atomic_store_explicit(&detect_quiescence, !library, memory_order_relaxed);
 
+  // disable shutdown until all threads are started and running
+  atomic_store_explicit(&shutdown_allowed, true, memory_order_relaxed);
+
   DTRACE0(RT_START);
   uint32_t start = 0;
 
@@ -1341,6 +1348,9 @@ bool ponyint_sched_start(bool library)
       return false;
   }
 
+  // enable shutdown now that all threads are started and running
+  atomic_store_explicit(&shutdown_allowed, true, memory_order_relaxed);
+
 #if defined(USE_SYSTEMATIC_TESTING)
   // start processing
   SYSTEMATIC_TESTING_START(scheduler, ponyint_asio_get_backend_tid(), ponyint_asio_get_backend_sleep_object());