From 8770b1daa99483cbaf03cb45aea98a28ae95ad69 Mon Sep 17 00:00:00 2001 From: Jeff Saremi Date: Sat, 25 Feb 2023 10:14:21 -0800 Subject: [PATCH 1/2] killing multiple workers on each iteration of reaper, if needed --- lib/puma_worker_killer/reaper.rb | 33 ++++++++++++++++++-------------- test/puma_worker_killer_test.rb | 1 + 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/puma_worker_killer/reaper.rb b/lib/puma_worker_killer/reaper.rb index 25d943b..045f473 100644 --- a/lib/puma_worker_killer/reaper.rb +++ b/lib/puma_worker_killer/reaper.rb @@ -21,22 +21,27 @@ def reap total = get_total_memory @on_calculation&.call(total) - if total > @max_ram - @cluster.master.log "PumaWorkerKiller: Out of memory. #{@cluster.workers.count} workers consuming total: #{total} mb out of max: #{@max_ram} mb. Sending TERM to pid #{@cluster.largest_worker.pid} consuming #{@cluster.largest_worker_memory} mb." - - # Fetch the largest_worker so that both `@pre_term` and `term_worker` are called with the same worker - # Avoids a race condition where: - # Worker A consume 100 mb memory - # Worker B consume 99 mb memory - # pre_term gets called with Worker A - # A new request comes in, Worker B takes it, and consumes 101 mb memory - # term_largest_worker (previously here) gets called and terms Worker B (thus not passing the about-to-be-terminated worker to `@pre_term`) - largest_worker = @cluster.largest_worker - @pre_term&.call(largest_worker) - @cluster.term_worker(largest_worker) + exceeded_memory = 0 + to_kill = [] + @cluster.workers.to_a.reverse_each do |item| + exceeded_memory += item[1] + to_kill << item + break if total - exceeded_memory < @max_ram + end + if total > @max_ram + log_entry = "PumaWorkerKiller: Out of memory. #{@cluster.workers.count} workers consuming total: #{total} mb out of max: #{@max_ram} mb. " \ + "Releasing #{exceeded_memory} mb from #{to_kill.length} workers." + to_kill.each do |item| + worker = item[0] + mem = item[1] + log_entry += "\r\n\tSending TERM to pid #{worker.pid} consuming #{mem} mb." + @pre_term&.call(worker) + @cluster.term_worker(worker) + end + @cluster.master.log(log_entry) elsif @reaper_status_logs - @cluster.master.log "PumaWorkerKiller: Consuming #{total} mb with master and #{@cluster.workers.count} workers." + @cluster.master.log("PumaWorkerKiller: Consuming #{total} mb with master and #{@cluster.workers.count} workers.") end end end diff --git a/test/puma_worker_killer_test.rb b/test/puma_worker_killer_test.rb index 4f87c08..93956d5 100644 --- a/test/puma_worker_killer_test.rb +++ b/test/puma_worker_killer_test.rb @@ -31,6 +31,7 @@ def test_kills_large_app WaitForIt.new(command, options) do |spawn| assert_contains(spawn, 'Out of memory') + assert_contains(spawn, /Releasing .* from 2 workers/) end end From 06b85f2334037fd7d9a7f1fef986bdfaaa6f5b29 Mon Sep 17 00:00:00 2001 From: Jeff Saremi Date: Sat, 25 Feb 2023 10:28:18 -0800 Subject: [PATCH 2/2] moving lines around --- lib/puma_worker_killer/reaper.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/puma_worker_killer/reaper.rb b/lib/puma_worker_killer/reaper.rb index 045f473..8dc9084 100644 --- a/lib/puma_worker_killer/reaper.rb +++ b/lib/puma_worker_killer/reaper.rb @@ -21,15 +21,15 @@ def reap total = get_total_memory @on_calculation&.call(total) - exceeded_memory = 0 - to_kill = [] - @cluster.workers.to_a.reverse_each do |item| - exceeded_memory += item[1] - to_kill << item - break if total - exceeded_memory < @max_ram - end - if total > @max_ram + exceeded_memory = 0 + to_kill = [] + @cluster.workers.to_a.reverse_each do |item| + exceeded_memory += item[1] + to_kill << item + break if total - exceeded_memory < @max_ram + end + log_entry = "PumaWorkerKiller: Out of memory. #{@cluster.workers.count} workers consuming total: #{total} mb out of max: #{@max_ram} mb. " \ "Releasing #{exceeded_memory} mb from #{to_kill.length} workers." to_kill.each do |item|