From 7d8dd57740e26076844f64877962738f875f7131 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 26 Jan 2023 16:27:45 +0100 Subject: [PATCH] feat(memory): JIRA-5517 Peak memory usage per job Add a peak memory usage histogram that is triggered on job proccesed and is tagged with the job. --- README.md | 13 +++++++ config/telemetry.php | 7 ++++ src/Enums/MeterName.php | 1 + src/Enums/MeterUnit.php | 1 + src/HorizonTelemetryServiceProvider.php | 12 +++++- .../ProcessedJobsPeakMemoryUsageListener.php | 39 +++++++++++++++++++ ...JobsPeakMemoryUsagePreparationListener.php | 22 +++++++++++ 7 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 src/Listeners/ProcessedJobsPeakMemoryUsageListener.php create mode 100644 src/Listeners/ProcessedJobsPeakMemoryUsagePreparationListener.php diff --git a/README.md b/README.md index c24d7a1..43fd90c 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,19 @@ The default schedule for this is `true`, to disable this event listener, add `MeterName::ProcessedJobs->value => false` under a `horizon` key in your `telemetry.php` config file. +#### [`ProcessedJobsPeakMemoryUsageListener`](src/Listeners/ProcessedJobsPeakMemoryUsageListener.php) + +The `ProcessedJobsPeakMemoryUsageListener` listener will create a histogram over peak memory usage each time a job is processed. +This metric will be registered under the name `horizon_processed_jobs_peak_memory_usage`. + +Two listeners are actually registered here, the secondary listener is [`ProcessedJobsPeakMemoryUsagePreparationListener`](src/Listeners/ProcessedJobsPeakMemoryUsagePreparationListener.php) +which will take care of clearing the peak memory usage before a job starts, so for long-running queue workers, we get +the correct number. + +The default schedule for this is `true`, to disable this event listener, +add `MeterName::ProcessedJobsPeakMemoryUsage->value => false` under a `horizon` key in your `telemetry.php` config file. + + ## Testing ```bash diff --git a/config/telemetry.php b/config/telemetry.php index b173779..3c518fc 100644 --- a/config/telemetry.php +++ b/config/telemetry.php @@ -46,5 +46,12 @@ * The default is `true`, set to `false` to disable this. */ MeterName::ProcessedJobs->value => true, + + /** + * Whether the processed job memory usage metric listener is enabled. + * + * The default is `true`, set to `false` to disable this. + */ + MeterName::ProcessedJobsPeakMemoryUsage->value => true, ], ]; diff --git a/src/Enums/MeterName.php b/src/Enums/MeterName.php index 0b22b20..e5baf5f 100644 --- a/src/Enums/MeterName.php +++ b/src/Enums/MeterName.php @@ -11,6 +11,7 @@ enum MeterName: string case CurrentJobs = 'horizon_current_jobs'; case FailedJobs = 'horizon_failed_jobs'; case ProcessedJobs = 'horizon_processed_jobs'; + case ProcessedJobsPeakMemoryUsage = 'horizon_processed_jobs_peak_memory_usage'; public function with(string ...$names): string { diff --git a/src/Enums/MeterUnit.php b/src/Enums/MeterUnit.php index 5076ead..80a97ce 100644 --- a/src/Enums/MeterUnit.php +++ b/src/Enums/MeterUnit.php @@ -9,4 +9,5 @@ enum MeterUnit: string case MasterSupervisors = 'master supervisors'; case Processes = 'processes'; case Jobs = 'jobs'; + case Bytes = 'bytes'; } diff --git a/src/HorizonTelemetryServiceProvider.php b/src/HorizonTelemetryServiceProvider.php index 772eb6c..7ac7e55 100644 --- a/src/HorizonTelemetryServiceProvider.php +++ b/src/HorizonTelemetryServiceProvider.php @@ -9,10 +9,13 @@ use Illuminate\Contracts\Events\Dispatcher; use Illuminate\Queue\Events\JobFailed; use Illuminate\Queue\Events\JobProcessed; +use Illuminate\Queue\Events\JobProcessing; use Illuminate\Support\ServiceProvider; use Worksome\HorizonTelemetry\Enums\MeterName; use Worksome\HorizonTelemetry\Listeners\FailedJobsListener; use Worksome\HorizonTelemetry\Listeners\ProcessedJobsListener; +use Worksome\HorizonTelemetry\Listeners\ProcessedJobsPeakMemoryUsageListener; +use Worksome\HorizonTelemetry\Listeners\ProcessedJobsPeakMemoryUsagePreparationListener; use Worksome\HorizonTelemetry\Metrics\CurrentJobsMetric; use Worksome\HorizonTelemetry\Metrics\CurrentMasterSupervisorsMetric; use Worksome\HorizonTelemetry\Metrics\CurrentProcessesMetric; @@ -29,13 +32,18 @@ public function boot(): void /** @var Repository $config */ $config = $this->app->make(Repository::class); - if ($config->get(self::CONFIG_PREFIX . MeterName::FailedJobs->value)) { + if ($config->get(self::CONFIG_PREFIX . MeterName::FailedJobs->value, true)) { $dispatcher->listen(JobFailed::class, FailedJobsListener::class); } - if ($config->get(self::CONFIG_PREFIX . MeterName::ProcessedJobs->value)) { + if ($config->get(self::CONFIG_PREFIX . MeterName::ProcessedJobs->value, true)) { $dispatcher->listen(JobProcessed::class, ProcessedJobsListener::class); } + + if ($config->get(self::CONFIG_PREFIX . MeterName::ProcessedJobsPeakMemoryUsage->value, true)) { + $dispatcher->listen(JobProcessed::class, ProcessedJobsPeakMemoryUsageListener::class); + $dispatcher->listen(JobProcessing::class, ProcessedJobsPeakMemoryUsagePreparationListener::class); + } }); $this->callAfterResolving(Schedule::class, function (Schedule $schedule) { diff --git a/src/Listeners/ProcessedJobsPeakMemoryUsageListener.php b/src/Listeners/ProcessedJobsPeakMemoryUsageListener.php new file mode 100644 index 0000000..fc5d347 --- /dev/null +++ b/src/Listeners/ProcessedJobsPeakMemoryUsageListener.php @@ -0,0 +1,39 @@ +meterProvider->getMeter(MeterName::ProcessedJobsPeakMemoryUsage); + + $histogram = $meter->createHistogram( + MeterName::ProcessedJobsPeakMemoryUsage->value, + MeterUnit::Bytes->value, + 'The memory usage per job', + ); + + + $histogram->record( + memory_get_peak_usage(), + [ + 'name' => $event->job->resolveName(), + 'queue' => $event->job->getQueue(), + ] + ); + } +} diff --git a/src/Listeners/ProcessedJobsPeakMemoryUsagePreparationListener.php b/src/Listeners/ProcessedJobsPeakMemoryUsagePreparationListener.php new file mode 100644 index 0000000..6b5dddb --- /dev/null +++ b/src/Listeners/ProcessedJobsPeakMemoryUsagePreparationListener.php @@ -0,0 +1,22 @@ +