Retry jobs which fail or have gone missing

Model/CleanRunningJobs.php

@@ -5,19 +5,28 @@
 
 use EthanYehuda\CronjobManager\Api\Data\ScheduleInterface;
 use EthanYehuda\CronjobManager\Api\ScheduleRepositoryAdapterInterface;
+use EthanYehuda\CronjobManager\Scope\Config;
 use Magento\Cron\Model\ResourceModel\Schedule\CollectionFactory;
 use Magento\Cron\Model\Schedule;
+use Magento\Cron\Model\ScheduleFactory;
 use Magento\Framework\Stdlib\DateTime\DateTime;
 
 /**
  * Update jobs with dead processes from running to error
  */
 class CleanRunningJobs
 {
+    /** @var Config */
+    private $config;
+
     /**
      * @var ProcessManagement
      */
     private $processManagement;
+
+    /** @var ScheduleFactory */
+    private $scheduleFactory;
+
     /**
      * @var ScheduleRepositoryAdapterInterface
      */
@@ -32,11 +41,15 @@ class CleanRunningJobs
     private $dateTime;
 
     public function __construct(
+        Config $config,
+        ScheduleFactory $scheduleFactory,
         ScheduleRepositoryAdapterInterface $scheduleRepository,
         ProcessManagement $processManagement,
         DateTime $dateTime,
         Clock $clock
     ) {
+        $this->config = $config;
+        $this->scheduleFactory = $scheduleFactory;
         $this->processManagement = $processManagement;
         $this->scheduleRepository = $scheduleRepository;
         $this->dateTime = $dateTime;
@@ -73,6 +86,16 @@ public function execute()
                 ->setMessages(implode("\n", $messages));
 
             $this->scheduleRepository->save($schedule);
+
+            // We check 'scheduled_at' to avoid scheduling jobs run via command line
+            if ($schedule->getScheduledAt() && $this->config->isRetryJobsGoneAway()) {
+                $this->scheduleFactory->create()
+                    ->setJobCode($schedule->getJobCode())
+                    ->setStatus(Schedule::STATUS_PENDING)
+                    ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S'))
+                    ->setScheduledAt(strftime('%Y-%m-%d %H:%M:%S'))
+                    ->save();
+            }
         }
     }
 }

Plugin/Cron/Model/ScheduleResourcePlugin.php

@@ -4,27 +4,33 @@
 namespace EthanYehuda\CronjobManager\Plugin\Cron\Model;
 
 use EthanYehuda\CronjobManager\Model\ErrorNotification;
-use Magento\Cron\Model\ConfigInterface;
+use EthanYehuda\CronjobManager\Scope\Config;
 use Magento\Cron\Model\ResourceModel\Schedule as ScheduleResource;
 use Magento\Cron\Model\Schedule;
+use Magento\Cron\Model\ScheduleFactory;
 use Magento\Framework\DataObject;
 
 class ScheduleResourcePlugin
 {
+    /** @var Config */
+    private $config;
+
     /**
      * @var ErrorNotification
      */
     private $errorNotification;
 
-    /** @var ConfigInterface */
-    private $config;
+    /** @var ScheduleFactory */
+    private $scheduleFactory;
 
     public function __construct(
-        ConfigInterface $config,
-        ErrorNotification $errorNotification
+        Config $config,
+        ErrorNotification $errorNotification,
+        ScheduleFactory $scheduleFactory
     ) {
         $this->config = $config;
         $this->errorNotification = $errorNotification;
+        $this->scheduleFactory = $scheduleFactory;
     }
 
     public function beforeSave(
@@ -75,7 +81,7 @@ protected function recordJobGroup(DataObject $dataObject): void
     }
 
     /**
-     * Email notification if status has been set to ERROR
+     * Email notification and job retry if status has been set to ERROR
      */
     public function afterSave(
         ScheduleResource $subject,
@@ -86,6 +92,16 @@ public function afterSave(
             && $object->getStatus() === Schedule::STATUS_ERROR
         ) {
             $this->errorNotification->sendFor($object);
+
+            // We check 'scheduled_at' to avoid scheduling jobs run via command line
+            if ($object->getScheduledAt() && $this->config->isRetryFailedJobs()) {
+                $this->scheduleFactory->create()
+                    ->setJobCode($object->getJobCode())
+                    ->setStatus(Schedule::STATUS_PENDING)
+                    ->setCreatedAt(strftime('%Y-%m-%d %H:%M:%S'))
+                    ->setScheduledAt(strftime('%Y-%m-%d %H:%M:%S'))
+                    ->save();
+            }
         }
         return $result;
     }

Scope/Config.php

@@ -0,0 +1,47 @@
+<?php
+namespace EthanYehuda\CronjobManager\Scope;
+
+use Magento\Cron\Model\ConfigInterface;
+use Magento\Framework\App\Config\ScopeConfigInterface;
+
+class Config
+{
+    protected const RETRY_FAILED_JOBS = 'system/cron_job_manager/retry_failed_jobs';
+    protected const RETRY_JOBS_GONE_AWAY = 'system/cron_job_manager/retry_jobs_gone_away';
+
+    /** @var ConfigInterface */
+    protected $magentoConfig;
+
+    /** @var ScopeConfigInterface */
+    protected $scopeConfig;
+
+    public function __construct(
+        ConfigInterface $magentoConfig,
+        ScopeConfigInterface $scopeConfig
+    ) {
+        $this->magentoConfig = $magentoConfig;
+        $this->scopeConfig = $scopeConfig;
+    }
+
+    public function getJobs():array
+    {
+        return $this->magentoConfig->getJobs();
+    }
+
+    public function isRetryFailedJobs(): bool
+    {
+        return $this->scopeConfig->isSetFlag(self::RETRY_FAILED_JOBS);
+    }
+
+    public function isRetryJobsGoneAway(): bool
+    {
+        if ($this->isRetryFailedJobs()) {
+            // If we are to retry all jobs, then we don't also want to retry
+            // jobs which have been detected as missing/gone away. Doing so
+            // would create duplicate entries in the schedule.
+            return false;
+        }
+
+        return $this->scopeConfig->isSetFlag(self::RETRY_JOBS_GONE_AWAY);
+    }
+}

etc/adminhtml/system.xml

@@ -10,6 +10,15 @@
                     <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
                     <comment>If a process has status "running", but there is no corresponding process alive on the server, the job will be set to "error". This prevents buildup of false "running" jobs.</comment>
                 </field>
+                <field id="retry_jobs_gone_away" translate="label comment" type="select" sortOrder="15" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
+                    <label>Retry jobs where process has gone missing</label>
+                    <comment>When enabled, any jobs which have been detected as missing (see above) will be scheduled anew for immediate execution</comment>
+                    <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
+                    <depends>
+                        <field id="clean_running_schedule">1</field>
+                        <field id="retry_failed_jobs">0</field>
+                    </depends>
+                </field>
                 <field id="email_notification" translate="label comment" type="select" sortOrder="100" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
                     <label>Send notification email if job has error</label>
                     <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
@@ -35,6 +44,11 @@
                         <field id="email_notification">1</field>
                     </depends>
                 </field>
+                <field id="retry_failed_jobs" translate="label comment" type="select" sortOrder="200" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
+                    <label>Retry jobs which fail</label>
+                    <comment>When enabled, any jobs which result in an "error" status will be scheduled anew for immediate execution</comment>
+                    <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
+                </field>
             </group>
         </section>
     </system>

etc/config.xml

@@ -9,7 +9,9 @@
                 <email_identity>general</email_identity>
                 <email_recipients></email_recipients>
                 <email_template>system_cron_job_manager_email_template</email_template>
+                <retry_failed_jobs>0</retry_failed_jobs>
+                <retry_jobs_gone_away>1</retry_jobs_gone_away>
             </cron_job_manager>
         </system>
     </default>
-</config>
+</config>