feat(jobs): improve scheduled tasks with retry logic and queue cleanup

- Add retry configuration to CoolifyTask (3 tries, 600s timeout)
- Add retry configuration to ScheduledTaskJob (3 tries, configurable timeout)
- Add retry configuration to DatabaseBackupJob (2 tries)
- Implement exponential backoff for all jobs (30s, 60s, 120s intervals)
- Add failed() handlers with comprehensive error logging to scheduled-errors channel
- Add execution tracking: started_at, retry_count, duration (decimal), error_details
- Add configurable timeout field to scheduled tasks (60-3600s, default 300s)
- Update UI to include timeout configuration in task creation/editing forms
- Increase ScheduledJobManager lock expiration from 60s to 90s for high-load environments
- Implement safe queue cleanup with restart vs runtime modes
  - Restart mode: aggressive cleanup (marks all processing jobs as failed)
  - Runtime mode: conservative cleanup (only marks jobs >12h as failed, skips deployments)
- Add cleanup:redis --restart flag for system startup
- Integrate cleanup into Dev.php init() for development environment
- Increase scheduled-errors log retention from 7 to 14 days
- Create comprehensive test suite (unit and feature tests)
- Add TESTING_GUIDE.md with manual testing instructions

Fixes issues with jobs failing after single attempt and "attempted too many times" errors
This commit is contained in:
Andras Bacsai committed 2025-11-10 11:11:18 +01:00
1 parent 775216e7a5
commit b22e79caec
22 files changed
+762 -37

No files matched your search

+70
View File
@@ -0,0 +1,70 @@
<?php
use App\Jobs\CoolifyTask;
use App\Models\Server;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
it('can dispatch CoolifyTask successfully', function () {
// Skip if no servers available
$server = Server::where('ip', '!=', '1.2.3.4')->first();
if (! $server) {
$this->markTestSkipped('No servers available for testing');
}
Queue::fake();
// Create an activity for the task
$activity = activity()
->withProperties([
'server_uuid' => $server->uuid,
'command' => 'echo "test"',
'type' => 'inline',
])
->event('inline')
->log('[]');
// Dispatch the job
CoolifyTask::dispatch(
activity: $activity,
ignore_errors: false,
call_event_on_finish: null,
call_event_data: null
);
// Assert job was dispatched
Queue::assertPushed(CoolifyTask::class);
});
it('has correct retry configuration on CoolifyTask', function () {
$server = Server::where('ip', '!=', '1.2.3.4')->first();
if (! $server) {
$this->markTestSkipped('No servers available for testing');
}
$activity = activity()
->withProperties([
'server_uuid' => $server->uuid,
'command' => 'echo "test"',
'type' => 'inline',
])
->event('inline')
->log('[]');
$job = new CoolifyTask(
activity: $activity,
ignore_errors: false,
call_event_on_finish: null,
call_event_data: null
);
// Assert retry configuration
expect($job->tries)->toBe(3);
expect($job->maxExceptions)->toBe(1);
expect($job->timeout)->toBe(600);
expect($job->backoff())->toBe([30, 90, 180]);
});
@@ -0,0 +1,56 @@
<?php
use App\Jobs\CoolifyTask;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\ScheduledTaskJob;
it('CoolifyTask has correct retry properties defined', function () {
$reflection = new ReflectionClass(CoolifyTask::class);
// Check public properties exist
expect($reflection->hasProperty('tries'))->toBeTrue()
->and($reflection->hasProperty('maxExceptions'))->toBeTrue()
->and($reflection->hasProperty('timeout'))->toBeTrue()
->and($reflection->hasMethod('backoff'))->toBeTrue();
// Get default values from class definition
$defaultProperties = $reflection->getDefaultProperties();
expect($defaultProperties['tries'])->toBe(3)
->and($defaultProperties['maxExceptions'])->toBe(1)
->and($defaultProperties['timeout'])->toBe(600);
});
it('ScheduledTaskJob has correct retry properties defined', function () {
$reflection = new ReflectionClass(ScheduledTaskJob::class);
// Check public properties exist
expect($reflection->hasProperty('tries'))->toBeTrue()
->and($reflection->hasProperty('maxExceptions'))->toBeTrue()
->and($reflection->hasProperty('timeout'))->toBeTrue()
->and($reflection->hasMethod('backoff'))->toBeTrue()
->and($reflection->hasMethod('failed'))->toBeTrue();
// Get default values from class definition
$defaultProperties = $reflection->getDefaultProperties();
expect($defaultProperties['tries'])->toBe(3)
->and($defaultProperties['maxExceptions'])->toBe(1)
->and($defaultProperties['timeout'])->toBe(300);
});
it('DatabaseBackupJob has correct retry properties defined', function () {
$reflection = new ReflectionClass(DatabaseBackupJob::class);
// Check public properties exist
expect($reflection->hasProperty('tries'))->toBeTrue()
->and($reflection->hasProperty('maxExceptions'))->toBeTrue()
->and($reflection->hasMethod('backoff'))->toBeTrue()
->and($reflection->hasMethod('failed'))->toBeTrue();
// Get default values from class definition
$defaultProperties = $reflection->getDefaultProperties();
expect($defaultProperties['tries'])->toBe(2)
->and($defaultProperties['maxExceptions'])->toBe(1);
});