mirror of
https://github.com/tiennm99/coolify.git
synced 2026-10-03 05:19:37 +00:00
feat(jobs): improve scheduled tasks with retry logic and queue cleanup
- Add retry configuration to CoolifyTask (3 tries, 600s timeout) - Add retry configuration to ScheduledTaskJob (3 tries, configurable timeout) - Add retry configuration to DatabaseBackupJob (2 tries) - Implement exponential backoff for all jobs (30s, 60s, 120s intervals) - Add failed() handlers with comprehensive error logging to scheduled-errors channel - Add execution tracking: started_at, retry_count, duration (decimal), error_details - Add configurable timeout field to scheduled tasks (60-3600s, default 300s) - Update UI to include timeout configuration in task creation/editing forms - Increase ScheduledJobManager lock expiration from 60s to 90s for high-load environments - Implement safe queue cleanup with restart vs runtime modes - Restart mode: aggressive cleanup (marks all processing jobs as failed) - Runtime mode: conservative cleanup (only marks jobs >12h as failed, skips deployments) - Add cleanup:redis --restart flag for system startup - Integrate cleanup into Dev.php init() for development environment - Increase scheduled-errors log retention from 7 to 14 days - Create comprehensive test suite (unit and feature tests) - Add TESTING_GUIDE.md with manual testing instructions Fixes issues with jobs failing after single attempt and "attempted too many times" errors
This commit is contained in:
1 parent
775216e7a5
commit
b22e79caec
22 files changed
+762
-37
No files matched your search
@@ -0,0 +1,70 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CoolifyTask;
|
||||
use App\Models\Server;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
it('can dispatch CoolifyTask successfully', function () {
|
||||
// Skip if no servers available
|
||||
$server = Server::where('ip', '!=', '1.2.3.4')->first();
|
||||
|
||||
if (! $server) {
|
||||
$this->markTestSkipped('No servers available for testing');
|
||||
}
|
||||
|
||||
Queue::fake();
|
||||
|
||||
// Create an activity for the task
|
||||
$activity = activity()
|
||||
->withProperties([
|
||||
'server_uuid' => $server->uuid,
|
||||
'command' => 'echo "test"',
|
||||
'type' => 'inline',
|
||||
])
|
||||
->event('inline')
|
||||
->log('[]');
|
||||
|
||||
// Dispatch the job
|
||||
CoolifyTask::dispatch(
|
||||
activity: $activity,
|
||||
ignore_errors: false,
|
||||
call_event_on_finish: null,
|
||||
call_event_data: null
|
||||
);
|
||||
|
||||
// Assert job was dispatched
|
||||
Queue::assertPushed(CoolifyTask::class);
|
||||
});
|
||||
|
||||
it('has correct retry configuration on CoolifyTask', function () {
|
||||
$server = Server::where('ip', '!=', '1.2.3.4')->first();
|
||||
|
||||
if (! $server) {
|
||||
$this->markTestSkipped('No servers available for testing');
|
||||
}
|
||||
|
||||
$activity = activity()
|
||||
->withProperties([
|
||||
'server_uuid' => $server->uuid,
|
||||
'command' => 'echo "test"',
|
||||
'type' => 'inline',
|
||||
])
|
||||
->event('inline')
|
||||
->log('[]');
|
||||
|
||||
$job = new CoolifyTask(
|
||||
activity: $activity,
|
||||
ignore_errors: false,
|
||||
call_event_on_finish: null,
|
||||
call_event_data: null
|
||||
);
|
||||
|
||||
// Assert retry configuration
|
||||
expect($job->tries)->toBe(3);
|
||||
expect($job->maxExceptions)->toBe(1);
|
||||
expect($job->timeout)->toBe(600);
|
||||
expect($job->backoff())->toBe([30, 90, 180]);
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CoolifyTask;
|
||||
use App\Jobs\DatabaseBackupJob;
|
||||
use App\Jobs\ScheduledTaskJob;
|
||||
|
||||
it('CoolifyTask has correct retry properties defined', function () {
|
||||
$reflection = new ReflectionClass(CoolifyTask::class);
|
||||
|
||||
// Check public properties exist
|
||||
expect($reflection->hasProperty('tries'))->toBeTrue()
|
||||
->and($reflection->hasProperty('maxExceptions'))->toBeTrue()
|
||||
->and($reflection->hasProperty('timeout'))->toBeTrue()
|
||||
->and($reflection->hasMethod('backoff'))->toBeTrue();
|
||||
|
||||
// Get default values from class definition
|
||||
$defaultProperties = $reflection->getDefaultProperties();
|
||||
|
||||
expect($defaultProperties['tries'])->toBe(3)
|
||||
->and($defaultProperties['maxExceptions'])->toBe(1)
|
||||
->and($defaultProperties['timeout'])->toBe(600);
|
||||
});
|
||||
|
||||
it('ScheduledTaskJob has correct retry properties defined', function () {
|
||||
$reflection = new ReflectionClass(ScheduledTaskJob::class);
|
||||
|
||||
// Check public properties exist
|
||||
expect($reflection->hasProperty('tries'))->toBeTrue()
|
||||
->and($reflection->hasProperty('maxExceptions'))->toBeTrue()
|
||||
->and($reflection->hasProperty('timeout'))->toBeTrue()
|
||||
->and($reflection->hasMethod('backoff'))->toBeTrue()
|
||||
->and($reflection->hasMethod('failed'))->toBeTrue();
|
||||
|
||||
// Get default values from class definition
|
||||
$defaultProperties = $reflection->getDefaultProperties();
|
||||
|
||||
expect($defaultProperties['tries'])->toBe(3)
|
||||
->and($defaultProperties['maxExceptions'])->toBe(1)
|
||||
->and($defaultProperties['timeout'])->toBe(300);
|
||||
});
|
||||
|
||||
it('DatabaseBackupJob has correct retry properties defined', function () {
|
||||
$reflection = new ReflectionClass(DatabaseBackupJob::class);
|
||||
|
||||
// Check public properties exist
|
||||
expect($reflection->hasProperty('tries'))->toBeTrue()
|
||||
->and($reflection->hasProperty('maxExceptions'))->toBeTrue()
|
||||
->and($reflection->hasMethod('backoff'))->toBeTrue()
|
||||
->and($reflection->hasMethod('failed'))->toBeTrue();
|
||||
|
||||
// Get default values from class definition
|
||||
$defaultProperties = $reflection->getDefaultProperties();
|
||||
|
||||
expect($defaultProperties['tries'])->toBe(2)
|
||||
->and($defaultProperties['maxExceptions'])->toBe(1);
|
||||
});
|
||||
Reference in new issue
Block a user