mirror of
https://github.com/tiennm99/coolify.git
synced 2026-04-17 17:21:04 +00:00
feat(jobs): improve scheduled tasks with retry logic and queue cleanup
- Add retry configuration to CoolifyTask (3 tries, 600s timeout) - Add retry configuration to ScheduledTaskJob (3 tries, configurable timeout) - Add retry configuration to DatabaseBackupJob (2 tries) - Implement exponential backoff for all jobs (30s, 60s, 120s intervals) - Add failed() handlers with comprehensive error logging to scheduled-errors channel - Add execution tracking: started_at, retry_count, duration (decimal), error_details - Add configurable timeout field to scheduled tasks (60-3600s, default 300s) - Update UI to include timeout configuration in task creation/editing forms - Increase ScheduledJobManager lock expiration from 60s to 90s for high-load environments - Implement safe queue cleanup with restart vs runtime modes - Restart mode: aggressive cleanup (marks all processing jobs as failed) - Runtime mode: conservative cleanup (only marks jobs >12h as failed, skips deployments) - Add cleanup:redis --restart flag for system startup - Integrate cleanup into Dev.php init() for development environment - Increase scheduled-errors log retention from 7 to 14 days - Create comprehensive test suite (unit and feature tests) - Add TESTING_GUIDE.md with manual testing instructions Fixes issues with jobs failing after single attempt and "attempted too many times" errors
This commit is contained in:
70
tests/Feature/CoolifyTaskRetryTest.php
Normal file
70
tests/Feature/CoolifyTaskRetryTest.php
Normal file
@@ -0,0 +1,70 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\CoolifyTask;
|
||||
use App\Models\Server;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
it('can dispatch CoolifyTask successfully', function () {
|
||||
// Skip if no servers available
|
||||
$server = Server::where('ip', '!=', '1.2.3.4')->first();
|
||||
|
||||
if (! $server) {
|
||||
$this->markTestSkipped('No servers available for testing');
|
||||
}
|
||||
|
||||
Queue::fake();
|
||||
|
||||
// Create an activity for the task
|
||||
$activity = activity()
|
||||
->withProperties([
|
||||
'server_uuid' => $server->uuid,
|
||||
'command' => 'echo "test"',
|
||||
'type' => 'inline',
|
||||
])
|
||||
->event('inline')
|
||||
->log('[]');
|
||||
|
||||
// Dispatch the job
|
||||
CoolifyTask::dispatch(
|
||||
activity: $activity,
|
||||
ignore_errors: false,
|
||||
call_event_on_finish: null,
|
||||
call_event_data: null
|
||||
);
|
||||
|
||||
// Assert job was dispatched
|
||||
Queue::assertPushed(CoolifyTask::class);
|
||||
});
|
||||
|
||||
it('has correct retry configuration on CoolifyTask', function () {
|
||||
$server = Server::where('ip', '!=', '1.2.3.4')->first();
|
||||
|
||||
if (! $server) {
|
||||
$this->markTestSkipped('No servers available for testing');
|
||||
}
|
||||
|
||||
$activity = activity()
|
||||
->withProperties([
|
||||
'server_uuid' => $server->uuid,
|
||||
'command' => 'echo "test"',
|
||||
'type' => 'inline',
|
||||
])
|
||||
->event('inline')
|
||||
->log('[]');
|
||||
|
||||
$job = new CoolifyTask(
|
||||
activity: $activity,
|
||||
ignore_errors: false,
|
||||
call_event_on_finish: null,
|
||||
call_event_data: null
|
||||
);
|
||||
|
||||
// Assert retry configuration
|
||||
expect($job->tries)->toBe(3);
|
||||
expect($job->maxExceptions)->toBe(1);
|
||||
expect($job->timeout)->toBe(600);
|
||||
expect($job->backoff())->toBe([30, 90, 180]);
|
||||
});
|
||||
Reference in New Issue
Block a user