feat(jobs): improve scheduled tasks with retry logic and queue cleanup

- Add retry configuration to CoolifyTask (3 tries, 600s timeout)
- Add retry configuration to ScheduledTaskJob (3 tries, configurable timeout)
- Add retry configuration to DatabaseBackupJob (2 tries)
- Implement exponential backoff for all jobs (30s, 60s, 120s intervals)
- Add failed() handlers with comprehensive error logging to scheduled-errors channel
- Add execution tracking: started_at, retry_count, duration (decimal), error_details
- Add configurable timeout field to scheduled tasks (60-3600s, default 300s)
- Update UI to include timeout configuration in task creation/editing forms
- Increase ScheduledJobManager lock expiration from 60s to 90s for high-load environments
- Implement safe queue cleanup with restart vs runtime modes
  - Restart mode: aggressive cleanup (marks all processing jobs as failed)
  - Runtime mode: conservative cleanup (only marks jobs >12h as failed, skips deployments)
- Add cleanup:redis --restart flag for system startup
- Integrate cleanup into Dev.php init() for development environment
- Increase scheduled-errors log retention from 7 to 14 days
- Create comprehensive test suite (unit and feature tests)
- Add TESTING_GUIDE.md with manual testing instructions

Fixes issues with jobs failing after single attempt and "attempted too many times" errors
This commit is contained in:
Andras Bacsai
2025-11-10 11:11:18 +01:00
parent 775216e7a5
commit b22e79caec
22 changed files with 762 additions and 37 deletions
@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('scheduled_tasks', function (Blueprint $table) {
$table->integer('timeout')->default(300)->after('frequency');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('scheduled_tasks', function (Blueprint $table) {
$table->dropColumn('timeout');
});
}
};
@@ -0,0 +1,31 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('scheduled_task_executions', function (Blueprint $table) {
$table->timestamp('started_at')->nullable()->after('scheduled_task_id');
$table->integer('retry_count')->default(0)->after('status');
$table->decimal('duration', 10, 2)->nullable()->after('retry_count')->comment('Duration in seconds');
$table->text('error_details')->nullable()->after('message');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('scheduled_task_executions', function (Blueprint $table) {
$table->dropColumn(['started_at', 'retry_count', 'duration', 'error_details']);
});
}
};