feat(Cleanup): implement failure marking for stuck scheduled tasks and database backups during startup

This commit is contained in:
Andras Bacsai
2025-11-11 12:32:52 +01:00
parent 64c7d301ce
commit 0cfce06869
4 changed files with 394 additions and 0 deletions

View File

@@ -4,6 +4,9 @@ namespace App\Console\Commands;
use App\Jobs\CheckHelperImageJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledTaskExecution;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Artisan;
@@ -55,6 +58,34 @@ class Dev extends Command
echo "Error in cleanup:redis: {$e->getMessage()}\n";
}
try {
$updatedTaskCount = ScheduledTaskExecution::where('status', 'running')->update([
'status' => 'failed',
'message' => 'Marked as failed during Coolify startup - job was interrupted',
'finished_at' => Carbon::now(),
]);
if ($updatedTaskCount > 0) {
echo "Marked {$updatedTaskCount} stuck scheduled task executions as failed\n";
}
} catch (\Throwable $e) {
echo "Could not cleanup stuck scheduled task executions: {$e->getMessage()}\n";
}
try {
$updatedBackupCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([
'status' => 'failed',
'message' => 'Marked as failed during Coolify startup - job was interrupted',
'finished_at' => Carbon::now(),
]);
if ($updatedBackupCount > 0) {
echo "Marked {$updatedBackupCount} stuck database backup executions as failed\n";
}
} catch (\Throwable $e) {
echo "Could not cleanup stuck database backup executions: {$e->getMessage()}\n";
}
CheckHelperImageJob::dispatch();
}
}

View File

@@ -10,9 +10,12 @@ use App\Models\ApplicationDeploymentQueue;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledTaskExecution;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\User;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\File;
@@ -103,6 +106,34 @@ class Init extends Command
echo "Could not cleanup inprogress deployments: {$e->getMessage()}\n";
}
try {
$updatedTaskCount = ScheduledTaskExecution::where('status', 'running')->update([
'status' => 'failed',
'message' => 'Marked as failed during Coolify startup - job was interrupted',
'finished_at' => Carbon::now(),
]);
if ($updatedTaskCount > 0) {
echo "Marked {$updatedTaskCount} stuck scheduled task executions as failed\n";
}
} catch (\Throwable $e) {
echo "Could not cleanup stuck scheduled task executions: {$e->getMessage()}\n";
}
try {
$updatedBackupCount = ScheduledDatabaseBackupExecution::where('status', 'running')->update([
'status' => 'failed',
'message' => 'Marked as failed during Coolify startup - job was interrupted',
'finished_at' => Carbon::now(),
]);
if ($updatedBackupCount > 0) {
echo "Marked {$updatedBackupCount} stuck database backup executions as failed\n";
}
} catch (\Throwable $e) {
echo "Could not cleanup stuck database backup executions: {$e->getMessage()}\n";
}
try {
$localhost = $this->servers->where('id', 0)->first();
if ($localhost) {