tiennm99/coolify
1
0
Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-09-10 06:20:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(jobs): add queue delay resilience to scheduled job execution

Browse Source 
Implement dedup key-based cron tracking to make scheduled jobs resilient to queue
delays. Even if a job is delayed by minutes, it will catch the missed cron window
by tracking previousRunDate in cache instead of relying on isDue() alone.

- Add dedupKey parameter to shouldRunNow() in ScheduledJobManager
  - When provided, uses getPreviousRunDate() + cache tracking for resilience
  - Falls back to isDue() for docker cleanups without dedup key
  - Prevents double-dispatch within same cron window

- Optimize ServerConnectionCheckJob dispatch
  - Skip SSH checks if Sentinel is healthy (enabled and live)
  - Reduces redundant checks when Sentinel heartbeat proves connectivity

- Remove hourly Sentinel update checks
  - Consolidate to daily CheckAndStartSentinelJob dispatch
  - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow

- Add logging for skipped database backups with context (backup_id, database_id, status)

- Refactor skip reason methods to accept server parameter, avoiding redundant queries

- Add comprehensive test suite for scheduling with various delay scenarios and timezones
This commit is contained in:
Andras Bacsai
2026-02-28 15:06:25 +01:00
parent f68793ed69
commit a0c177f6f2
5 changed files with 322 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Jobs/ServerManagerJob.php
+9 -10
View File
@@ -64,11 +64,11 @@ class ServerManagerJob implements ShouldQueue
private function getServers(): Collection
{
$allServers = Server::where('ip', '!=', '1.2.3.4');
$allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
if (isCloud()) {
$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
$own = Team::find(0)->servers;
$own = Team::find(0)->servers()->with('settings')->get();
return $servers->merge($own);
} else {
@@ -82,6 +82,10 @@ class ServerManagerJob implements ShouldQueue
if ($this->shouldRunNow($this->checkFrequency)) {
$servers->each(function (Server $server) {
try {
// Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
return;
}
ServerConnectionCheckJob::dispatch($server);
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
@@ -134,9 +138,7 @@ class ServerManagerJob implements ShouldQueue
// Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
if ($shouldRestartSentinel) {
dispatch(function () use ($server) {
$server->restartContainer('coolify-sentinel');
});
CheckAndStartSentinelJob::dispatch($server);
}
// Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled)
@@ -160,11 +162,8 @@ class ServerManagerJob implements ShouldQueue
ServerPatchCheckJob::dispatch($server);
}
// Sentinel update checks (hourly) - check for updates to Sentinel version
// No timezone needed for hourly - runs at top of every hour
if ($isSentinelEnabled && $this->shouldRunNow('0 * * * *')) {
CheckAndStartSentinelJob::dispatch($server);
}
// Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
// Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
}
private function shouldRunNow(string $frequency, ?string $timezone = null): bool