tiennm99/coolify
1
0
Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-08-23 18:25:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(jobs): implement exponential backoff for unreachable servers

Browse Source 
Reduce load on unreachable servers by implementing exponential backoff
during connectivity failures. Check frequency decreases based on
consecutive failure count:
  0-2: every cycle
  3-5: ~15 min intervals
  6-11: ~30 min intervals
  12+: ~60 min intervals

Uses server ID hash to distribute checks across cycles and prevent
thundering herd.

ServerCheckJob and ServerConnectionCheckJob increment unreachable_count
on failures. ServerManagerJob applies backoff logic before dispatching
checks. Includes comprehensive test coverage.
This commit is contained in:
Andras Bacsai
2026-03-26 10:51:36 +01:00
parent d77e4c864f
commit dd2c9c291a
4 changed files with 245 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Jobs/ServerManagerJob.php
+41 -1
View File
@@ -86,6 +86,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
return;
}
if ($this->shouldSkipDueToBackoff($server)) {
return;
}
ServerConnectionCheckJob::dispatch($server);
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
@@ -129,7 +132,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
if ($sentinelOutOfSync) {
// Dispatch ServerCheckJob if Sentinel is out of sync
if (shouldRunCronNow($this->checkFrequency, $serverTimezone, "server-check:{$server->id}", $this->executionTime)) {
ServerCheckJob::dispatch($server);
if (! $this->shouldSkipDueToBackoff($server)) {
ServerCheckJob::dispatch($server);
}
}
}
@@ -165,4 +170,39 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
// Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
// Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
}
/**
* Determine the backoff cycle interval based on how many consecutive times a server has been unreachable.
* Higher counts → less frequent checks (based on 5-min cloud cycle):
* 0-2: every cycle, 3-5: ~15 min, 6-11: ~30 min, 12+: ~60 min
*/
private function getBackoffCycleInterval(int $unreachableCount): int
{
return match (true) {
$unreachableCount <= 2 => 1,
$unreachableCount <= 5 => 3,
$unreachableCount <= 11 => 6,
default => 12,
};
}
/**
* Check if a server should be skipped this cycle due to unreachable backoff.
* Uses server ID hash to distribute checks across cycles (avoid thundering herd).
*/
private function shouldSkipDueToBackoff(Server $server): bool
{
$unreachableCount = $server->unreachable_count ?? 0;
$interval = $this->getBackoffCycleInterval($unreachableCount);
if ($interval <= 1) {
return false;
}
$cyclePeriodMinutes = isCloud() ? 5 : 1;
$cycleIndex = intdiv($this->executionTime->minute, $cyclePeriodMinutes);
$serverHash = abs(crc32((string) $server->id));
return ($cycleIndex + $serverHash) % $interval !== 0;
}
}