mirror of
https://github.com/tiennm99/coolify.git
synced 2026-08-18 18:23:17 +00:00
feat(jobs): implement exponential backoff for unreachable servers
Reduce load on unreachable servers by implementing exponential backoff during connectivity failures. Check frequency decreases based on consecutive failure count: 0-2: every cycle 3-5: ~15 min intervals 6-11: ~30 min intervals 12+: ~60 min intervals Uses server ID hash to distribute checks across cycles and prevent thundering herd. ServerCheckJob and ServerConnectionCheckJob increment unreachable_count on failures. ServerManagerJob applies backoff logic before dispatching checks. Includes comprehensive test coverage.
This commit is contained in:
@@ -15,6 +15,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\TimeoutExceededException;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
@@ -36,11 +37,12 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
|
||||
public function failed(?\Throwable $exception): void
|
||||
{
|
||||
if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
|
||||
if ($exception instanceof TimeoutExceededException) {
|
||||
Log::warning('ServerCheckJob timed out', [
|
||||
'server_id' => $this->server->id,
|
||||
'server_name' => $this->server->name,
|
||||
]);
|
||||
$this->server->increment('unreachable_count');
|
||||
|
||||
// Delete the queue job so it doesn't appear in Horizon's failed list.
|
||||
$this->job?->delete();
|
||||
|
||||
Reference in New Issue
Block a user