fix(jobs): handle queue timeouts gracefully in Horizon (#8360)

app/Jobs/ServerCheckJob.php

@@ -15,6 +15,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 
 class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 {
@@ -33,6 +34,19 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function __construct(public Server $server) {}
 
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
+        }
+    }
+
     public function handle()
     {
         try {

app/Jobs/ServerConnectionCheckJob.php

@@ -101,7 +101,24 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                 'is_usable' => false,
             ]);
 
-            throw $e;
+            return;
+        }
+    }
+
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerConnectionCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+            $this->server->settings->update([
+                'is_reachable' => false,
+                'is_usable' => false,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
         }
     }
 

app/Jobs/ServerStorageCheckJob.php

@@ -10,6 +10,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\RateLimiter;
 use Laravel\Horizon\Contracts\Silenced;
 
@@ -28,6 +29,19 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
 
     public function __construct(public Server $server, public int|string|null $percentage = null) {}
 
+    public function failed(?\Throwable $exception): void
+    {
+        if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+            Log::warning('ServerStorageCheckJob timed out', [
+                'server_id' => $this->server->id,
+                'server_name' => $this->server->name,
+            ]);
+
+            // Delete the queue job so it doesn't appear in Horizon's failed list.
+            $this->job?->delete();
+        }
+    }
+
     public function handle()
     {
         try {