fix(servers): isolate cloud status checks from SSH checks

Track provider state independently, skip SSH work for placeholder IPs, and clean up failed cloud server provisioning.
2026-08-20 10:23:30 +00:00 · 2026-07-11 22:52:10 +02:00
parent e01b8a057e
commit c8a332a3bc
22 changed files with 954 additions and 180 deletions
@@ -12,6 +12,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;

 /**
@@ -53,11 +54,13 @@ class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBe
    /**
     * Get all functional servers to check for orphaned containers.
     */
-    private function getServersToCheck(): \Illuminate\Support\Collection
+    private function getServersToCheck(): Collection
    {
        $query = Server::whereRelation('settings', 'is_usable', true)
            ->whereRelation('settings', 'is_reachable', true)
-            ->where('ip', '!=', '1.2.3.4');
+            ->whereNotNull('ip')
+            ->where('ip', '!=', '')
+            ->whereNotIn('ip', Server::PLACEHOLDER_IPS);

        if (isCloud()) {
            $query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);
@@ -99,7 +102,7 @@ class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBe
    /**
     * Get all PR containers on a server (containers with pullRequestId > 0).
     */
-    private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection
+    private function getPRContainersOnServer(Server $server): Collection
    {
        try {
            $output = instant_remote_process([
@@ -457,7 +457,9 @@ class ScheduledJobManager implements ShouldQueue
    private function getServersForCleanupQuery(): Builder
    {
        $query = Server::with('settings')
-            ->where('ip', '!=', '1.2.3.4');
+            ->whereNotNull('ip')
+            ->where('ip', '!=', '')
+            ->whereNotIn('ip', Server::PLACEHOLDER_IPS);

        if (isCloud()) {
            $query
@@ -0,0 +1,59 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Server;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ServerCloudProviderStatusCheckJob implements ShouldBeEncrypted, ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 1;
+
+    public $timeout = 120;
+
+    public function __construct(public Server $server)
+    {
+        $this->onQueue('high');
+    }
+
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping('server-cloud-provider-status-'.$this->server->uuid))->expireAfter(130)->dontRelease()];
+    }
+
+    public function handle(): void
+    {
+        try {
+            if (! $this->server->cloudProviderToken) {
+                return;
+            }
+
+            match ($this->server->cloudProviderToken->provider) {
+                'hetzner' => $this->server->hetzner_server_id
+                    ? $this->server->refreshHetznerState()
+                    : null,
+                'vultr' => $this->server->vultr_instance_id
+                    ? $this->server->refreshVultrState()
+                    : null,
+                'digitalocean' => $this->server->digitalocean_droplet_id
+                    ? $this->server->refreshDigitalOceanState()
+                    : null,
+                default => null,
+            };
+        } catch (\Throwable $e) {
+            Log::debug('Cloud provider status check failed', [
+                'server_id' => $this->server->id,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+}
@@ -6,7 +6,6 @@ use App\Events\ServerReachabilityChanged;
 use App\Helpers\SshMultiplexingHelper;
 use App\Models\Server;
 use App\Services\ConfigurationRepository;
-use App\Services\HetznerService;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -42,8 +41,12 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
        $configRepository->disableSshMux();
    }

-    public function handle()
+    public function handle(): void
    {
+        if ($this->server->hasPlaceholderIp()) {
+            return;
+        }
+
        $wasReachable = (bool) $this->server->settings->is_reachable;
        $wasNotified = (bool) $this->server->unreachable_notification_sent;

@@ -62,19 +65,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                return;
            }

-            // Check Hetzner server status if applicable
-            if ($this->server->hetzner_server_id && $this->server->cloudProviderToken) {
-                $this->checkHetznerStatus();
-            }
-
-            if ($this->server->vultr_instance_id && $this->server->cloudProviderToken) {
-                $this->checkVultrStatus();
-            }
-
-            if ($this->server->digitalocean_droplet_id && $this->server->cloudProviderToken) {
-                $this->checkDigitalOceanStatus();
-            }
-
            // Temporarily disable mux if requested
            if ($this->disableMux) {
                $this->disableSshMux();
@@ -136,17 +126,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
    public function failed(?\Throwable $exception): void
    {
        if ($exception instanceof TimeoutExceededException) {
-            $wasReachable = (bool) $this->server->settings->is_reachable;
-            $wasNotified = (bool) $this->server->unreachable_notification_sent;
-
-            $this->server->settings->update([
-                'is_reachable' => false,
-                'is_usable' => false,
-            ]);
-            $this->server->increment('unreachable_count');
-
-            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
-
            // Delete the queue job so it doesn't appear in Horizon's failed list.
            $this->job?->delete();
        }
@@ -171,63 +150,6 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
        }
    }

-    private function checkHetznerStatus(): void
-    {
-        $status = null;
-
-        try {
-            $hetznerService = new HetznerService($this->server->cloudProviderToken->token);
-            $serverData = $hetznerService->getServer($this->server->hetzner_server_id);
-            $status = $serverData['status'] ?? null;
-
-        } catch (\Throwable) {
-            // Silently ignore — server may have been deleted from Hetzner.
-        }
-        if ($this->server->hetzner_server_status !== $status) {
-            $this->server->update(['hetzner_server_status' => $status]);
-            $this->server->hetzner_server_status = $status;
-            if ($status === 'off') {
-                throw new \Exception('Server is powered off');
-            }
-        }
-
-    }
-
-    private function checkVultrStatus(): void
-    {
-        try {
-            $status = $this->server->refreshVultrState();
-        } catch (\Throwable) {
-            // Silently ignore transient Vultr API errors.
-
-            return;
-        }
-
-        if (in_array($status, ['stopped', 'suspended', 'deleted'], true)) {
-            throw new \Exception('Vultr instance is not running');
-        }
-    }
-
-    private function checkDigitalOceanStatus(): void
-    {
-        try {
-            $status = $this->server->refreshDigitalOceanState();
-        } catch (\Throwable $e) {
-            Log::debug('ServerConnectionCheck: DigitalOcean status check failed', [
-                'server_id' => $this->server->id,
-                'error' => $e->getMessage(),
-            ]);
-
-            return;
-        }
-
-        $this->server->digitalocean_droplet_status = $status;
-
-        if (in_array($status, ['off', 'archive', 'deleted'], true)) {
-            throw new \Exception('DigitalOcean droplet is not running');
-        }
-    }
-
    private function checkConnection(): bool
    {
        try {
@@ -55,6 +55,13 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
        // Get all servers to process
        $servers = $this->getServers();

+        // Provider state checks run independently so slow APIs cannot block SSH checks.
+        $this->dispatchCloudProviderStatusChecks($servers);
+
+        $servers = $servers
+            ->reject(fn (Server $server) => $server->hasPlaceholderIp())
+            ->values();
+
        // Dispatch ServerConnectionCheck for all servers efficiently
        $this->dispatchConnectionChecks($servers);

@@ -64,24 +71,45 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue

    private function getServers(): Collection
    {
-        $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
+        $allServers = Server::with(['settings', 'cloudProviderToken']);

        if (isCloud()) {
            $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
-            $own = Team::find(0)->servers()->with('settings')->get();
+            $own = Team::find(0)->servers()->with(['settings', 'cloudProviderToken'])->get();

-            return $servers->merge($own);
+            return $servers->merge($own)->unique('id')->values();
        } else {
            return $allServers->get();
        }
    }

+    private function dispatchCloudProviderStatusChecks(Collection $servers): void
+    {
+        if (! shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime)) {
+            return;
+        }
+
+        $servers->each(function (Server $server) {
+            $hasCloudResource = $server->hetzner_server_id
+                || $server->vultr_instance_id
+                || $server->digitalocean_droplet_id;
+
+            if ($hasCloudResource && $server->cloudProviderToken) {
+                ServerCloudProviderStatusCheckJob::dispatch($server);
+            }
+        });
+    }
+
    private function dispatchConnectionChecks(Collection $servers): void
    {

        if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
            $servers->each(function (Server $server) {
                try {
+                    if ($server->hasPlaceholderIp()) {
+                        return;
+                    }
+
                    // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
                    if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
                        return;