$attributes */ function createReconcileServer(array $attributes = []): V5Server { return V5Server::query()->create([ 'team_id' => 1, 'created_by_user_id' => 1, 'name' => 'edge-01', 'host' => '203.0.113.10', 'ssh_user' => 'root', 'ssh_port' => 22, 'status' => 'installed', 'capabilities' => ['coold'], 'wireguard_management_ip' => '100.64.0.10', ...$attributes, ]); } /** * @param array $attributes */ function createReconcileApplication(V5Server $server, array $attributes = []): V5Application { return V5Application::query()->create([ 'team_id' => $server->team_id, 'project_id' => 1, 'environment_id' => 1, 'server_id' => $server->id, 'created_by_user_id' => 1, 'name' => 'nginx-test', 'image' => 'docker.io/library/nginx:alpine', 'container_name' => 'coolify-v5-nginx-test', 'status' => 'running', 'runtime_container_id' => 'container-live', ...$attributes, ]); } it('marks a stale running application exited when its container is gone', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->with($server->fluxHostId())->andReturn([]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh()) ->status->toBe('exited') ->status_message->toBe('Container not found on server during reconcile.'); expect($server->refresh()) ->status->toBe('installed') ->last_status_check->toBe('reconcile') ->last_status_checked_at->not->toBeNull(); Event::assertDispatched(V5CanvasResourceUpdated::class, fn (V5CanvasResourceUpdated $event) => $event->applicationId === $application->id); }); it('updates application status from the observed container state and refreshes container statuses', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server, ['runtime_container_id' => null]); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([ [ 'id' => 'container-live', 'name' => 'coolify-v5-nginx-test', 'image' => 'docker.io/library/nginx:alpine', 'state' => 'Restarting', ], ]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh()) ->status->toBe('restarting') ->runtime_container_id->toBe('container-live'); $containerStatus = ContainerStatus::query() ->where('server_id', $server->id) ->where('container_id', 'container-live') ->first(); expect($containerStatus)->not->toBeNull() ->and($containerStatus->status)->toBe('restarting') ->and($containerStatus->last_seen_at)->not->toBeNull(); }); it('leaves a creating application without a container id alone during reconcile', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server, [ 'status' => 'creating', 'runtime_container_id' => null, ]); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh())->status->toBe('creating'); }); it('marks an installed server unreachable when the node cannot be reached', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andThrow(new RuntimeException('Flux did not return a response before the timeout.')); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($server->refresh()) ->status->toBe('unreachable') ->last_status_check->toBe('reconcile') ->last_status_output->toContain('Flux did not return a response'); expect($application->refresh())->status->toBe('running'); Event::assertDispatched(V5CanvasResourceUpdated::class, fn (V5CanvasResourceUpdated $event) => $event->serverId === $server->id); }); it('does not mark a failed server unreachable when the node cannot be reached', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(['status' => 'failed']); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andThrow(new RuntimeException('boom')); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($server->refresh()) ->status->toBe('failed') ->last_status_check->toBe('reconcile'); }); it('restores an unreachable server to installed on a successful reconcile', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(['status' => 'unreachable']); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($server->refresh()) ->status->toBe('installed') ->last_status_check->toBe('reconcile'); Event::assertDispatched(V5CanvasResourceUpdated::class, fn (V5CanvasResourceUpdated $event) => $event->serverId === $server->id); }); it('dispatches per-server reconcile jobs only for managed servers', function () { createReconcileV5Tables(); Queue::fake(); $managed = createReconcileServer(['host' => '203.0.113.10', 'wireguard_management_ip' => '100.64.0.10']); $unreachable = createReconcileServer([ 'name' => 'edge-02', 'host' => '203.0.113.11', 'wireguard_management_ip' => '100.64.0.11', 'status' => 'unreachable', ]); $withoutCoold = createReconcileServer([ 'name' => 'edge-03', 'host' => '203.0.113.12', 'wireguard_management_ip' => '100.64.0.12', 'capabilities' => [], ]); $notInstalled = createReconcileServer([ 'name' => 'edge-04', 'host' => '203.0.113.13', 'wireguard_management_ip' => '100.64.0.13', 'status' => 'added', ]); (new V5ReconcileServersJob)->handle(); Queue::assertPushed(V5ReconcileServerStateJob::class, 2); Queue::assertPushed(V5ReconcileServerStateJob::class, fn (V5ReconcileServerStateJob $job) => $job->serverId === $managed->id); Queue::assertPushed(V5ReconcileServerStateJob::class, fn (V5ReconcileServerStateJob $job) => $job->serverId === $unreachable->id); Queue::assertNotPushed(V5ReconcileServerStateJob::class, fn (V5ReconcileServerStateJob $job) => $job->serverId === $withoutCoold->id); Queue::assertNotPushed(V5ReconcileServerStateJob::class, fn (V5ReconcileServerStateJob $job) => $job->serverId === $notInstalled->id); }); it('drops a reconcile write when the stored observation is newer than the snapshot', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server, [ 'status' => 'running', 'runtime_container_id' => 'container-live', // A fresher webhook already advanced the watermark past this pull. 'status_observed_at' => now()->addMinutes(5), ]); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([ ['id' => 'container-live', 'name' => 'coolify-v5-nginx-test', 'state' => 'exited'], ]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh()) ->status->toBe('running') ->status_observed_at->not->toBeNull(); Event::assertNotDispatched(V5CanvasResourceUpdated::class, fn (V5CanvasResourceUpdated $event) => $event->applicationId === $application->id); }); it('advances status_observed_at when reconcile writes an application status', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server, [ 'status' => 'running', 'runtime_container_id' => 'container-live', 'status_observed_at' => null, ]); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([ ['id' => 'container-live', 'name' => 'coolify-v5-nginx-test', 'state' => 'restarting'], ]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh()) ->status->toBe('restarting') ->status_observed_at->not->toBeNull(); }); it('normalizes the coold configured state to a valid status on reconcile', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([ ['id' => 'container-live', 'name' => 'coolify-v5-nginx-test', 'state' => 'configured'], ]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh()->status)->toBe('configured'); $containerStatus = ContainerStatus::query() ->where('server_id', $server->id) ->where('container_id', 'container-live') ->first(); expect($containerStatus)->not->toBeNull() ->and($containerStatus->status)->toBe('configured'); }); it('normalizes the coold configured state on the flux webhook path', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server, ['runtime_container_id' => 'container-live']); ApplyFluxResourceStatusUpdate::run([ 'resource_type' => 'application', 'server_uuid' => $server->uuid, 'application_uuid' => $application->uuid, 'status' => 'configured', 'runtime_container_id' => 'container-live', ]); expect($application->refresh()->status)->toBe('configured'); }); it('does not broadcast when reconcile finds the application status unchanged', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); $server = createReconcileServer(); $application = createReconcileApplication($server, [ 'status' => 'running', 'runtime_container_id' => 'container-live', ]); $fluxClient = Mockery::mock(FluxClient::class); $fluxClient->shouldReceive('listContainers')->once()->andReturn([ ['id' => 'container-live', 'name' => 'coolify-v5-nginx-test', 'state' => 'running'], ]); (new V5ReconcileServerStateJob($server->id))->handle($fluxClient); expect($application->refresh()->status)->toBe('running'); Event::assertNotDispatched(V5CanvasResourceUpdated::class); }); it('runs the reconcile jobs on the dedicated v5-reconcile queue', function () { expect((new V5ReconcileServersJob)->queue)->toBe('v5-reconcile') ->and((new V5ReconcileServerStateJob(1))->queue)->toBe('v5-reconcile'); }); it('prunes stale and orphaned container statuses but keeps fresh and live rows', function () { createReconcileV5Tables(); Event::fake([V5CanvasResourceUpdated::class, V5ClusterUpdated::class]); Queue::fake(); $server = createReconcileServer(); createReconcileApplication($server, ['runtime_container_id' => 'container-live']); $staleRow = ContainerStatus::query()->create([ 'team_id' => 1, 'server_id' => $server->id, 'container_id' => 'container-stale', 'status' => 'running', 'last_seen_at' => now()->subHours(V5ReconcileServersJob::CONTAINER_STATUS_TTL_HOURS + 1), ]); $staleButLiveRow = ContainerStatus::query()->create([ 'team_id' => 1, 'server_id' => $server->id, 'container_id' => 'container-live', 'status' => 'running', 'last_seen_at' => now()->subHours(V5ReconcileServersJob::CONTAINER_STATUS_TTL_HOURS + 1), ]); $freshRow = ContainerStatus::query()->create([ 'team_id' => 1, 'server_id' => $server->id, 'container_id' => 'container-fresh', 'status' => 'running', 'last_seen_at' => now()->subHour(), ]); $orphanedServerRow = ContainerStatus::query()->create([ 'team_id' => 1, 'server_id' => $server->id + 999, 'container_id' => 'container-orphaned', 'status' => 'running', 'last_seen_at' => now()->subHour(), ]); (new V5ReconcileServersJob)->handle(); expect(ContainerStatus::query()->find($staleRow->id))->toBeNull() ->and(ContainerStatus::query()->find($orphanedServerRow->id))->toBeNull() ->and(ContainerStatus::query()->find($staleButLiveRow->id))->not->toBeNull() ->and(ContainerStatus::query()->find($freshRow->id))->not->toBeNull(); });