0]); $this->owner = User::factory()->create(); $this->team = Team::factory()->create(['name' => 'Teardown Team']); $this->team->members()->attach($this->owner->id, ['role' => 'owner']); $this->privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]); }); /** * @param array $attributes */ function createTeardownServer(Team $team, User $owner, PrivateKey $privateKey, array $attributes = []): V5Server { return V5Server::query()->create([ 'team_id' => $team->id, 'created_by_user_id' => $owner->id, 'private_key_id' => $privateKey->id, 'name' => 'prod-'.fake()->unique()->numberBetween(1, 9999), 'host' => fake()->unique()->ipv4(), 'ssh_user' => 'root', 'ssh_port' => 22, 'status' => 'installed', ...$attributes, ]); } function createTeardownApplication(Team $team, User $owner, V5Server $server): V5Application { $project = Project::query()->create(['name' => fake()->unique()->company(), 'team_id' => $team->id]); // Project creation auto-provisions a default environment; reuse it rather // than colliding with the (name, project_id) unique index. $environment = $project->environments()->first() ?? Environment::query()->create(['name' => 'production', 'project_id' => $project->id]); // The checked-in testing-schema.sql dump predates the v5_applications.uuid // column, so skip the V5Model uuid-generating event (matching how the other // V5 suites hand-roll their schema) to insert against the dump. return V5Application::withoutEvents(fn () => V5Application::query()->forceCreate([ 'team_id' => $team->id, 'project_id' => $project->id, 'environment_id' => $environment->id, 'server_id' => $server->id, 'created_by_user_id' => $owner->id, 'name' => 'nginx-test', 'image' => 'docker.io/library/nginx:alpine', 'container_name' => 'coolify-v5-nginx-'.fake()->unique()->numerify('#######'), 'status' => 'running', ])); } /** * @param array $overrides * @param array> $applications * @return array */ function teardownServerPayload(array $overrides = [], array $applications = []): array { return [ 'id' => 1, 'uuid' => 'teardown-server-uuid', 'name' => 'prod-01', 'host' => '203.0.113.10', 'ssh_user' => 'root', 'ssh_port' => 22, 'node_address' => '203.0.113.10', 'wireguard_management_ip' => '100.64.0.10', 'is_ingress' => false, 'ingress_type' => null, 'status' => 'installed', 'last_bootstrapped_at' => now()->toISOString(), 'private_key' => "-----BEGIN OPENSSH PRIVATE KEY-----\ntest-key\n-----END OPENSSH PRIVATE KEY-----\n", 'applications' => $applications, ...$overrides, ]; } it('dispatches the v5 teardown job for the team servers when the team is deleted', function () { Queue::fake(); $serverOne = createTeardownServer($this->team, $this->owner, $this->privateKey, [ 'host' => '203.0.113.10', 'is_ingress' => true, 'ingress_type' => 'caddy', 'wireguard_management_ip' => '100.64.0.10', 'last_bootstrapped_at' => now(), ]); createTeardownApplication($this->team, $this->owner, $serverOne); $serverTwo = createTeardownServer($this->team, $this->owner, $this->privateKey, [ 'host' => '203.0.113.11', 'last_bootstrapped_at' => now(), ]); $this->team->delete(); // The team deletion completes (the teardown hook never blocks it). The // per-row cascade is a database-level guarantee from the v5 migrations' // cascadeOnDelete() foreign keys; the in-memory SQLite test dump does not // enforce it, so it is not asserted here. expect(Team::find($this->team->id))->toBeNull(); Queue::assertPushed(V5TeardownTeamJob::class, function (V5TeardownTeamJob $job): bool { if ($job->teamId !== $this->team->id || count($job->servers) !== 2) { return false; } $ingressServer = collect($job->servers)->firstWhere('host', '203.0.113.10'); return $ingressServer !== null && $ingressServer['is_ingress'] === true && str_contains((string) $ingressServer['private_key'], 'BEGIN OPENSSH PRIVATE KEY') && count($ingressServer['applications']) === 1 && str_starts_with((string) $ingressServer['applications'][0]['container_name'], 'coolify-v5-nginx-'); }); }); it('does not dispatch teardown when the team owns no v5 servers', function () { Queue::fake(); $this->team->delete(); Queue::assertNotPushed(V5TeardownTeamJob::class); }); it('runs the full per-server teardown sequence over ssh and flux', function () { Process::fake(['*' => Process::result(output: '')]); $fluxClient = Mockery::mock(FluxClient::class, function (MockInterface $mock): void { $mock->shouldReceive('revokeFirewallRule')->andReturn('Firewall rule revoked.'); $mock->shouldReceive('stopIngress') ->once() ->with(Mockery::type('string'), 'caddy') ->andReturn('Caddy ingress stopped.'); }); app()->instance(FluxClient::class, $fluxClient); $payload = teardownServerPayload([ 'is_ingress' => true, 'ingress_type' => 'caddy', ], [ ['id' => 5, 'container_name' => 'coolify-v5-nginx-abc', 'runtime_container_id' => 'runtime-1'], ]); (new V5TeardownTeamJob($this->team->id, [$payload]))->handle(); // Application container removal ran over SSH. Process::assertRan(fn ($process) => is_array($process->command) && ($process->command[0] ?? null) === 'ssh' && str_contains((string) end($process->command), "rm -f 'coolify-v5-nginx-abc'")); // On-host bootstrap identity removal ran over SSH. Process::assertRan(fn ($process) => is_array($process->command) && ($process->command[0] ?? null) === 'ssh' && str_contains((string) end($process->command), '/etc/coolify/v5-node.json')); }); it('keeps tearing down other servers when one host is unreachable', function () { Process::fake(['*' => Process::result(output: '')]); // The ingress server's Flux stop throws (unreachable host); teardown of the // rest must still proceed and the job must not fail. $fluxClient = Mockery::mock(FluxClient::class, function (MockInterface $mock): void { $mock->shouldReceive('revokeFirewallRule')->andReturn('Firewall rule revoked.'); $mock->shouldReceive('stopIngress') ->with(Mockery::type('string'), 'caddy') ->andThrow(new RuntimeException('flux socket unreachable')); }); app()->instance(FluxClient::class, $fluxClient); $unreachableIngress = teardownServerPayload([ 'id' => 1, 'host' => '203.0.113.10', 'wireguard_management_ip' => '100.64.0.10', 'is_ingress' => true, 'ingress_type' => 'caddy', ]); $healthyServer = teardownServerPayload([ 'id' => 2, 'host' => '203.0.113.11', 'wireguard_management_ip' => '100.64.0.11', ]); (new V5TeardownTeamJob($this->team->id, [$unreachableIngress, $healthyServer]))->handle(); // The healthy server was still cleaned up despite the first host failing. Process::assertRan(fn ($process) => is_array($process->command) && in_array('root@203.0.113.11', $process->command, true) && str_contains((string) end($process->command), '/etc/coolify/v5-node.json')); // The unreachable ingress server's marker removal was still attempted. Process::assertRan(fn ($process) => is_array($process->command) && in_array('root@203.0.113.10', $process->command, true) && str_contains((string) end($process->command), '/etc/coolify/v5-node.json')); }); it('captures each server agent token jti and expiry in the dispatched payload', function () { Queue::fake(); $server = createTeardownServer($this->team, $this->owner, $this->privateKey, [ 'host' => '203.0.113.42', 'agent_token_jti' => 'jti-captured', 'agent_token_expires_at' => now()->addDay(), 'last_bootstrapped_at' => now(), ]); $this->team->delete(); Queue::assertPushed(V5TeardownTeamJob::class, function (V5TeardownTeamJob $job): bool { $payload = collect($job->servers)->firstWhere('host', '203.0.113.42'); return $payload !== null && $payload['agent_token_jti'] === 'jti-captured' && $payload['agent_token_expires_at'] !== null; }); }); it('revokes each server host token during teardown', function () { Process::fake(['*' => Process::result(output: '')]); $expiresAtOne = now()->addDay(); $expiresAtTwo = now()->addDays(2); $fluxClient = Mockery::mock(FluxClient::class, function (MockInterface $mock) use ($expiresAtOne, $expiresAtTwo): void { $mock->shouldReceive('revokeFirewallRule')->andReturn('Firewall rule revoked.'); $mock->shouldReceive('revokeToken')->once()->with('jti-1', $expiresAtOne->timestamp); $mock->shouldReceive('revokeToken')->once()->with('jti-2', $expiresAtTwo->timestamp); }); app()->instance(FluxClient::class, $fluxClient); // last_bootstrapped_at null so only the explicit token revocation runs (no // SSH marker removal, which would revoke a second time). $serverOne = teardownServerPayload([ 'id' => 1, 'host' => '203.0.113.10', 'last_bootstrapped_at' => null, 'agent_token_jti' => 'jti-1', 'agent_token_expires_at' => $expiresAtOne->toISOString(), ]); $serverTwo = teardownServerPayload([ 'id' => 2, 'host' => '203.0.113.11', 'last_bootstrapped_at' => null, 'agent_token_jti' => 'jti-2', 'agent_token_expires_at' => $expiresAtTwo->toISOString(), ]); (new V5TeardownTeamJob($this->team->id, [$serverOne, $serverTwo]))->handle(); expect(RevokedAgentToken::query()->where('jti', 'jti-1')->exists())->toBeTrue() ->and(RevokedAgentToken::query()->where('jti', 'jti-2')->exists())->toBeTrue(); }); it('logs the incomplete-teardown error naming hosts that could not be torn down', function () { Process::fake(['*' => Process::result(output: '')]); Log::spy(); // The ingress host's Flux stop throws (unreachable) so it is reported // incomplete; the healthy host tears down cleanly. $fluxClient = Mockery::mock(FluxClient::class, function (MockInterface $mock): void { $mock->shouldReceive('revokeFirewallRule')->andReturn('Firewall rule revoked.'); $mock->shouldReceive('revokeToken')->andReturnNull(); $mock->shouldReceive('stopIngress') ->with(Mockery::type('string'), 'caddy') ->andThrow(new RuntimeException('flux socket unreachable')); }); app()->instance(FluxClient::class, $fluxClient); $unreachableIngress = teardownServerPayload([ 'id' => 1, 'host' => '203.0.113.10', 'wireguard_management_ip' => '100.64.0.10', 'is_ingress' => true, 'ingress_type' => 'caddy', ]); $healthyServer = teardownServerPayload([ 'id' => 2, 'host' => '203.0.113.11', 'wireguard_management_ip' => '100.64.0.11', ]); (new V5TeardownTeamJob($this->team->id, [$unreachableIngress, $healthyServer]))->handle(); // The healthy host was still cleaned up despite the ingress host failing. Process::assertRan(fn ($process) => is_array($process->command) && in_array('root@203.0.113.11', $process->command, true) && str_contains((string) end($process->command), '/etc/coolify/v5-node.json')); // The operator-facing signal names only the unreachable host. Log::shouldHaveReceived('error')->withArgs( fn (string $message, array $context = []) => str_contains($message, 'v5 team teardown incomplete') && ($context['team_id'] ?? null) === $this->team->id && collect($context['hosts'] ?? [])->contains(fn ($host) => ($host['host'] ?? null) === '203.0.113.10') && collect($context['hosts'] ?? [])->doesntContain(fn ($host) => ($host['host'] ?? null) === '203.0.113.11') )->once(); }); it('does not remove the bootstrap marker for servers that were never bootstrapped', function () { Process::fake(['*' => Process::result(output: '')]); $payload = teardownServerPayload([ 'last_bootstrapped_at' => null, 'is_ingress' => false, ]); (new V5TeardownTeamJob($this->team->id, [$payload]))->handle(); Process::assertNothingRan(); });