feat(v5): add server reconciliation and canvas APIs

Split V5 dashboard behavior into domain controllers and policies,
add agent token rotation/revocation, status reconciliation jobs,
ingress firewall syncing, and canvas connection APIs.

Add migrations for V5 status tracking, server capabilities, resource
connection aliases, and revoked agent tokens.
This commit is contained in:
Andras Bacsai
2026-07-06 17:40:37 +02:00
parent df854feee8
commit 6ae45684f9
147 changed files with 22444 additions and 10407 deletions
@@ -12,9 +12,7 @@ class FluxResourceStatusController extends Controller
{
public function __invoke(Request $request): JsonResponse
{
$token = config('flux.laravel_api_token');
if (! is_string($token) || $token === '' || ! hash_equals($token, (string) $request->bearerToken())) {
if (! $this->authorizedBearer($request)) {
abort(401);
}
@@ -40,6 +38,7 @@ class FluxResourceStatusController extends Controller
'state' => ['required_without:status', 'string', 'max:64'],
'status_message' => ['nullable', 'string', 'max:1000'],
'message' => ['nullable', 'string', 'max:1000'],
'observed_at' => ['nullable', 'string', 'date'],
])->validate();
$resource = ApplyFluxResourceStatusUpdate::run($validated);
@@ -60,4 +59,53 @@ class FluxResourceStatusController extends Controller
'message' => 'Resource status updated.',
]);
}
/**
* Constant-time match the presented bearer token against every accepted
* inbound token. Accepting an array (config('flux.laravel_api_tokens'),
* falling back to the single config('flux.laravel_api_token')) lets an
* operator rotate by serving old+new tokens simultaneously.
*
* SECURITY: this is still a shared global secret every flux instance
* presents the same token, so it cannot be scoped or revoked per-flux, and
* a leak forces a fleet-wide rotation. The target design is per-flux,
* individually rotatable tokens; until then the array support above is the
* mitigation that makes rotation possible without downtime.
*/
private function authorizedBearer(Request $request): bool
{
$presented = (string) $request->bearerToken();
if ($presented === '') {
return false;
}
foreach ($this->acceptedTokens() as $token) {
if (hash_equals($token, $presented)) {
return true;
}
}
return false;
}
/**
* @return array<int, string>
*/
private function acceptedTokens(): array
{
$tokens = config('flux.laravel_api_tokens', []);
$tokens = is_array($tokens) ? $tokens : [];
$single = config('flux.laravel_api_token');
if (is_string($single) && $single !== '') {
$tokens[] = $single;
}
return array_values(array_filter(
array_map(fn ($token): string => is_string($token) ? $token : '', $tokens),
fn (string $token): bool => $token !== ''
));
}
}
@@ -0,0 +1,635 @@
<?php
namespace App\Http\Controllers\V5;
use App\Actions\V5\Application\DestroyNginxApplication;
use App\Actions\V5\Proxy\StartCaddyIngress;
use App\Enums\V5\ApplicationStatus;
use App\Enums\V5\IngressStatus;
use App\Enums\V5\ServerStatus;
use App\Exceptions\V5\UnsupportedCooldVerb;
use App\Http\Controllers\Controller;
use App\Http\Controllers\V5\Concerns\HandlesIngressSyncErrors;
use App\Http\Controllers\V5\Concerns\ResolvesCurrentTeam;
use App\Http\Controllers\V5\Concerns\ResolvesProjectSelection;
use App\Http\Controllers\V5\Concerns\SerializesCanvasResources;
use App\Jobs\V5DeployApplicationJob;
use App\Models\Environment;
use App\Models\Project;
use App\Models\Team;
use App\Models\V5\Application as V5Application;
use App\Models\V5\ApplicationDomain as V5ApplicationDomain;
use App\Models\V5\ResourceConnection;
use App\Models\V5\Server as V5Server;
use App\Rules\ValidHostname;
use App\Services\Flux\FluxClient;
use App\Support\V5\CanvasResourceSerializer;
use App\Support\V5\ConnectionFirewallSync;
use App\Support\V5\StatusObservation;
use Carbon\CarbonImmutable;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Http\Response;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use Illuminate\Validation\Rule;
class ApplicationController extends Controller
{
use HandlesIngressSyncErrors;
use ResolvesCurrentTeam;
use ResolvesProjectSelection;
use SerializesCanvasResources;
private const DEFAULT_NGINX_IMAGE = 'docker.io/library/nginx:alpine';
public function __construct(private readonly ConnectionFirewallSync $firewallSync) {}
public function store(Request $request): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$projects = $this->projects($currentTeam);
[$selectedProject, $selectedEnvironment] = $this->selectedProjectAndEnvironment($request, $projects);
if ($selectedProject === null || $selectedEnvironment === null) {
return response()->json([
'message' => 'Select a project and environment before deploying nginx.',
], 422);
}
$project = $this->projectQuery($currentTeam)
->where('uuid', $selectedProject['uuid'])
->first();
if (! $project instanceof Project) {
abort(403);
}
$environment = $this->selectedEnvironment($project, $selectedEnvironment['uuid']);
if (! $environment instanceof Environment) {
abort(403);
}
$validated = $request->validate([
'server_uuid' => ['nullable', 'string', 'max:255'],
'image' => ['nullable', 'string', 'max:255', 'regex:/^[a-zA-Z0-9][a-zA-Z0-9._\/:@-]*$/'],
]);
$image = trim($validated['image'] ?? '') ?: self::DEFAULT_NGINX_IMAGE;
$server = V5Server::query()
->where('team_id', $currentTeam->id)
->when(
isset($validated['server_uuid']),
fn (Builder $query) => $query->where('uuid', $validated['server_uuid']),
fn (Builder $query) => $query
->orderByRaw('last_bootstrapped_at is null')
->orderBy('name')
)
->first();
if (! $server instanceof V5Server) {
return response()->json([
'message' => 'Add a v5 server before deploying nginx.',
], 422);
}
if ($server->status !== ServerStatus::Installed->value || $server->last_bootstrapped_at === null) {
return response()->json([
'message' => "Bootstrap server {$server->name} before deploying to it.",
], 422);
}
$canvasPosition = $this->nextApplicationCanvasPosition($currentTeam, $project, $environment);
$application = V5Application::query()->create([
'team_id' => $currentTeam->id,
'project_id' => $project->id,
'environment_id' => $environment->id,
'server_id' => $server->id,
'created_by_user_id' => $request->user()->id,
'name' => 'nginx-test',
'image' => $image,
'container_name' => 'coolify-v5-nginx-'.strtolower((string) Str::ulid()),
'status' => ApplicationStatus::Creating->value,
'status_message' => 'Starting nginx container.',
'mesh_namespace' => 'default',
'canvas_x' => $canvasPosition['canvas_x'],
'canvas_y' => $canvasPosition['canvas_y'],
]);
V5DeployApplicationJob::dispatch($application->id);
return response()->json([
'application' => $this->serializeApplication($application),
], 202);
}
public function refresh(Request $request, FluxClient $fluxClient): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$projects = $this->projects($currentTeam);
[$selectedProject, $selectedEnvironment] = $this->selectedProjectAndEnvironment($request, $projects);
if ($selectedProject === null || $selectedEnvironment === null) {
return response()->json([
'message' => 'Select a project and environment before refreshing applications.',
], 422);
}
$applications = $this->applicationQuery($currentTeam, $selectedProject, $selectedEnvironment)
->with('server')
->get();
$errors = [];
$applications
->groupBy('server_id')
->each(function (Collection $serverApplications) use ($fluxClient, &$errors): void {
/** @var V5Application|null $firstApplication */
$firstApplication = $serverApplications->first();
$server = $firstApplication?->server;
$hostId = $server?->fluxHostId();
if (! $server instanceof V5Server || ! is_string($hostId) || $hostId === '') {
$errors[] = 'A server is missing its Flux host id.';
return;
}
// The moment we query coold is the observation time for the rows
// this refresh writes, so a fresher webhook always wins the
// status_observed_at watermark and is never clobbered.
$observedAt = CarbonImmutable::now();
try {
$containers = collect($fluxClient->listContainers($hostId));
} catch (\Throwable $e) {
$errors[] = $e->getMessage();
return;
}
$serverApplications->each(function (V5Application $application) use ($containers, $observedAt): void {
$container = $containers->first(function (array $container) use ($application): bool {
return ($application->runtime_container_id !== null && ($container['id'] ?? null) === $application->runtime_container_id)
|| ($container['name'] ?? null) === $application->container_name;
});
if (! is_array($container)) {
// A creating application without a container id simply has
// not materialized yet; the deploy job will settle it.
if ($application->status === ApplicationStatus::Creating->value && $application->runtime_container_id === null) {
return;
}
if (StatusObservation::isStale($observedAt, $application->status_observed_at, 'application status', ['application_id' => $application->id])) {
return;
}
$application->update([
'status' => ApplicationStatus::Exited->value,
'status_message' => 'Container not found on server.',
'status_observed_at' => $observedAt,
]);
return;
}
if (StatusObservation::isStale($observedAt, $application->status_observed_at, 'application status', ['application_id' => $application->id])) {
return;
}
$rawState = is_string($container['state'] ?? null) && $container['state'] !== '' ? $container['state'] : null;
$application->update([
'status' => StatusObservation::normalize($rawState, ApplicationStatus::class) ?? ApplicationStatus::Unknown->value,
'status_message' => 'Container state refreshed from coold.',
'status_observed_at' => $observedAt,
'runtime_container_id' => is_string($container['id'] ?? null) ? $container['id'] : $application->runtime_container_id,
]);
});
});
V5Server::query()
->where('team_id', $currentTeam->id)
->orderBy('name')
->get()
->filter(fn (V5Server $server) => $server->isIngress())
->each(function (V5Server $server) use ($fluxClient, &$errors): void {
$hostId = $server->fluxHostId();
if (! is_string($hostId) || $hostId === '') {
$errors[] = "Caddy ingress server {$server->name} is missing its Flux host id.";
return;
}
try {
$containers = collect($fluxClient->listContainers($hostId));
} catch (\Throwable $e) {
$errors[] = $e->getMessage();
return;
}
$container = $containers->first(fn (array $container) => ($container['name'] ?? null) === 'coolify-v5-caddy');
$rawState = is_array($container) && is_string($container['state'] ?? null) && $container['state'] !== '' ? $container['state'] : null;
$state = $rawState !== null
? (StatusObservation::normalize($rawState, IngressStatus::class) ?? IngressStatus::Unknown->value)
: IngressStatus::Exited->value;
$server->update([
'ingress_type' => 'caddy',
'ingress_status' => $state,
'last_status_check' => 'flux',
'last_status_output' => 'Caddy ingress state refreshed from coold.',
'last_status_checked_at' => now(),
]);
});
return response()->json([
'applications' => $this->applicationQuery($currentTeam, $selectedProject, $selectedEnvironment)
->with('server')
->orderBy('created_at')
->get()
->map(fn (V5Application $application) => $this->serializeApplication($application))
->all(),
'caddyIngresses' => $this->caddyIngresses($currentTeam),
'errors' => $errors,
]);
}
public function logs(Request $request, V5Application $application): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('view', [$application, $currentTeam]);
$application->loadMissing('server');
$server = $application->server;
$hostId = $server?->fluxHostId();
$containerId = $application->runtime_container_id;
$logs = null;
$logsError = null;
// A container id only appears once the deploy actually created one; a
// deploy that failed before that (e.g. host not connected) has none, so
// there is nothing to fetch and the frontend just shows the status.
if (is_string($containerId) && $containerId !== '' && $server instanceof V5Server && $server->status !== ServerStatus::Unreachable->value && is_string($hostId) && $hostId !== '') {
try {
$logs = app(FluxClient::class)->containerLogs($hostId, $containerId);
} catch (UnsupportedCooldVerb $exception) {
$logsError = "This node's coold does not support container logs.";
} catch (\RuntimeException $exception) {
Log::warning('V5 application container logs request failed', [
'application_id' => $application->id,
'message' => $exception->getMessage(),
]);
$logsError = 'Could not fetch container logs through Flux. Check the Flux and coold status, then try again.';
}
}
return response()->json([
'status' => $application->status,
'statusMessage' => $application->status_message,
'containerId' => $containerId,
'logs' => $logs,
'logsError' => $logsError,
]);
}
public function updatePosition(Request $request, V5Application $application): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('update', [$application, $currentTeam]);
$validated = $request->validate([
'canvas_x' => ['required', 'integer', 'min:-100000', 'max:100000'],
'canvas_y' => ['required', 'integer', 'min:-100000', 'max:100000'],
]);
$application->update([
'canvas_x' => $validated['canvas_x'],
'canvas_y' => $validated['canvas_y'],
]);
return response()->json([
'application' => $this->serializeApplication($application->refresh()->load('server')),
]);
}
public function updateIngress(Request $request, V5Application $application): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('updateIngress', [$application, $currentTeam]);
$validated = $request->validate([
'ingress_enabled' => ['required', 'boolean'],
'internal_port' => ['nullable', 'integer', 'min:1', 'max:65535'],
'domains' => [Rule::requiredIf(fn () => $request->boolean('ingress_enabled')), 'array', 'min:1'],
'domains.*' => ['required', 'string', 'max:255', 'distinct:ignore_case', new ValidHostname],
]);
$application->loadMissing('server');
if ($validated['ingress_enabled'] && ! $application->server?->isIngress()) {
return response()->json([
'message' => 'Enable ingress on the server before enabling app ingress.',
], 422);
}
if ($validated['ingress_enabled'] && array_key_exists('domains', $validated)) {
$conflict = $this->conflictingApplicationDomain($application, $validated['domains']);
if ($conflict instanceof V5ApplicationDomain) {
return response()->json([
'message' => "The domain {$conflict->domain} is already used by application \"{$conflict->application?->name}\" on this server.",
], 422);
}
}
$originalAttributes = $application->only(['ingress_enabled', 'internal_port']);
$originalDomains = $application->domains()->pluck('domain')->all();
DB::transaction(function () use ($application, $validated): void {
$application->update([
'ingress_enabled' => $validated['ingress_enabled'],
'internal_port' => $validated['internal_port'] ?? null,
]);
if (array_key_exists('domains', $validated)) {
$application->domains()->delete();
collect($validated['domains'])
->map(fn (string $domain) => trim($domain))
->filter()
->unique()
->each(fn (string $domain) => V5ApplicationDomain::query()->create([
'application_id' => $application->id,
'domain' => $domain,
]));
}
});
$application->refresh()->load(['server', 'domains']);
if ($application->server?->isIngress() && $application->server->status === ServerStatus::Installed->value) {
try {
StartCaddyIngress::run($application->server);
} catch (\RuntimeException $exception) {
$this->restoreApplicationIngress($application, $originalAttributes, $originalDomains);
return $this->ingressSyncErrorResponse($exception);
}
}
return response()->json([
'application' => $this->serializeApplication($application),
]);
}
public function updateCaddyIngressPosition(Request $request, V5Server $server): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('updateCanvasPosition', [$server, $currentTeam]);
$validated = $request->validate([
'canvas_x' => ['required', 'integer', 'min:-100000', 'max:100000'],
'canvas_y' => ['required', 'integer', 'min:-100000', 'max:100000'],
]);
$server->update([
'canvas_x' => $validated['canvas_x'],
'canvas_y' => $validated['canvas_y'],
]);
return response()->json([
'caddyIngress' => $this->serializeCaddyIngress($server->refresh()),
]);
}
public function destroy(Request $request, V5Application $application, FluxClient $fluxClient): Response|JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('delete', [$application, $currentTeam]);
$application->loadMissing(['server', 'domains']);
$server = $application->server;
$connections = $this->applicationResourceConnections($application);
if ($request->boolean('delete_locally')) {
$this->deleteApplicationLocally($application, $connections);
return response()->noContent();
}
$oldFirewallRules = $connections
->flatMap(function (ResourceConnection $connection): Collection {
// Deletion must never be blocked by an endpoint that already lost
// its server; those rules can no longer be revoked anyway.
try {
return $this->firewallSync->rulesFor($connection->load('rules'));
} catch (\RuntimeException $exception) {
report($exception);
return collect();
}
});
$originalIngressAttributes = null;
$originalIngressDomains = [];
$ingressConfigurationChanged = false;
try {
$this->firewallSync->sync($fluxClient, $oldFirewallRules, collect());
} catch (\RuntimeException $exception) {
report($exception);
return response()->json([
'message' => 'Could not sync firewall rules through Flux.',
'detail' => $exception->getMessage(),
], 502);
}
if ($server instanceof V5Server && $server->isIngress() && $server->status === ServerStatus::Installed->value && $application->ingress_enabled) {
$originalIngressAttributes = $application->only(['ingress_enabled', 'internal_port']);
$originalIngressDomains = $application->domains()->pluck('domain')->all();
DB::transaction(function () use ($application): void {
$application->update([
'ingress_enabled' => false,
'internal_port' => null,
]);
$application->domains()->delete();
});
try {
StartCaddyIngress::run($server);
$ingressConfigurationChanged = true;
} catch (\RuntimeException $exception) {
$this->restoreApplicationIngress($application, $originalIngressAttributes, $originalIngressDomains);
return $this->ingressSyncErrorResponse($exception);
}
}
$error = DestroyNginxApplication::run($application);
if ($error !== null) {
if ($originalIngressAttributes !== null) {
$this->restoreApplicationIngress($application, $originalIngressAttributes, $originalIngressDomains);
if ($ingressConfigurationChanged && $server instanceof V5Server) {
try {
StartCaddyIngress::run($server);
} catch (\RuntimeException $exception) {
report($exception);
}
}
}
try {
$this->firewallSync->sync($fluxClient, collect(), $oldFirewallRules);
} catch (\RuntimeException $exception) {
report($exception);
}
return response()->json([
'message' => $error,
'can_delete_locally' => true,
], 422);
}
$this->deleteApplicationLocally($application, $connections);
return response()->noContent();
}
/**
* @param Collection<int, ResourceConnection> $connections
*/
private function deleteApplicationLocally(V5Application $application, Collection $connections): void
{
DB::transaction(function () use ($application, $connections): void {
$connections->each(function (ResourceConnection $connection): void {
$connection->rules()->delete();
$connection->delete();
});
$application->delete();
});
}
/**
* @return Collection<int, ResourceConnection>
*/
private function applicationResourceConnections(V5Application $application): Collection
{
return ResourceConnection::query()
->where('team_id', $application->team_id)
->where(function (Builder $query) use ($application): void {
$query
->where(function (Builder $query) use ($application): void {
$query
->where('resource_one_type', $application->getMorphClass())
->where('resource_one_id', $application->id);
})
->orWhere(function (Builder $query) use ($application): void {
$query
->where('resource_two_type', $application->getMorphClass())
->where('resource_two_id', $application->id);
});
})
->with('rules')
->get();
}
/**
* @return array{canvas_x: int, canvas_y: int}
*/
private function nextApplicationCanvasPosition(Team $currentTeam, Project $project, Environment $environment): array
{
$existingApplications = V5Application::query()
->where('team_id', $currentTeam->id)
->where('project_id', $project->id)
->where('environment_id', $environment->id)
->get(['canvas_x', 'canvas_y']);
$horizontalStep = CanvasResourceSerializer::CARD_WIDTH + CanvasResourceSerializer::CARD_GAP;
$verticalStep = CanvasResourceSerializer::CARD_HEIGHT + CanvasResourceSerializer::CARD_GAP;
for ($row = 0; $row < 100; $row++) {
for ($column = 0; $column < 100; $column++) {
$candidate = [
'canvas_x' => $column * $horizontalStep,
'canvas_y' => $row * $verticalStep,
];
if (! $this->canvasPositionCollides($candidate, $existingApplications)) {
return $candidate;
}
}
}
return [
'canvas_x' => $existingApplications->max('canvas_x') + $horizontalStep,
'canvas_y' => 0,
];
}
/**
* @param array{canvas_x: int, canvas_y: int} $candidate
* @param Collection<int, V5Application> $existingApplications
*/
private function canvasPositionCollides(array $candidate, Collection $existingApplications): bool
{
return $existingApplications->contains(function (V5Application $application) use ($candidate) {
return abs($candidate['canvas_x'] - $application->canvas_x) < CanvasResourceSerializer::CARD_WIDTH + CanvasResourceSerializer::CARD_GAP
&& abs($candidate['canvas_y'] - $application->canvas_y) < CanvasResourceSerializer::CARD_HEIGHT + CanvasResourceSerializer::CARD_GAP;
});
}
/**
* @param array<int, string> $domains
*/
private function conflictingApplicationDomain(V5Application $application, array $domains): ?V5ApplicationDomain
{
$normalizedDomains = collect($domains)
->map(fn (string $domain) => Str::lower(trim($domain)))
->filter()
->values();
if ($normalizedDomains->isEmpty()) {
return null;
}
return V5ApplicationDomain::query()
->whereIn(DB::raw('LOWER(domain)'), $normalizedDomains->all())
->whereHas('application', fn (Builder $query) => $query
->where('server_id', $application->server_id)
->whereKeyNot($application->id)
->where('ingress_enabled', true))
->with('application:id,name')
->first();
}
/**
* @param array<string, mixed> $attributes
* @param array<int, string> $domains
*/
private function restoreApplicationIngress(V5Application $application, array $attributes, array $domains): void
{
DB::transaction(function () use ($application, $attributes, $domains): void {
$application->update($attributes);
$application->domains()->delete();
foreach ($domains as $domain) {
V5ApplicationDomain::query()->create([
'application_id' => $application->id,
'domain' => $domain,
]);
}
});
}
}
@@ -0,0 +1,207 @@
<?php
namespace App\Http\Controllers\V5;
use App\Http\Controllers\Controller;
use App\Http\Controllers\V5\Concerns\ResolvesCurrentTeam;
use App\Http\Controllers\V5\Concerns\ResolvesProjectSelection;
use App\Http\Controllers\V5\Concerns\ValidatesBuilderConfiguration;
use App\Models\PrivateKey;
use App\Models\Team;
use App\Models\V5\Cluster as V5Cluster;
use App\Services\Flux\FluxHealth;
use App\Support\V5\ClusterSerializer;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Validation\Rule;
use Inertia\Inertia;
use Inertia\Response;
class ClusterController extends Controller
{
use ResolvesCurrentTeam;
use ResolvesProjectSelection;
use ValidatesBuilderConfiguration;
public function index(Request $request, FluxHealth $fluxHealth): Response
{
$currentTeam = $request->attributes->get('v5.currentTeam');
$projects = $this->projects($currentTeam);
[$selectedProject, $selectedEnvironment] = $this->selectedProjectAndEnvironment($request, $projects);
return Inertia::render('Clusters', [
'currentTeam' => $this->serializeCurrentTeam($currentTeam),
'flux' => $fluxHealth->check(),
'clusters' => $this->clusters($currentTeam),
'privateKeys' => $this->privateKeys($currentTeam),
'projects' => $projects,
'selectedProjectUuid' => $selectedProject['uuid'] ?? null,
'selectedEnvironmentUuid' => $selectedEnvironment['uuid'] ?? null,
]);
}
public function show(Request $request, V5Cluster $cluster): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('view', [$cluster, $currentTeam]);
return response()->json([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
]);
}
public function store(Request $request): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('create', [V5Cluster::class, $currentTeam]);
$validated = $request->validate([
'name' => [
'required',
'string',
'max:255',
Rule::unique('v5_clusters', 'name')->where('team_id', $currentTeam->id),
],
'description' => ['nullable', 'string', 'max:1000'],
'wireguard_interface' => ['sometimes', 'string', 'max:32', 'regex:/^[a-zA-Z0-9_.-]+$/'],
'wireguard_management_pool' => ['sometimes', 'string', 'max:64', $this->ipv4CidrRule()],
'wireguard_listen_port' => ['sometimes', 'integer', 'min:1', 'max:65535'],
'container_network_pool' => ['sometimes', 'string', 'max:64', $this->ipv4CidrRule()],
'container_network_prefix' => ['sometimes', 'integer', 'min:1', 'max:32'],
'namespaces' => ['sometimes', 'array', 'min:1'],
'namespaces.*' => ['string', 'distinct', 'regex:/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/'],
'default_deny_containers' => ['sometimes', 'boolean'],
'coold_version' => ['sometimes', 'string', 'max:64'],
'corrosion_version' => ['sometimes', 'string', 'max:64'],
'corrosion_gossip_port' => ['sometimes', 'integer', 'min:1', 'max:65535'],
'corrosion_api_port' => ['sometimes', 'integer', 'min:1', 'max:65535'],
'builder_enabled' => ['sometimes', 'boolean'],
'builder_capacity' => $this->builderCapacityRules(
$this->requestedBuilderEnabled($request, true)
),
'builder_cpu_quota' => ['sometimes', 'string', 'max:32'],
'builder_memory_max' => ['sometimes', 'string', 'max:32'],
'builder_timeout_secs' => ['sometimes', 'integer', 'min:1', 'max:86400'],
]);
$cluster = V5Cluster::query()->create([
...$this->defaultClusterConfiguration(),
...collect($validated)->except(['name', 'description'])->all(),
'team_id' => $currentTeam->id,
'created_by_user_id' => $request->user()->id,
'name' => $validated['name'],
'description' => $validated['description'] ?? null,
]);
return response()->json([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
], 201);
}
public function destroy(Request $request, V5Cluster $cluster): \Illuminate\Http\Response|JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('delete', [$cluster, $currentTeam]);
if ($cluster->servers()->exists()) {
return response()->json([
'message' => 'Only empty clusters can be deleted.',
], 422);
}
$cluster->delete();
return response()->noContent();
}
/**
* @return array<int, array<string, mixed>>
*/
private function clusters(mixed $currentTeam): array
{
if (! $currentTeam instanceof Team) {
return [];
}
$serializer = app(ClusterSerializer::class);
return V5Cluster::query()
->where('team_id', $currentTeam->id)
->with(['servers' => fn ($query) => $query
->with('privateKey')
->orderBy('name')])
->withCount('servers')
->orderBy('name')
->get()
->map(fn (V5Cluster $cluster) => $serializer->serialize($cluster))
->all();
}
/**
* @return array<int, array{id: string, name: string}>
*/
private function privateKeys(mixed $currentTeam): array
{
if (! $currentTeam instanceof Team) {
return [];
}
return PrivateKey::query()
->where('team_id', $currentTeam->id)
->where('is_git_related', false)
->orderBy('name')
->get(['id', 'uuid', 'name'])
->map(fn (PrivateKey $privateKey) => [
'id' => $privateKey->uuid,
'name' => $privateKey->name,
])
->all();
}
/**
* @return array<string, mixed>
*/
private function defaultClusterConfiguration(): array
{
return [
'wireguard_interface' => V5Cluster::DEFAULT_WIREGUARD_INTERFACE,
'wireguard_management_pool' => V5Cluster::DEFAULT_WIREGUARD_MANAGEMENT_POOL,
'wireguard_listen_port' => V5Cluster::DEFAULT_WIREGUARD_LISTEN_PORT,
'container_network_pool' => V5Cluster::DEFAULT_CONTAINER_NETWORK_POOL,
'container_network_prefix' => V5Cluster::DEFAULT_CONTAINER_NETWORK_PREFIX,
'namespaces' => V5Cluster::DEFAULT_NAMESPACES,
'default_deny_containers' => true,
'coold_version' => V5Cluster::DEFAULT_COOLD_VERSION,
'corrosion_version' => V5Cluster::DEFAULT_CORROSION_VERSION,
'corrosion_gossip_port' => V5Cluster::DEFAULT_CORROSION_GOSSIP_PORT,
'corrosion_api_port' => V5Cluster::DEFAULT_CORROSION_API_PORT,
'builder_enabled' => true,
'builder_capacity' => V5Cluster::DEFAULT_BUILDER_CAPACITY,
'builder_cpu_quota' => V5Cluster::DEFAULT_BUILDER_CPU_QUOTA,
'builder_memory_max' => V5Cluster::DEFAULT_BUILDER_MEMORY_MAX,
'builder_timeout_secs' => V5Cluster::DEFAULT_BUILDER_TIMEOUT_SECS,
];
}
private function ipv4CidrRule(): \Closure
{
return function (string $attribute, mixed $value, \Closure $fail): void {
if (! is_string($value) || ! str_contains($value, '/')) {
$fail('The :attribute must be a valid IPv4 CIDR range.');
return;
}
[$ip, $prefix] = explode('/', $value, 2);
if (
filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false
|| ! ctype_digit($prefix)
|| (int) $prefix < 0
|| (int) $prefix > 32
) {
$fail('The :attribute must be a valid IPv4 CIDR range.');
}
};
}
}
@@ -0,0 +1,40 @@
<?php
namespace App\Http\Controllers\V5\Concerns;
use Illuminate\Http\JsonResponse;
use Illuminate\Support\Str;
trait HandlesIngressSyncErrors
{
protected function ingressSyncErrorResponse(\RuntimeException $exception): JsonResponse
{
return response()->json([
'message' => $this->friendlyIngressSyncError($exception->getMessage()),
'detail' => $exception->getMessage(),
], 502);
}
protected function friendlyIngressSyncError(string $message): string
{
$normalized = Str::lower($message);
if (str_contains($normalized, 'invalid http response') || str_contains($normalized, 'could not talk to flux')) {
return 'Could not reach Flux. Check that Flux is running in the Coolify container and try again.';
}
if (str_contains($normalized, 'dispatch timeout') || str_contains($normalized, 'timed out')) {
return 'coold did not respond in time. Check that the server agent is running and connected to Flux.';
}
if (str_contains($normalized, 'validate caddyfile')) {
return 'Caddy rejected the generated ingress configuration. Check the domains and internal port, then try again.';
}
if (str_contains($normalized, 'start caddy ingress') || str_contains($normalized, 'reload caddy ingress')) {
return 'Could not start Caddy ingress on the server. Check that Podman is running and port 80 is available.';
}
return 'Could not update ingress. Check Flux and coold logs, then try again.';
}
}
@@ -0,0 +1,22 @@
<?php
namespace App\Http\Controllers\V5\Concerns;
use App\Models\Team;
use Illuminate\Http\Request;
trait ResolvesCurrentTeam
{
/**
* Resolve the current team set by the EnsureCurrentTeam middleware, or
* abort with a 404 so resources outside the team stay invisible.
*/
protected function currentTeamOrFail(Request $request): Team
{
$currentTeam = $request->attributes->get('v5.currentTeam');
abort_unless($currentTeam instanceof Team, 404);
return $currentTeam;
}
}
@@ -0,0 +1,115 @@
<?php
namespace App\Http\Controllers\V5\Concerns;
use App\Models\Environment;
use App\Models\Project;
use App\Models\Team;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Http\Request;
trait ResolvesProjectSelection
{
protected const SELECTED_PROJECT_SESSION_KEY = 'v5.selectedProjectUuid';
protected const SELECTED_ENVIRONMENT_SESSION_KEY = 'v5.selectedEnvironmentUuid';
/**
* @return array{id: int}|null
*/
protected function serializeCurrentTeam(mixed $currentTeam): ?array
{
if (! $currentTeam instanceof Team) {
return null;
}
return [
'id' => $currentTeam->id,
];
}
/**
* @param array<int, array{uuid: string, name: string, environments: array<int, array{uuid: string, name: string}>}> $projects
* @return array{0: array{uuid: string, name: string, environments: array<int, array{uuid: string, name: string}>}|null, 1: array{uuid: string, name: string}|null}
*/
protected function selectedProjectAndEnvironment(Request $request, array $projects): array
{
$selectedProjectUuid = $request->session()->get(self::SELECTED_PROJECT_SESSION_KEY);
$selectedEnvironmentUuid = $request->session()->get(self::SELECTED_ENVIRONMENT_SESSION_KEY);
$selectedProject = null;
foreach ($projects as $project) {
if ($project['uuid'] === $selectedProjectUuid) {
$selectedProject = $project;
break;
}
}
$selectedProject ??= $projects[0] ?? null;
$selectedEnvironment = null;
foreach ($selectedProject['environments'] ?? [] as $environment) {
if ($environment['uuid'] === $selectedEnvironmentUuid) {
$selectedEnvironment = $environment;
break;
}
}
$selectedEnvironment ??= $selectedProject['environments'][0] ?? null;
return [$selectedProject, $selectedEnvironment];
}
protected function selectedEnvironment(Project $project, ?string $environmentUuid): ?Environment
{
if ($environmentUuid === null) {
return $project->environments->first();
}
$environment = $project->environments->firstWhere('uuid', $environmentUuid);
if (! $environment instanceof Environment) {
abort(422, 'The selected environment is not available for the selected project.');
}
return $environment;
}
/**
* @return array<int, array{uuid: string, name: string, environments: array<int, array{uuid: string, name: string}>}>
*/
protected function projects(mixed $currentTeam): array
{
if (! $currentTeam instanceof Team) {
return [];
}
return $this->projectQuery($currentTeam)
->get()
->map(fn (Project $project) => [
'uuid' => $project->uuid,
'name' => $project->name,
'environments' => $project->environments
->map(fn ($environment) => [
'uuid' => $environment->uuid,
'name' => $environment->name,
])
->all(),
])
->all();
}
protected function projectQuery(Team $currentTeam): Builder
{
return Project::query()
->select(['id', 'uuid', 'name', 'team_id'])
->where('team_id', $currentTeam->id)
->with(['environments' => fn ($query) => $query
->select(['id', 'uuid', 'name', 'project_id'])
->orderByRaw("CASE WHEN LOWER(name) = 'production' THEN 0 ELSE 1 END")
->orderByRaw('LOWER(name)')])
->orderByRaw('LOWER(name)');
}
}
@@ -0,0 +1,63 @@
<?php
namespace App\Http\Controllers\V5\Concerns;
use App\Models\Team;
use App\Models\V5\Application as V5Application;
use App\Models\V5\Server as V5Server;
use App\Support\V5\CanvasResourceSerializer;
use Illuminate\Database\Eloquent\Builder;
trait SerializesCanvasResources
{
/**
* @return array<string, mixed>
*/
protected function serializeApplication(V5Application $application): array
{
return app(CanvasResourceSerializer::class)->serializeApplication($application);
}
/**
* @return array<string, mixed>
*/
protected function serializeCaddyIngress(V5Server $server, int $index = 0): array
{
return app(CanvasResourceSerializer::class)->serializeCaddyIngress($server, $index);
}
/**
* @return array<int, array<string, mixed>>
*/
protected function caddyIngresses(mixed $currentTeam): array
{
if (! $currentTeam instanceof Team) {
return [];
}
return V5Server::query()
->where('team_id', $currentTeam->id)
->orderBy('name')
->get()
->filter(fn (V5Server $server) => $server->isIngress())
->values()
->map(fn (V5Server $server, int $index) => $this->serializeCaddyIngress($server, $index))
->all();
}
/**
* @param array{uuid: string} $selectedProject
* @param array{uuid: string} $selectedEnvironment
* @return Builder<V5Application>
*/
protected function applicationQuery(Team $currentTeam, array $selectedProject, array $selectedEnvironment): Builder
{
return V5Application::query()
->where('team_id', $currentTeam->id)
->whereHas('project', fn (Builder $query) => $query
->where('team_id', $currentTeam->id)
->where('uuid', $selectedProject['uuid']))
->whereHas('environment', fn (Builder $query) => $query
->where('uuid', $selectedEnvironment['uuid']));
}
}
@@ -0,0 +1,30 @@
<?php
namespace App\Http\Controllers\V5\Concerns;
use Illuminate\Http\Request;
trait ValidatesBuilderConfiguration
{
/**
* @return array<int, string>
*/
protected function builderCapacityRules(bool $builderEnabled, bool $required = false): array
{
return [
$required ? 'required' : 'sometimes',
'integer',
$builderEnabled ? 'min:1' : 'min:0',
'max:1000',
];
}
protected function requestedBuilderEnabled(Request $request, bool $default): bool
{
if (! $request->has('builder_enabled')) {
return $default;
}
return $request->boolean('builder_enabled');
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,330 @@
<?php
namespace App\Http\Controllers\V5;
use App\Http\Controllers\Controller;
use App\Http\Controllers\V5\Concerns\ResolvesCurrentTeam;
use App\Http\Controllers\V5\Concerns\ResolvesProjectSelection;
use App\Models\Environment;
use App\Models\Project;
use App\Models\Team;
use App\Models\V5\Application as V5Application;
use App\Models\V5\ResourceConnection;
use App\Services\Flux\FluxClient;
use App\Support\V5\ConnectionFirewallSync;
use App\Support\V5\ResourceConnectionSerializer;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Http\Response;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Validation\Rule;
class ResourceConnectionController extends Controller
{
use ResolvesCurrentTeam;
use ResolvesProjectSelection;
public function __construct(
private readonly ConnectionFirewallSync $firewallSync,
private readonly ResourceConnectionSerializer $connectionSerializer,
) {}
public function store(Request $request): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$projects = $this->projects($currentTeam);
[$selectedProject, $selectedEnvironment] = $this->selectedProjectAndEnvironment($request, $projects);
if ($selectedProject === null || $selectedEnvironment === null) {
return response()->json([
'message' => 'Select a project and environment before connecting resources.',
], 422);
}
$project = $this->projectQuery($currentTeam)
->where('uuid', $selectedProject['uuid'])
->first();
if (! $project instanceof Project) {
abort(403);
}
$environment = $this->selectedEnvironment($project, $selectedEnvironment['uuid']);
if (! $environment instanceof Environment) {
abort(403);
}
$validated = $request->validate([
'resource_one' => ['required', 'array'],
'resource_one.type' => ['required', 'string', Rule::in(['application'])],
'resource_one.uuid' => ['required', 'string', 'max:255'],
'resource_two' => ['required', 'array'],
'resource_two.type' => ['required', 'string', Rule::in(['application'])],
'resource_two.uuid' => ['required', 'string', 'max:255'],
]);
$resourceOne = $this->resolveConnectableResource($currentTeam, $project, $environment, $validated['resource_one']);
$resourceTwo = $this->resolveConnectableResource($currentTeam, $project, $environment, $validated['resource_two']);
if ($this->resourceIdentity($resourceOne) === $this->resourceIdentity($resourceTwo)) {
return response()->json([
'message' => 'A resource cannot connect to itself.',
], 422);
}
$connection = ResourceConnection::query()->firstOrCreate(
[
'team_id' => $currentTeam->id,
'resource_pair_key' => $this->resourcePairKey($resourceOne, $resourceTwo),
],
[
'project_id' => $project->id,
'environment_id' => $environment->id,
'resource_one_type' => $resourceOne->getMorphClass(),
'resource_one_id' => $resourceOne->getKey(),
'resource_two_type' => $resourceTwo->getMorphClass(),
'resource_two_id' => $resourceTwo->getKey(),
'created_by_user_id' => $request->user()->id,
],
);
return response()->json([
'connection' => $this->connectionSerializer->serialize($connection->load('rules')),
], $connection->wasRecentlyCreated ? 201 : 200);
}
/**
* Update the connection's rules, then converge the node firewalls.
*
* Ordering & failure semantics:
* 1. Snapshot the current DB rules and their firewall representation; abort
* with 502 before mutating anything when the snapshot cannot be built.
* 2. Commit the requested rules in a DB transaction the DB always holds
* the desired state.
* 3. Converge the node firewalls through Flux. Nodes whose coold lacks the
* firewall verbs (UnsupportedCooldVerb) are tolerated: the committed
* rules are kept and the request succeeds.
* 4. On a real Flux failure the previous rules are restored in a second DB
* transaction, the node firewalls are rolled back to the restored rules
* best-effort (warning-logged when that also fails deterministic rule
* ids keep a later re-sync idempotent), and the original error surfaces
* to the caller as a 502 {message, detail} response.
*/
public function update(Request $request, ResourceConnection $connection, FluxClient $fluxClient): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('update', [$connection, $currentTeam]);
$validated = $request->validate([
'ports_by_direction' => ['present', 'array'],
'ports_by_direction.*' => ['array'],
'ports_by_direction.*.*' => ['integer', 'min:1', 'max:65535', 'distinct'],
]);
$connection->load('rules');
$oldRulePayloads = $connection->rules
->map(fn ($rule): array => [
'source_resource_type' => $rule->source_resource_type,
'source_resource_id' => $rule->source_resource_id,
'target_resource_type' => $rule->target_resource_type,
'target_resource_id' => $rule->target_resource_id,
'protocol' => $rule->protocol,
'port' => $rule->port,
])
->all();
try {
$oldFirewallRules = $this->firewallSync->rulesFor($connection);
} catch (\RuntimeException $exception) {
report($exception);
Log::warning('V5 resource connection firewall snapshot failed', [
'connection_id' => $connection->id,
'message' => $exception->getMessage(),
]);
return response()->json([
'message' => 'Could not sync firewall rules through Flux.',
'detail' => 'The connection was left unchanged. Check the server diagnostics and try again.',
], 502);
}
DB::transaction(function () use ($connection, $validated): void {
$connection->rules()->delete();
$resourcesByUuid = $this->connectionSerializer->applicationsByUuid($connection);
foreach ($validated['ports_by_direction'] as $direction => $ports) {
[$sourceResourceUuid, $targetResourceUuid] = array_pad(explode('->', (string) $direction, 2), 2, null);
$sourceResource = is_string($sourceResourceUuid) ? $resourcesByUuid->get($sourceResourceUuid) : null;
$targetResource = is_string($targetResourceUuid) ? $resourcesByUuid->get($targetResourceUuid) : null;
if (! $sourceResource instanceof V5Application || ! $targetResource instanceof V5Application) {
continue;
}
foreach (array_unique($ports) as $port) {
$connection->rules()->create([
'source_resource_type' => $this->resourceTypeForConnectionUuid($connection, $sourceResource->uuid),
'source_resource_id' => $sourceResource->id,
'target_resource_type' => $this->resourceTypeForConnectionUuid($connection, $targetResource->uuid),
'target_resource_id' => $targetResource->id,
'protocol' => 'tcp',
'port' => (int) $port,
]);
}
}
});
$connection->refresh()->load('rules');
$newFirewallRules = null;
try {
$newFirewallRules = $this->firewallSync->rulesFor($connection);
$this->firewallSync->sync($fluxClient, $oldFirewallRules, $newFirewallRules);
} catch (\RuntimeException $exception) {
$this->restoreConnectionRules($connection, $oldRulePayloads);
$this->rollBackFirewallRules($fluxClient, $connection, $newFirewallRules, $oldFirewallRules);
report($exception);
Log::warning('V5 resource connection firewall sync failed', [
'connection_id' => $connection->id,
'message' => $exception->getMessage(),
]);
return response()->json([
'message' => 'Could not sync firewall rules through Flux.',
'detail' => 'The previous rules were restored. Check the server diagnostics and try again.',
], 502);
}
return response()->json([
'connection' => $this->connectionSerializer->serialize($connection),
]);
}
/**
* Delete the connection using revoke-first ordering.
*
* The node firewall rules are revoked before any DB rows are removed. When
* a revoke fails with a real error the delete is aborted with a 502
* {message, detail} response so the DB never loses track of rules that may
* still be open on a reachable node; UnsupportedCooldVerb and
* already-missing rules are tolerated. When the firewall snapshot cannot
* be built (an endpoint lost its server host id) the node cannot be
* addressed at all, so the failure is reported and the delete proceeds.
* Deterministic rule ids make a retried delete revoke the same node-side
* rules idempotently.
*/
public function destroy(Request $request, ResourceConnection $connection, FluxClient $fluxClient): Response|JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('delete', [$connection, $currentTeam]);
try {
$oldFirewallRules = $this->firewallSync->rulesFor($connection->load('rules'));
} catch (\RuntimeException $exception) {
report($exception);
$oldFirewallRules = collect();
}
try {
$this->firewallSync->sync($fluxClient, $oldFirewallRules, collect());
} catch (\RuntimeException $exception) {
report($exception);
Log::warning('V5 resource connection firewall revoke failed', [
'connection_id' => $connection->id,
'message' => $exception->getMessage(),
]);
return response()->json([
'message' => 'Could not sync firewall rules through Flux.',
'detail' => 'The connection was not deleted. Check the server diagnostics and try again.',
], 502);
}
$connection->delete();
return response()->noContent();
}
/**
* @param array{type: string, uuid: string} $resource
*/
private function resolveConnectableResource(Team $team, Project $project, Environment $environment, array $resource): Model
{
return match ($resource['type']) {
'application' => V5Application::query()
->where('team_id', $team->id)
->where('project_id', $project->id)
->where('environment_id', $environment->id)
->where('uuid', $resource['uuid'])
->firstOrFail(),
};
}
private function resourcePairKey(Model $resourceOne, Model $resourceTwo): string
{
return collect([
$this->resourceIdentity($resourceOne),
$this->resourceIdentity($resourceTwo),
])->sort()->implode('|');
}
private function resourceIdentity(Model $resource): string
{
return $resource->getMorphClass().':'.$resource->getKey();
}
private function resourceTypeForConnectionUuid(ResourceConnection $connection, string $resourceUuid): string
{
$resourcesByUuid = $this->connectionSerializer->applicationsByUuid($connection);
$resource = $resourcesByUuid->get($resourceUuid);
return $resource instanceof V5Application && (int) $connection->resource_one_id === $resource->id
? $connection->resource_one_type
: $connection->resource_two_type;
}
/**
* @param array<int, array<string, mixed>> $rulePayloads
*/
private function restoreConnectionRules(ResourceConnection $connection, array $rulePayloads): void
{
DB::transaction(function () use ($connection, $rulePayloads): void {
$connection->rules()->delete();
foreach ($rulePayloads as $rulePayload) {
$connection->rules()->create($rulePayload);
}
});
}
/**
* Best-effort roll back of a partially converged node firewall to the
* restored rules after a failed forward sync. Skipped when the forward
* sync never started (the node was not touched). Failures are only logged
* because the DB already holds the restored, authoritative rules and the
* deterministic rule ids keep a later re-sync idempotent.
*
* @param Collection<int, array{id: string, hostId: string, rule: array{id: string, namespace: string, src: string, dst: string, proto: string, port: int}}>|null $attemptedFirewallRules
* @param Collection<int, array{id: string, hostId: string, rule: array{id: string, namespace: string, src: string, dst: string, proto: string, port: int}}> $restoredFirewallRules
*/
private function rollBackFirewallRules(FluxClient $fluxClient, ResourceConnection $connection, ?Collection $attemptedFirewallRules, Collection $restoredFirewallRules): void
{
if (! $attemptedFirewallRules instanceof Collection) {
return;
}
try {
$this->firewallSync->sync($fluxClient, $attemptedFirewallRules, $restoredFirewallRules);
} catch (\RuntimeException $exception) {
Log::warning('V5 resource connection firewall rollback failed; node firewall may diverge from the restored rules until the next sync', [
'connection_id' => $connection->id,
'message' => $exception->getMessage(),
]);
}
}
}
@@ -0,0 +1,761 @@
<?php
namespace App\Http\Controllers\V5;
use App\Actions\V5\Proxy\StartCaddyIngress;
use App\Actions\V5\Proxy\StopCaddyIngress;
use App\Actions\V5\Server\RemoveBootstrapMarker;
use App\Enums\V5\ServerStatus;
use App\Events\V5ClusterUpdated;
use App\Http\Controllers\Controller;
use App\Http\Controllers\V5\Concerns\HandlesIngressSyncErrors;
use App\Http\Controllers\V5\Concerns\ResolvesCurrentTeam;
use App\Http\Controllers\V5\Concerns\ValidatesBuilderConfiguration;
use App\Jobs\V5BootstrapServerJob;
use App\Models\PrivateKey;
use App\Models\V5\Application as V5Application;
use App\Models\V5\Cluster as V5Cluster;
use App\Models\V5\Server as V5Server;
use App\Rules\ValidServerIp;
use App\Services\Flux\FluxClient;
use App\Support\V5\ClusterSerializer;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Http\Response;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
use Illuminate\Validation\Rule;
class ServerController extends Controller
{
use HandlesIngressSyncErrors;
use ResolvesCurrentTeam;
use ValidatesBuilderConfiguration;
public function store(Request $request, V5Cluster $cluster): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('create', [V5Server::class, $currentTeam, $cluster]);
$validated = $request->validate([
'name' => ['required', 'string', 'max:255'],
'host' => [
'required',
'string',
'max:255',
$this->noControlCharactersRule(),
new ValidServerIp,
Rule::unique('v5_servers', 'host')
->where('team_id', $currentTeam->id)
->where('ssh_port', (int) $request->input('ssh_port', 22)),
],
'ssh_user' => ['required', 'string', 'max:255', 'regex:/^[A-Za-z0-9._-]+$/', $this->noControlCharactersRule()],
'ssh_port' => ['required', 'integer', 'min:1', 'max:65535'],
'private_key_uuid' => [
'required',
'string',
Rule::exists('private_keys', 'uuid')->where('team_id', $currentTeam->id),
],
'node_address' => [
'nullable',
'string',
'max:255',
$this->noControlCharactersRule(),
new ValidServerIp,
Rule::unique('v5_servers', 'node_address')->where('team_id', $currentTeam->id),
],
'builder_enabled' => ['sometimes', 'boolean'],
'builder_capacity' => $this->builderCapacityRules(
$this->requestedBuilderEnabled($request, $cluster->builder_enabled)
),
'builder_cpu_quota' => ['sometimes', 'string', 'max:32'],
'wireguard_listen_port_override' => ['nullable', 'integer', 'min:1', 'max:65535'],
'wireguard_endpoint_override' => [
'nullable',
'string',
'max:255',
$this->noControlCharactersRule(),
$this->hostPortRule(),
Rule::unique('v5_servers', 'wireguard_endpoint_override')->where('cluster_id', $cluster->id),
],
'ingress_enabled' => ['sometimes', 'boolean'],
'ingress_type' => [
Rule::requiredIf(fn () => $request->boolean('ingress_enabled')),
'nullable',
'string',
Rule::in(['caddy']),
],
]);
$capacity = $this->clusterServerCapacity($cluster);
if ($capacity !== null && $cluster->servers()->count() >= $capacity) {
return response()->json([
'message' => "This cluster's network pools are full ({$capacity} server(s) max). Grow the pools or remove a server first.",
], 422);
}
$builderEnabled = (bool) ($validated['builder_enabled'] ?? $cluster->builder_enabled);
$ingressEnabled = (bool) ($validated['ingress_enabled'] ?? false);
$ingressType = $ingressEnabled ? $validated['ingress_type'] : null;
$builderCapacity = (int) ($validated['builder_capacity'] ?? $cluster->builder_capacity);
$builderCpuQuota = $validated['builder_cpu_quota'] ?? $cluster->builder_cpu_quota;
$devWireguardOverrides = $this->devLimaWireguardOverrides($validated['host'], (int) $validated['ssh_port']);
$privateKey = PrivateKey::query()
->where('team_id', $currentTeam->id)
->where('uuid', $validated['private_key_uuid'])
->firstOrFail();
V5Server::query()->create([
'team_id' => $currentTeam->id,
'cluster_id' => $cluster->id,
'created_by_user_id' => $request->user()->id,
'name' => $validated['name'],
'host' => $validated['host'],
'ssh_user' => $validated['ssh_user'],
'ssh_port' => $validated['ssh_port'],
'private_key_id' => $privateKey->id,
'status' => ServerStatus::Added->value,
'ingress_type' => $ingressType,
'is_ingress' => $ingressEnabled,
'builder_enabled' => $builderEnabled,
'builder_capacity' => $builderCapacity,
'builder_cpu_quota' => $builderCpuQuota,
'node_address' => $validated['node_address'] ?? $validated['host'],
'wireguard_listen_port_override' => $validated['wireguard_listen_port_override'] ?? $devWireguardOverrides['listen_port'],
'wireguard_endpoint_override' => $validated['wireguard_endpoint_override'] ?? $devWireguardOverrides['endpoint'],
]);
return response()->json([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
], 201);
}
public function update(Request $request, V5Cluster $cluster, V5Server $server): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('update', [$server, $currentTeam, $cluster]);
$validated = $request->validate([
'builder_enabled' => ['required', 'boolean'],
'builder_capacity' => $this->builderCapacityRules(
$request->boolean('builder_enabled'),
required: true
),
'builder_cpu_quota' => ['required', 'string', 'max:32'],
'ingress_enabled' => ['sometimes', 'boolean'],
'ingress_type' => [
Rule::requiredIf(fn () => $request->boolean('ingress_enabled')),
'nullable',
'string',
Rule::in(['caddy']),
],
]);
$wasIngress = $server->isIngress();
$builderEnabled = (bool) $validated['builder_enabled'];
$ingressEnabled = (bool) ($validated['ingress_enabled'] ?? $wasIngress);
$ingressType = $ingressEnabled ? ($validated['ingress_type'] ?? $server->ingress_type ?? 'caddy') : null;
$originalServerAttributes = $server->only([
'is_ingress',
'ingress_type',
'ingress_status',
'builder_enabled',
'builder_capacity',
'builder_cpu_quota',
]);
// Stop the ingress before persisting the change: StopCaddyIngress needs
// the server's current ingress state, and a failed stop must leave the
// capability untouched.
if ($wasIngress && ! $ingressEnabled && $server->status === ServerStatus::Installed->value) {
try {
StopCaddyIngress::run($server);
} catch (\RuntimeException $exception) {
return $this->ingressSyncErrorResponse($exception);
}
}
$server->update([
'is_ingress' => $ingressEnabled,
'ingress_type' => $ingressType,
'builder_enabled' => $builderEnabled,
'builder_capacity' => (int) $validated['builder_capacity'],
'builder_cpu_quota' => $validated['builder_cpu_quota'],
]);
$server->refresh();
if (! $wasIngress && $ingressEnabled && $server->status === ServerStatus::Installed->value) {
try {
StartCaddyIngress::run($server);
} catch (\RuntimeException $exception) {
$server->update($originalServerAttributes);
return $this->ingressSyncErrorResponse($exception);
}
}
return response()->json([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
]);
}
public function check(Request $request, V5Cluster $cluster, V5Server $server): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('check', [$server, $currentTeam, $cluster]);
if (! $server->privateKey instanceof PrivateKey) {
return response()->json([
'status' => 'failed',
'output' => 'No private key is attached to this server.',
'checkedAt' => now()->toJSON(),
]);
}
$keyDirectory = storage_path('app/ssh/keys');
if (! is_dir($keyDirectory)) {
mkdir($keyDirectory, 0700, true);
}
$keyLocation = tempnam($keyDirectory, 'v5_ssh_key_');
if ($keyLocation === false) {
return response()->json([
'status' => 'failed',
'output' => 'Could not create a temporary SSH key file.',
'checkedAt' => now()->toJSON(),
]);
}
file_put_contents($keyLocation, $server->privateKey->private_key);
chmod($keyLocation, 0600);
$target = "{$server->ssh_user}@{$server->host}";
$command = [
'ssh',
'-o',
'BatchMode=yes',
'-o',
'LogLevel=ERROR',
'-o',
'StrictHostKeyChecking=no',
'-o',
'UserKnownHostsFile=/dev/null',
'-o',
'ConnectTimeout=10',
'-o',
'IdentitiesOnly=yes',
'-i',
$keyLocation,
'-p',
(string) $server->ssh_port,
$target,
"printf 'SSH connection OK\n'; hostname; uname -srm; command -v docker || true; command -v podman || true",
];
try {
$result = Process::timeout(15)->run($command);
$output = trim($result->output()."\n".$result->errorOutput());
$status = $result->successful() ? 'reachable' : 'failed';
} catch (\Throwable $e) {
$output = $e->getMessage();
$status = 'failed';
} finally {
@unlink($keyLocation);
}
return response()->json([
'status' => $status,
'output' => str($output !== '' ? $output : 'No output returned.')->limit(10000)->toString(),
'checkedAt' => now()->toJSON(),
]);
}
public function cooldLogs(Request $request, V5Cluster $cluster, V5Server $server, FluxClient $fluxClient): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('viewDiagnostics', [$server, $currentTeam, $cluster]);
$validated = $request->validate([
'tail' => ['sometimes', 'integer', 'min:1', 'max:1000'],
]);
$hostId = $server->fluxHostId();
if (! is_string($hostId) || $hostId === '') {
return response()->json([
'message' => 'This server is missing its Flux host id.',
], 422);
}
try {
$output = $fluxClient->cooldLogs($hostId, (int) ($validated['tail'] ?? 200));
} catch (\Throwable $e) {
Log::warning('V5 coold logs request failed', [
'server_id' => $server->id,
'message' => $e->getMessage(),
]);
if ($server->privateKey instanceof PrivateKey) {
try {
return response()->json([
'output' => $this->cooldLogsOverSsh($server, (int) ($validated['tail'] ?? 200)),
'source' => 'ssh',
'fetchedAt' => now()->toJSON(),
]);
} catch (\Throwable $sshException) {
Log::warning('V5 coold logs SSH fallback failed', [
'server_id' => $server->id,
'message' => $sshException->getMessage(),
]);
}
}
return response()->json([
'message' => 'Could not fetch coold logs through Flux. Check the Flux and coold status, then try again.',
], 502);
}
return response()->json([
'output' => $output,
'source' => 'flux',
'fetchedAt' => now()->toJSON(),
]);
}
private function cooldLogsOverSsh(V5Server $server, int $tail): string
{
return $this->runServerSshCommand(
$server,
'sudo -n journalctl -u coold -n '.max(1, min($tail, 1000)).' --no-pager -q || journalctl -u coold -n '.max(1, min($tail, 1000)).' --no-pager -q',
'SSH coold log command failed.',
);
}
private function runServerSshCommand(V5Server $server, string $remoteCommand, string $failureMessage): string
{
$keyDirectory = storage_path('app/ssh/keys');
if (! is_dir($keyDirectory)) {
mkdir($keyDirectory, 0700, true);
}
$keyLocation = tempnam($keyDirectory, 'v5_ssh_key_');
if ($keyLocation === false) {
throw new \RuntimeException('Could not create a temporary SSH key file.');
}
file_put_contents($keyLocation, $server->privateKey->private_key);
chmod($keyLocation, 0600);
try {
$result = Process::timeout(15)->run([
'ssh',
'-o',
'BatchMode=yes',
'-o',
'LogLevel=ERROR',
'-o',
'StrictHostKeyChecking=no',
'-o',
'UserKnownHostsFile=/dev/null',
'-o',
'ConnectTimeout=10',
'-o',
'IdentitiesOnly=yes',
'-i',
$keyLocation,
'-p',
(string) $server->ssh_port,
"{$server->ssh_user}@{$server->host}",
$remoteCommand,
]);
$output = trim($result->output()."\n".$result->errorOutput());
if (! $result->successful()) {
throw new \RuntimeException($output !== '' ? $output : $failureMessage);
}
return str($output)->limit(10000)->toString();
} finally {
@unlink($keyLocation);
}
}
public function corrosionTables(Request $request, V5Cluster $cluster, V5Server $server, FluxClient $fluxClient): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('viewDiagnostics', [$server, $currentTeam, $cluster]);
$validated = $request->validate([
'limit' => ['sometimes', 'integer', 'min:1', 'max:1000'],
]);
$hostId = $server->fluxHostId();
if (! is_string($hostId) || $hostId === '') {
return response()->json([
'message' => 'This server is missing its Flux host id.',
], 422);
}
try {
$output = $fluxClient->corrosionTables($hostId, (int) ($validated['limit'] ?? 200));
} catch (\Throwable $e) {
Log::warning('V5 corrosion tables request failed', [
'server_id' => $server->id,
'message' => $e->getMessage(),
]);
if ($server->privateKey instanceof PrivateKey) {
try {
return response()->json([
'output' => $this->corrosionTablesOverSsh($server, $cluster, (int) ($validated['limit'] ?? 200)),
'source' => 'ssh',
'fetchedAt' => now()->toJSON(),
]);
} catch (\Throwable $sshException) {
Log::warning('V5 corrosion tables SSH fallback failed', [
'server_id' => $server->id,
'message' => $sshException->getMessage(),
]);
}
}
return response()->json([
'message' => 'Could not fetch corrosion tables through Flux. Check the Flux and coold status, then try again.',
], 502);
}
return response()->json([
'output' => $output,
'source' => 'flux',
'fetchedAt' => now()->toJSON(),
]);
}
private function corrosionTablesOverSsh(V5Server $server, V5Cluster $cluster, int $limit): string
{
$limit = max(1, min($limit, 1000));
$script = <<<'PYTHON'
python3 - <<'PY'
import json
import urllib.request
limit = __LIMIT__
url = "http://127.0.0.1:__PORT__/v1/queries"
def query(sql):
request = urllib.request.Request(
url,
data=json.dumps([sql, []]).encode(),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=10) as response:
return json.loads(response.read().decode())
def quote_identifier(value):
return '"' + value.replace('"', '""') + '"'
tables = []
for row in query("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name"):
name = row[0] if row else None
if not isinstance(name, str):
continue
identifier = quote_identifier(name)
columns = [column[1] for column in query(f"PRAGMA table_info({identifier})") if len(column) > 1]
rows = query(f"SELECT * FROM {identifier} LIMIT {limit}")
tables.append({"name": name, "columns": columns, "rows": rows})
print(json.dumps({"limit": limit, "tables": tables}, separators=(",", ":")))
PY
PYTHON;
return $this->runServerSshCommand($server, str_replace(
['__LIMIT__', '__PORT__'],
[(string) $limit, (string) $cluster->corrosion_api_port],
$script,
), 'SSH corrosion table command failed.');
}
public function firewallRules(Request $request, V5Cluster $cluster, V5Server $server, FluxClient $fluxClient): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('viewDiagnostics', [$server, $currentTeam, $cluster]);
$validated = $request->validate([
'namespace' => ['sometimes', 'string', 'max:63'],
]);
$hostId = $server->fluxHostId();
if (! is_string($hostId) || $hostId === '') {
return response()->json([
'message' => 'This server is missing its Flux host id.',
], 422);
}
try {
$rules = $fluxClient->listFirewallRules($hostId, (string) ($validated['namespace'] ?? ''));
} catch (\Throwable $e) {
Log::warning('V5 firewall rules request failed', [
'server_id' => $server->id,
'message' => $e->getMessage(),
]);
return response()->json([
'message' => 'Could not fetch firewall rules through Flux. Check the Flux and coold status, then try again.',
], 502);
}
return response()->json([
'rules' => $rules,
'source' => 'flux',
'fetchedAt' => now()->toJSON(),
]);
}
public function bootstrap(Request $request, V5Cluster $cluster, V5Server $server): JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('bootstrap', [$server, $currentTeam, $cluster]);
// Fail fast: the bootstrap job hard-fails on Flux enrollment (after
// the WireGuard mesh is already built) when no Flux URL is configured.
if (trim((string) config('coold.flux_url', '')) === '') {
return response()->json([
'message' => 'COOLIFY_COOLD_FLUX_URL is not configured, so bootstrapped servers cannot be enrolled into Flux. Set it and retry the bootstrap.',
], 422);
}
if ($server->last_bootstrapped_at !== null) {
return response()->json([
'message' => 'This server is already bootstrapped.',
], 409);
}
$installedServers = $cluster->servers()
->with('privateKey')
->whereNotNull('last_bootstrapped_at')
->orderBy('name')
->get();
$server->load('privateKey');
$servers = $installedServers->toBase()
->push($server)
->unique('id')
->values();
if ($servers->contains(fn (V5Server $server) => ! $server->privateKey instanceof PrivateKey)) {
return response()->json([
'message' => 'The new server and every already-bootstrapped server in this cluster must have a private key before extending the cluster.',
], 422);
}
$claim = DB::transaction(function () use ($cluster, $server, $installedServers): array {
$clusterServers = $cluster->servers()->lockForUpdate()->get();
$activeServer = $clusterServers->first(fn (V5Server $candidate): bool => $this->hasActiveBootstrapClaim($candidate));
if ($activeServer instanceof V5Server) {
return ['claimed' => false, 'active_server_id' => $activeServer->id];
}
// Sweep provably dead claims (lost job or killed worker) so retries
// are possible and the UI reflects reality.
$clusterServers
->filter(fn (V5Server $candidate): bool => in_array($candidate->last_bootstrap_status, ['queued', 'running'], true))
->each(fn (V5Server $candidate) => $candidate->update([
'last_bootstrap_status' => 'failed',
'last_bootstrap_output' => 'The previous bootstrap attempt timed out or its worker died. Retry the bootstrap.',
]));
$server->update([
'last_bootstrap_action' => $installedServers->isEmpty() ? 'bootstrap' : 'extend',
'last_bootstrap_status' => 'queued',
'last_bootstrap_output' => "Queued Coolify bootstrap for {$server->name}.",
'last_bootstrap_ran_at' => now(),
]);
return ['claimed' => true, 'active_server_id' => null];
});
if (! $claim['claimed']) {
return response()->json([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
'message' => $claim['active_server_id'] === $server->id
? 'Bootstrap is already queued or running for this server.'
: 'Another server bootstrap is already queued or running for this cluster.',
], 409);
}
V5ClusterUpdated::dispatch($currentTeam->id, $cluster->id);
V5BootstrapServerJob::dispatch($cluster->id, $server->id);
return response()->json([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
'message' => 'Bootstrap queued.',
], 202);
}
public function destroy(Request $request, V5Cluster $cluster, V5Server $server): Response|JsonResponse
{
$currentTeam = $this->currentTeamOrFail($request);
$this->authorize('delete', [$server, $currentTeam, $cluster]);
if (V5Application::query()->where('server_id', $server->id)->exists()) {
return response()->json([
'message' => 'Delete or move applications from this server before deleting it.',
], 422);
}
$warning = null;
if ($server->last_bootstrapped_at !== null) {
if ($server->isIngress() && $server->status === ServerStatus::Installed->value) {
try {
StopCaddyIngress::run($server);
} catch (\Throwable $exception) {
report($exception);
$warning = 'Could not stop the Caddy ingress on the server before deleting it.';
}
}
if (! RemoveBootstrapMarker::run($server)) {
$warning = 'Could not clean up the server over SSH. Remove /etc/coolify/v5-node.json manually before re-adding this server.';
}
}
$server->delete();
return response()->json(array_filter([
'cluster' => app(ClusterSerializer::class)->serializeFresh($cluster),
'warning' => $warning,
]));
}
/**
* A queued claim is active while the job could still pick it up; a running
* claim is active until the job timeout (plus margin) has passed. Anything
* older is provably dead because the job runs with $tries = 1.
*/
private function hasActiveBootstrapClaim(V5Server $server): bool
{
$ranAt = $server->last_bootstrap_ran_at;
return match ($server->last_bootstrap_status) {
'queued' => $ranAt !== null && $ranAt->gt(now()->subMinutes(15)),
'running' => $ranAt !== null && $ranAt->gt(now()->subSeconds(V5BootstrapServerJob::TIMEOUT_SECONDS + 300)),
default => false,
};
}
/**
* @return array{listen_port: int|null, endpoint: string|null}
*/
private function devLimaWireguardOverrides(string $host, int $sshPort): array
{
if (! app()->environment(['local', 'development', 'testing']) || $host !== 'host.docker.internal') {
return ['listen_port' => null, 'endpoint' => null];
}
if ($sshPort < 60001 || $sshPort > 60009) {
return ['listen_port' => null, 'endpoint' => null];
}
$wireguardPort = $sshPort - 8180;
return [
'listen_port' => $wireguardPort,
'endpoint' => "host.lima.internal:{$wireguardPort}",
];
}
private function clusterServerCapacity(V5Cluster $cluster): ?int
{
$namespaceCount = max(1, count($cluster->namespaces ?? V5Cluster::DEFAULT_NAMESPACES));
[, $poolPrefix] = array_pad(explode('/', (string) $cluster->container_network_pool, 2), 2, null);
$containerPrefix = (int) $cluster->container_network_prefix;
if (! is_string($poolPrefix) || ! ctype_digit($poolPrefix) || $containerPrefix < (int) $poolPrefix || $containerPrefix > 32) {
return null;
}
$containerCapacity = intdiv(2 ** ($containerPrefix - (int) $poolPrefix), $namespaceCount);
[, $managementPrefix] = array_pad(explode('/', (string) $cluster->wireguard_management_pool, 2), 2, null);
$managementCapacity = is_string($managementPrefix) && ctype_digit($managementPrefix) && (int) $managementPrefix <= 30
? (2 ** (32 - (int) $managementPrefix)) - 2
: null;
return $managementCapacity === null ? $containerCapacity : min($containerCapacity, $managementCapacity);
}
private function noControlCharactersRule(): \Closure
{
return function (string $attribute, mixed $value, \Closure $fail): void {
if (! is_string($value)) {
return;
}
if (preg_match('/[\x00-\x1F\x7F]/', $value) === 1) {
$fail('The :attribute contains invalid control characters.');
}
};
}
private function hostPortRule(): \Closure
{
return function (string $attribute, mixed $value, \Closure $fail): void {
if ($value === null || $value === '') {
return;
}
if (! is_string($value)) {
$fail('The :attribute must be in host:port format.');
return;
}
$value = trim($value);
if (preg_match('/^\[(?<host>.+)]:(?<port>\d+)$/', $value, $matches) === 1) {
$host = trim((string) $matches['host']);
$port = trim((string) $matches['port']);
} else {
$separatorPosition = strrpos($value, ':');
if ($separatorPosition === false) {
$fail('The :attribute must be in host:port format.');
return;
}
$host = trim(substr($value, 0, $separatorPosition));
$port = trim(substr($value, $separatorPosition + 1));
if (str_contains($host, ':')) {
$fail('The :attribute must use [ipv6]:port format for IPv6 addresses.');
return;
}
}
if ($host === '' || $port === '' || ! ctype_digit($port) || (int) $port < 1 || (int) $port > 65535) {
$fail('The :attribute must be in host:port format.');
return;
}
$failed = false;
(new ValidServerIp)->validate($attribute, $host, function () use (&$failed): void {
$failed = true;
});
if ($failed) {
$fail('The :attribute must be in host:port format.');
}
};
}
}