tiennm99/coolify
1
0
Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-04-17 17:21:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'next' into macau-v1

Browse Source 
Resolved conflicts in ServerManagerJob.php by:
- Keeping sentinel update check code from macau-v1
- Preserving sentinel restart code from next branch
- Ensuring no duplicate code blocks
This commit is contained in:
Andras Bacsai
2025-12-04 15:07:36 +01:00
parent 4002044877 9e0fa03434
commit 70ff73e954
72 changed files with 2481 additions and 677 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/Jobs/ApplicationDeploymentJob.php
View File

@@ -1813,9 +1813,9 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->application->update(['status' => 'running']);
$this->application_deployment_queue->addLogEntry('New container is healthy.');
break;
}
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') {
} elseif (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') {
$this->newVersionIsHealthy = false;
$this->application_deployment_queue->addLogEntry('New container is unhealthy.', type: 'error');
$this->query_logs();
break;
}
@@ -3187,6 +3187,18 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf");
$this->graceful_shutdown_container($this->container_name);
}
} catch (Exception $e) {
// If new version is healthy, this is just cleanup - don't fail the deployment
if ($this->newVersionIsHealthy || $force) {
$this->application_deployment_queue->addLogEntry(
"Warning: Could not remove old container: {$e->getMessage()}",
'stderr',
hidden: true
);
return; // Don't re-throw - cleanup failures shouldn't fail successful deployments
}
// Only re-throw if deployment hasn't succeeded yet
throw new DeploymentException("Failed to stop running container: {$e->getMessage()}", $e->getCode(), $e);
}
}

14
app/Jobs/CheckTraefikVersionForServerJob.php
View File

@@ -2,6 +2,7 @@
namespace App\Jobs;
use App\Events\ProxyStatusChangedUI;
use App\Models\Server;
use App\Notifications\Server\TraefikVersionOutdated;
use Illuminate\Bus\Queueable;
@@ -38,6 +39,8 @@ class CheckTraefikVersionForServerJob implements ShouldQueue
$this->server->update(['detected_traefik_version' => $currentVersion]);
if (! $currentVersion) {
ProxyStatusChangedUI::dispatch($this->server->team_id);
return;
}
@@ -48,16 +51,22 @@ class CheckTraefikVersionForServerJob implements ShouldQueue
// Handle empty/null response from SSH command
if (empty(trim($imageTag))) {
ProxyStatusChangedUI::dispatch($this->server->team_id);
return;
}
if (str_contains(strtolower(trim($imageTag)), ':latest')) {
ProxyStatusChangedUI::dispatch($this->server->team_id);
return;
}
// Parse current version to extract major.minor.patch
$current = ltrim($currentVersion, 'v');
if (! preg_match('/^(\d+\.\d+)\.(\d+)$/', $current, $matches)) {
ProxyStatusChangedUI::dispatch($this->server->team_id);
return;
}
@@ -77,6 +86,8 @@ class CheckTraefikVersionForServerJob implements ShouldQueue
$this->server->update(['traefik_outdated_info' => null]);
}
ProxyStatusChangedUI::dispatch($this->server->team_id);
return;
}
@@ -96,6 +107,9 @@ class CheckTraefikVersionForServerJob implements ShouldQueue
// Fully up to date
$this->server->update(['traefik_outdated_info' => null]);
}
// Dispatch UI update event so warning state refreshes in real-time
ProxyStatusChangedUI::dispatch($this->server->team_id);
}
/**

8
app/Jobs/DatabaseBackupJob.php
View File

@@ -508,7 +508,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
}
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -537,7 +537,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
$commands[] = $backupCommand;
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -560,7 +560,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$escapedDatabase = escapeshellarg($database);
$commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" $escapedDatabase > $this->backup_location";
}
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -583,7 +583,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$escapedDatabase = escapeshellarg($database);
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" $escapedDatabase > $this->backup_location";
}
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;

6
app/Jobs/PushServerUpdateJob.php
View File

@@ -300,8 +300,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
// Use ContainerStatusAggregator service for state machine logic
// Use preserveRestarting: true so applications show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0, preserveRestarting: true);
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
@@ -360,8 +361,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// Use ContainerStatusAggregator service for state machine logic
// NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0
// Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0);
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0, preserveRestarting: true);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {

129
app/Jobs/RestartProxyJob.php
View File

@@ -2,9 +2,12 @@
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Actions\Proxy\StopProxy;
use App\Actions\Proxy\GetProxyConfiguration;
use App\Actions\Proxy\SaveProxyConfiguration;
use App\Enums\ProxyTypes;
use App\Events\ProxyStatusChangedUI;
use App\Models\Server;
use App\Services\ProxyDashboardCacheService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -19,11 +22,13 @@ class RestartProxyJob implements ShouldBeEncrypted, ShouldQueue
public $tries = 1;
public $timeout = 60;
public $timeout = 120;
public ?int $activity_id = null;
public function middleware(): array
{
return [(new WithoutOverlapping('restart-proxy-'.$this->server->uuid))->expireAfter(60)->dontRelease()];
return [(new WithoutOverlapping('restart-proxy-'.$this->server->uuid))->expireAfter(120)->dontRelease()];
}
public function __construct(public Server $server) {}
@@ -31,15 +36,125 @@ class RestartProxyJob implements ShouldBeEncrypted, ShouldQueue
public function handle()
{
try {
StopProxy::run($this->server, restarting: true);
// Set status to restarting
$this->server->proxy->status = 'restarting';
$this->server->proxy->force_stop = false;
$this->server->save();
StartProxy::run($this->server, force: true, restarting: true);
// Build combined stop + start commands for a single activity
$commands = $this->buildRestartCommands();
// Create activity and dispatch immediately - returns Activity right away
// The remote_process runs asynchronously, so UI gets activity ID instantly
$activity = remote_process(
$commands,
$this->server,
callEventOnFinish: 'ProxyStatusChanged',
callEventData: $this->server->id
);
// Store activity ID and notify UI immediately with it
$this->activity_id = $activity->id;
ProxyStatusChangedUI::dispatch($this->server->team_id, $this->activity_id);
} catch (\Throwable $e) {
// Set error status
$this->server->proxy->status = 'error';
$this->server->save();
// Notify UI of error
ProxyStatusChangedUI::dispatch($this->server->team_id);
// Clear dashboard cache on error
ProxyDashboardCacheService::clearCache($this->server);
return handleError($e);
}
}
/**
* Build combined stop + start commands for proxy restart.
* This creates a single command sequence that shows all logs in one activity.
*/
private function buildRestartCommands(): array
{
$proxyType = $this->server->proxyType();
$containerName = $this->server->isSwarm() ? 'coolify-proxy_traefik' : 'coolify-proxy';
$proxy_path = $this->server->proxyPath();
$stopTimeout = 30;
// Get proxy configuration
$configuration = GetProxyConfiguration::run($this->server);
if (! $configuration) {
throw new \Exception('Configuration is not synced');
}
SaveProxyConfiguration::run($this->server, $configuration);
$docker_compose_yml_base64 = base64_encode($configuration);
$this->server->proxy->last_applied_settings = str($docker_compose_yml_base64)->pipe('md5')->value();
$this->server->save();
$commands = collect([]);
// === STOP PHASE ===
$commands = $commands->merge([
"echo 'Stopping proxy...'",
"docker stop -t=$stopTimeout $containerName 2>/dev/null || true",
"docker rm -f $containerName 2>/dev/null || true",
'# Wait for container to be fully removed',
'for i in {1..15}; do',
" if ! docker ps -a --format \"{{.Names}}\" | grep -q \"^$containerName$\"; then",
" echo 'Container removed successfully.'",
' break',
' fi',
' echo "Waiting for container to be removed... ($i/15)"',
' sleep 1',
' # Force remove on each iteration in case it got stuck',
" docker rm -f $containerName 2>/dev/null || true",
'done',
'# Final verification and force cleanup',
"if docker ps -a --format \"{{.Names}}\" | grep -q \"^$containerName$\"; then",
" echo 'Container still exists after wait, forcing removal...'",
" docker rm -f $containerName 2>/dev/null || true",
' sleep 2',
'fi',
"echo 'Proxy stopped successfully.'",
]);
// === START PHASE ===
if ($this->server->isSwarmManager()) {
$commands = $commands->merge([
"echo 'Starting proxy (Swarm mode)...'",
"mkdir -p $proxy_path/dynamic",
"cd $proxy_path",
"echo 'Creating required Docker Compose file.'",
"echo 'Starting coolify-proxy.'",
'docker stack deploy --detach=true -c docker-compose.yml coolify-proxy',
"echo 'Successfully started coolify-proxy.'",
]);
} else {
if (isDev() && $proxyType === ProxyTypes::CADDY->value) {
$proxy_path = '/data/coolify/proxy/caddy';
}
$caddyfile = 'import /dynamic/*.caddy';
$commands = $commands->merge([
"echo 'Starting proxy...'",
"mkdir -p $proxy_path/dynamic",
"cd $proxy_path",
"echo '$caddyfile' > $proxy_path/dynamic/Caddyfile",
"echo 'Creating required Docker Compose file.'",
"echo 'Pulling docker image.'",
'docker compose pull',
]);
// Ensure required networks exist BEFORE docker compose up
$commands = $commands->merge(ensureProxyNetworksExist($this->server));
$commands = $commands->merge([
"echo 'Starting coolify-proxy.'",
'docker compose up -d --wait --remove-orphans',
"echo 'Successfully started coolify-proxy.'",
]);
$commands = $commands->merge(connectProxyToNetworks($this->server));
}
return $commands->toArray();
}
}

2
app/Jobs/ScheduledTaskJob.php
View File

@@ -139,7 +139,7 @@ class ScheduledTaskJob implements ShouldQueue
if (count($this->containers) == 1 || str_starts_with($containerName, $this->task->container.'-'.$this->resource->uuid)) {
$cmd = "sh -c '".str_replace("'", "'\''", $this->task->command)."'";
$exec = "docker exec {$containerName} {$cmd}";
$this->task_output = instant_remote_process([$exec], $this->server, true);
$this->task_output = instant_remote_process([$exec], $this->server, true, false, $this->timeout);
$this->task_log->update([
'status' => 'success',
'message' => $this->task_output,

46
app/Jobs/ServerManagerJob.php
View File

@@ -111,34 +111,48 @@ class ServerManagerJob implements ShouldQueue
private function processServerTasks(Server $server): void
{
// Get server timezone (used for all scheduled tasks)
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
if (validate_timezone($serverTimezone) === false) {
$serverTimezone = config('app.timezone');
}
// Check if we should run sentinel-based checks
$lastSentinelUpdate = $server->sentinel_updated_at;
$waitTime = $server->waitBeforeDoingSshCheck();
$sentinelOutOfSync = Carbon::parse($lastSentinelUpdate)->isBefore($this->executionTime->subSeconds($waitTime));
$sentinelOutOfSync = Carbon::parse($lastSentinelUpdate)->isBefore($this->executionTime->copy()->subSeconds($waitTime));
if ($sentinelOutOfSync) {
// Dispatch jobs if Sentinel is out of sync
if ($this->shouldRunNow($this->checkFrequency)) {
// Dispatch ServerCheckJob if Sentinel is out of sync
if ($this->shouldRunNow($this->checkFrequency, $serverTimezone)) {
ServerCheckJob::dispatch($server);
}
}
// Dispatch ServerStorageCheckJob if due
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *');
$isSentinelEnabled = $server->isSentinelEnabled();
$shouldRestartSentinel = $isSentinelEnabled && $this->shouldRunNow('0 0 * * *', $serverTimezone);
// Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
if ($shouldRestartSentinel) {
dispatch(function () use ($server) {
$server->restartContainer('coolify-sentinel');
});
}
// Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled)
// When Sentinel is active, PushServerUpdateJob handles storage checks with real-time data
if ($sentinelOutOfSync) {
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *');
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
}
$shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency);
$shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency, $serverTimezone);
if ($shouldRunStorageCheck) {
ServerStorageCheckJob::dispatch($server);
}
}
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
if (validate_timezone($serverTimezone) === false) {
$serverTimezone = config('app.timezone');
}
// Dispatch ServerPatchCheckJob if due (weekly)
$shouldRunPatchCheck = $this->shouldRunNow('0 0 * * 0', $serverTimezone);
@@ -154,16 +168,6 @@ class ServerManagerJob implements ShouldQueue
CheckAndStartSentinelJob::dispatch($server);
}
}
// Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
$isSentinelEnabled = $server->isSentinelEnabled();
$shouldRestartSentinel = $isSentinelEnabled && $this->shouldRunNow('0 0 * * *', $serverTimezone);
if ($shouldRestartSentinel) {
dispatch(function () use ($server) {
$server->restartContainer('coolify-sentinel');
});
}
}
private function shouldRunNow(string $frequency, ?string $timezone = null): bool