tiennm99/coolify
1
0
Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-04-17 17:21:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix SSH multiplexing contention for concurrent scheduled tasks (#6736)

Browse Source 
When multiple scheduled tasks or database backups run concurrently on
the same server, they compete for the same SSH multiplexed connection
socket, causing race conditions and SSH exit code 255 errors.

This fix adds a `disableMultiplexing` parameter to bypass SSH
multiplexing for jobs that may run concurrently:

- Add `disableMultiplexing` param to `generateSshCommand()`
- Add `disableMultiplexing` param to `instant_remote_process()`
- Update `ScheduledTaskJob` to use `disableMultiplexing: true`
- Update `DatabaseBackupJob` to use `disableMultiplexing: true`
- Add debug logging to track execution without multiplexing
- Add unit tests for the new parameter

Each backup and scheduled task now gets an isolated SSH connection,
preventing contention on the shared multiplexed socket.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2025-12-05 09:37:56 +01:00
parent 558a885fdc
commit ed979f42ef
5 changed files with 135 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
app/Jobs/DatabaseBackupJob.php
View File

@@ -121,7 +121,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->container_name = "{$this->database->name}-$serviceUuid";
$this->directory_name = $serviceName.'-'.$this->container_name;
$commands[] = "docker exec $this->container_name env | grep POSTGRES_";
$envs = instant_remote_process($commands, $this->server);
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
$envs = str($envs)->explode("\n");
$user = $envs->filter(function ($env) {
@@ -152,7 +152,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->container_name = "{$this->database->name}-$serviceUuid";
$this->directory_name = $serviceName.'-'.$this->container_name;
$commands[] = "docker exec $this->container_name env | grep MYSQL_";
$envs = instant_remote_process($commands, $this->server);
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
$envs = str($envs)->explode("\n");
$rootPassword = $envs->filter(function ($env) {
@@ -175,7 +175,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->container_name = "{$this->database->name}-$serviceUuid";
$this->directory_name = $serviceName.'-'.$this->container_name;
$commands[] = "docker exec $this->container_name env";
$envs = instant_remote_process($commands, $this->server);
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
$envs = str($envs)->explode("\n");
$rootPassword = $envs->filter(function ($env) {
return str($env)->startsWith('MARIADB_ROOT_PASSWORD=');
@@ -217,7 +217,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
try {
$commands = [];
$commands[] = "docker exec $this->container_name env | grep MONGO_INITDB_";
$envs = instant_remote_process($commands, $this->server);
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
if (filled($envs)) {
$envs = str($envs)->explode("\n");
@@ -508,7 +508,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
}
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -537,7 +537,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
$commands[] = $backupCommand;
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -560,7 +560,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$escapedDatabase = escapeshellarg($database);
$commands[] = "docker exec $this->container_name mysqldump -u root -p\"{$this->database->mysql_root_password}\" $escapedDatabase > $this->backup_location";
}
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -583,7 +583,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$escapedDatabase = escapeshellarg($database);
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p\"{$this->database->mariadb_root_password}\" $escapedDatabase > $this->backup_location";
}
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout);
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
@@ -614,7 +614,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
private function calculate_size()
{
return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false);
return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false, false, null, disableMultiplexing: true);
}
private function upload_to_s3(): void
@@ -637,9 +637,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$fullImageName = $this->getFullImageName();
$containerExists = instant_remote_process(["docker ps -a -q -f name=backup-of-{$this->backup_log_uuid}"], $this->server, false);
$containerExists = instant_remote_process(["docker ps -a -q -f name=backup-of-{$this->backup_log_uuid}"], $this->server, false, false, null, disableMultiplexing: true);
if (filled($containerExists)) {
instant_remote_process(["docker rm -f backup-of-{$this->backup_log_uuid}"], $this->server, false);
instant_remote_process(["docker rm -f backup-of-{$this->backup_log_uuid}"], $this->server, false, false, null, disableMultiplexing: true);
}
if (isDev()) {
@@ -661,7 +661,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
instant_remote_process($commands, $this->server);
instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
$this->s3_uploaded = true;
} catch (\Throwable $e) {
@@ -670,7 +670,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
throw $e;
} finally {
$command = "docker rm -f backup-of-{$this->backup_log_uuid}";
instant_remote_process([$command], $this->server);
instant_remote_process([$command], $this->server, true, false, null, disableMultiplexing: true);
}
}

4
app/Jobs/ScheduledTaskJob.php
View File

@@ -139,7 +139,9 @@ class ScheduledTaskJob implements ShouldQueue
if (count($this->containers) == 1 || str_starts_with($containerName, $this->task->container.'-'.$this->resource->uuid)) {
$cmd = "sh -c '".str_replace("'", "'\''", $this->task->command)."'";
$exec = "docker exec {$containerName} {$cmd}";
$this->task_output = instant_remote_process([$exec], $this->server, true, false, $this->timeout);
// Disable SSH multiplexing to prevent race conditions when multiple tasks run concurrently
// See: https://github.com/coollabsio/coolify/issues/6736
$this->task_output = instant_remote_process([$exec], $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->task_log->update([
'status' => 'success',
'message' => $this->task_output,