From 0a6db152167c06384640d959ac84e73093f66463 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:51:11 +0200 Subject: [PATCH] fix(backup): use shared compression for database dumps Centralize compressor selection and apply helper-image compression to dump-all database backups. --- app/Jobs/DatabaseBackupJob.php | 19 ++++++++-- app/Jobs/VolumeBackupJob.php | 17 +++------ app/Support/BackupCompression.php | 20 ++++++++++ tests/Feature/Jobs/DatabaseBackupJobTest.php | 40 ++++++++++++++++++++ tests/Unit/BackupCompressionTest.php | 25 ++++++++++++ 5 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 app/Support/BackupCompression.php create mode 100644 tests/Unit/BackupCompressionTest.php diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index d432a3eeb..82e35b73c 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -18,6 +18,7 @@ use App\Notifications\Database\BackupFailed; use App\Notifications\Database\BackupSuccess; use App\Notifications\Database\BackupSuccessWithS3Warning; use App\Rules\SafeWebhookUrl; +use App\Support\BackupCompression; use App\Support\ClickhouseBackupCommand; use Carbon\Carbon; use Illuminate\Bus\Queueable; @@ -609,7 +610,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue } $escapedUsername = escapeshellarg($this->database->postgres_user); if ($this->backup->dump_all) { - $backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location"; + $backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername"; + $backupCommand = $this->buildCompressedDumpCommand($backupCommand).' > '.escapeshellarg($this->backup_location); } else { // Validate and escape database name to prevent command injection validateShellSafePath($database, 'database name'); @@ -635,7 +637,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $commands[] = 'mkdir -p '.$this->backup_dir; $escapedPassword = escapeshellarg($this->database->mysql_root_password); if ($this->backup->dump_all) { - $commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location"; + $dumpCommand = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false"; + $commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location); } else { // Validate and escape database name to prevent command injection validateShellSafePath($database, 'database name'); @@ -659,7 +662,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue $commands[] = 'mkdir -p '.$this->backup_dir; $escapedPassword = escapeshellarg($this->database->mariadb_root_password); if ($this->backup->dump_all) { - $commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location"; + $dumpCommand = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false"; + $commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location); } else { // Validate and escape database name to prevent command injection validateShellSafePath($database, 'database name'); @@ -806,6 +810,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue return "{$helperImage}:{$latestVersion}"; } + private function buildCompressedDumpCommand(string $dumpCommand): string + { + $cpuPercentage = BackupCompression::cpuPercentage($this->server->settings->backup_compression_cpu_percentage); + $compressorCommand = BackupCompression::compressorCommand($cpuPercentage); + $script = "compressor=\$({$compressorCommand}); exec \$compressor"; + + return $dumpCommand.' | docker run --rm -i '.escapeshellarg($this->getFullImageName()).' sh -c '.escapeshellarg($script); + } + private function markStaleExecutionsAsFailed(): void { try { diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index c1ddaedb8..39998a1f6 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -8,6 +8,7 @@ use App\Models\ScheduledVolumeBackup; use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Rules\SafeWebhookUrl; +use App\Support\BackupCompression; use Carbon\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -77,15 +78,14 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue $source = $this->backup->sourcePath(); $containerName = 'volume-backup-'.$this->execution->uuid; $image = coolifyHelperImage().':'.getHelperVersion(); - $compressionCpuPercentage = $this->compressionCpuPercentage($server); + $compressionCpuPercentage = BackupCompression::cpuPercentage($server->settings->backup_compression_cpu_percentage); $this->logCompressorInDevelopment($image, $server, $compressionCpuPercentage); $verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path) ? 'docker volume inspect '.escapeshellarg($source).' >/dev/null' : 'test -d '.escapeshellarg($source); - $archiveScript = "compressor='gzip -3'; " - ."if command -v pigz >/dev/null 2>&1; then compressor=\"pigz -3 -p \$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; fi; " - .'tar -I "$compressor" -cf - -C /volume .'; + $compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage); + $archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume ."; $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName) .' -v '.escapeshellarg($source.':/volume:ro') .' '.escapeshellarg($image) @@ -344,7 +344,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue return; } - $script = "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi"; + $script = BackupCompression::compressorCommand($compressionCpuPercentage); $compressor = instant_remote_process( ['docker run --rm '.escapeshellarg($image).' sh -c '.escapeshellarg($script)], $server, @@ -361,13 +361,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue ]); } - private function compressionCpuPercentage(Server $server): int - { - $percentage = (int) ($server->settings->backup_compression_cpu_percentage ?? 25); - - return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25; - } - private function removeExpiredBackups(Server $server): void { if ($this->hasRetentionLimits( diff --git a/app/Support/BackupCompression.php b/app/Support/BackupCompression.php new file mode 100644 index 000000000..ad70f1fa0 --- /dev/null +++ b/app/Support/BackupCompression.php @@ -0,0 +1,20 @@ +/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$cpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi"; + } +} diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.php index c21e1be22..7e47126ab 100644 --- a/tests/Feature/Jobs/DatabaseBackupJobTest.php +++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php @@ -1,9 +1,12 @@ and($source)->toContain('mc cp {$escapedBackupLocation} {$escapedS3Destination}') ->and($source)->not->toContain('mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/'); }); + +test('database dump compression uses the helper image and shared CPU setting', function (int $compressionCpuPercentage) { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $backup = new ScheduledDatabaseBackup(['timeout' => 3600]); + $job = new DatabaseBackupJob($backup); + $server = new Server; + $server->setRelation('settings', new ServerSetting([ + 'backup_compression_cpu_percentage' => $compressionCpuPercentage, + ])); + $job->server = $server; + + $command = (new ReflectionClass($job)) + ->getMethod('buildCompressedDumpCommand') + ->invoke($job, 'docker exec database pg_dumpall'); + + expect($command) + ->toStartWith('docker exec database pg_dumpall | docker run --rm -i') + ->toContain('coolify-helper') + ->toContain('command -v pigz') + ->toContain('pigz -3 -p') + ->toContain("\$(nproc) * {$compressionCpuPercentage} + 99") + ->toContain('gzip -3'); +})->with([ + 'low' => 25, + 'high' => 75, +]); + +test('all dump all database commands use shared helper compression', function () { + $source = file_get_contents(app_path('Jobs/DatabaseBackupJob.php')); + + expect($source) + ->toContain('$this->buildCompressedDumpCommand($backupCommand)') + ->toContain('mysqldump -u root') + ->toContain('mariadb-dump -u root') + ->and(substr_count($source, '$this->buildCompressedDumpCommand($dumpCommand)'))->toBe(2) + ->and($source)->not->toContain('| gzip >'); +}); diff --git a/tests/Unit/BackupCompressionTest.php b/tests/Unit/BackupCompressionTest.php new file mode 100644 index 000000000..19348248d --- /dev/null +++ b/tests/Unit/BackupCompressionTest.php @@ -0,0 +1,25 @@ +toBe($expected); +})->with([ + 'null defaults to low' => [null, 25], + 'unsupported value defaults to low' => [30, 25], + 'numeric string is accepted' => ['50', 50], + 'low' => [25, 25], + 'medium' => [50, 50], + 'high' => [75, 75], + 'maximum' => [100, 100], +]); + +it('builds the shared pigz command with gzip fallback', function () { + $command = BackupCompression::compressorCommand(75); + + expect($command) + ->toContain('command -v pigz') + ->toContain('pigz -3 -p') + ->toContain('$(nproc) * 75 + 99') + ->toContain('gzip -3'); +});