coolify/bootstrap/helpers/remoteProcess.php

<?php

use App\Enums\ActivityTypes;
use App\Enums\ProcessStatus;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SshRetryHandler;
use App\Jobs\CoolifyTask;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\PrivateKey;
use App\Models\Server;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Str;
use Spatie\Activitylog\Contracts\Activity;

function remote_process(
    Collection|array $command,
    Server $server,
    ?string $type = null,
    ?string $type_uuid = null,
    ?Model $model = null,
    bool $ignore_errors = false,
    $callEventOnFinish = null,
    $callEventData = null
): Activity {
    $type = $type ?? ActivityTypes::INLINE->value;
    $command = $command instanceof Collection ? $command->toArray() : $command;

    if ($server->isNonRoot()) {
        $command = parseCommandsByLineForSudo(collect($command), $server);
    }

    $command_string = implode("\n", $command);

    if (Auth::check()) {
        $teams = Auth::user()->teams->pluck('id');
        if (! $teams->contains($server->team_id) && ! $teams->contains(0)) {
            throw new Exception('User is not part of the team that owns this server');
        }
    }

    SshMultiplexingHelper::ensureMultiplexedConnection($server);

    $properties = [
        'server_uuid' => $server->uuid,
        'command' => $command_string,
        'type' => $type,
        'type_uuid' => $type_uuid,
        'status' => ProcessStatus::QUEUED->value,
        'team_id' => $server->team_id,
    ];

    $activityLog = activity()
        ->withProperties($properties)
        ->event($type);

    if ($model) {
        $activityLog->performedOn($model);
    }

    $activity = $activityLog->log('[]');

    dispatch(new CoolifyTask(
        activity: $activity,
        ignore_errors: $ignore_errors,
        call_event_on_finish: $callEventOnFinish,
        call_event_data: $callEventData,
    ));

    $activity->refresh();

    return $activity;
}

function instant_scp(string $source, string $dest, Server $server, $throwError = true)
{
    return SshRetryHandler::retry(
        function () use ($source, $dest, $server) {
            $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
            $process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            return $output === 'null' ? null : $output;
        },
        [
            'server' => $server->ip,
            'source' => $source,
            'dest' => $dest,
            'function' => 'instant_scp',
        ],
        $throwError
    );
}

/**
 * Download a remote file from a managed server onto the Coolify host via SCP.
 */
function instant_scp_from_server(string $remoteSource, string $localDest, Server $server, $throwError = true)
{
    return SshRetryHandler::retry(
        function () use ($remoteSource, $localDest, $server) {
            $scp_command = SshMultiplexingHelper::generateScpDownloadCommand($server, $remoteSource, $localDest);
            $process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            return $output === 'null' ? null : $output;
        },
        [
            'server' => $server->ip,
            'source' => $remoteSource,
            'dest' => $localDest,
            'function' => 'instant_scp_from_server',
        ],
        $throwError
    );
}

function instant_remote_process_with_timeout(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
{
    $command = $command instanceof Collection ? $command->toArray() : $command;
    if ($server->isNonRoot() && ! $no_sudo) {
        $command = parseCommandsByLineForSudo(collect($command), $server);
    }
    $command_string = implode("\n", $command);

    return SshRetryHandler::retry(
        function () use ($server, $command_string) {
            $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
            $process = Process::timeout(30)->run($sshCommand);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            // Sanitize output to ensure valid UTF-8 encoding
            $output = $output === 'null' ? null : sanitize_utf8_text($output);

            return $output;
        },
        [
            'server' => $server->ip,
            'command_preview' => substr($command_string, 0, 100),
            'function' => 'instant_remote_process_with_timeout',
        ],
        $throwError
    );
}

function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false): ?string
{
    $command = $command instanceof Collection ? $command->toArray() : $command;

    if ($server->isNonRoot() && ! $no_sudo) {
        $command = parseCommandsByLineForSudo(collect($command), $server);
    }
    $command_string = implode("\n", $command);
    $effectiveTimeout = $timeout ?? config('constants.ssh.command_timeout');

    return SshRetryHandler::retry(
        function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
            $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
            $process = Process::timeout($effectiveTimeout)->run($sshCommand);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            // Sanitize output to ensure valid UTF-8 encoding
            $output = $output === 'null' ? null : sanitize_utf8_text($output);

            return $output;
        },
        [
            'server' => $server->ip,
            'command_preview' => substr($command_string, 0, 100),
            'function' => 'instant_remote_process',
        ],
        $throwError
    );
}

function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
{
    $ignoredErrors = collect([
        'Permission denied (publickey',
        'Could not resolve hostname',
    ]);
    $ignored = $ignoredErrors->contains(fn ($error) => Str::contains($errorOutput, $error));

    // Ensure we always have a meaningful error message
    $errorMessage = trim($errorOutput);
    if (empty($errorMessage)) {
        $errorMessage = "SSH command failed with exit code: $exitCode";
    }

    if ($ignored) {
        // TODO: Create new exception and disable in sentry
        throw new RuntimeException($errorMessage, $exitCode);
    }
    throw new RuntimeException($errorMessage, $exitCode);
}

function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null, bool $includeAll = false): Collection
{
    if (is_null($application_deployment_queue)) {
        return collect([]);
    }
    $application = Application::find(data_get($application_deployment_queue, 'application_id'));
    $is_debug_enabled = data_get($application, 'settings.is_debug_enabled');
    $serverTimezone = getServerTimezone(data_get($application, 'destination.server'));

    // Members should never see debug logs, even if an admin enabled debug mode
    if ($is_debug_enabled && auth()->check() && auth()->user()->isMember()) {
        $is_debug_enabled = false;
    }

    $logs = data_get($application_deployment_queue, 'logs');
    if (empty($logs)) {
        return collect([]);
    }

    try {
        $decoded = json_decode(
            $logs,
            associative: true,
            flags: JSON_THROW_ON_ERROR
        );
    } catch (JsonException $e) {
        // If JSON decoding fails, try to clean up the logs and retry
        try {
            // Ensure valid UTF-8 encoding
            $cleaned_logs = sanitize_utf8_text($logs);
            $decoded = json_decode(
                $cleaned_logs,
                associative: true,
                flags: JSON_THROW_ON_ERROR
            );
        } catch (JsonException $e) {
            // If it still fails, return empty collection to prevent crashes
            return collect([]);
        }
    }

    if (! is_array($decoded)) {
        return collect([]);
    }

    $seenCommands = collect();
    $formatted = collect($decoded);
    if (! $is_debug_enabled && ! $includeAll) {
        $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false);
    }

    return $formatted
        ->sortBy(fn ($i) => data_get($i, 'order'))
        ->map(function ($i) use ($serverTimezone) {
            $timestamp = Carbon::parse(data_get($i, 'timestamp'));
            try {
                $timestamp->setTimezone($serverTimezone);
            } catch (Exception) {
                $timestamp->setTimezone('UTC');
            }
            data_set($i, 'timestamp', $timestamp->format('Y-M-d H:i:s'));

            return $i;
        })
        ->reduce(function ($deploymentLogLines, $logItem) use ($seenCommands) {
            $command = data_get($logItem, 'command');
            $isStderr = data_get($logItem, 'type') === 'stderr';
            $isNewCommand = ! is_null($command) && ! $seenCommands->first(function ($seenCommand) use ($logItem) {
                return data_get($seenCommand, 'command') === data_get($logItem, 'command') && data_get($seenCommand, 'batch') === data_get($logItem, 'batch');
            });

            if ($isNewCommand) {
                $deploymentLogLines->push([
                    'line' => $command,
                    'timestamp' => data_get($logItem, 'timestamp'),
                    'stderr' => $isStderr,
                    'hidden' => data_get($logItem, 'hidden'),
                    'command' => true,
                ]);

                $seenCommands->push([
                    'command' => $command,
                    'batch' => data_get($logItem, 'batch'),
                ]);
            }

            $lines = explode(PHP_EOL, data_get($logItem, 'output'));

            foreach ($lines as $line) {
                $deploymentLogLines->push([
                    'line' => $line,
                    'timestamp' => data_get($logItem, 'timestamp'),
                    'stderr' => $isStderr,
                    'hidden' => data_get($logItem, 'hidden'),
                ]);
            }

            return $deploymentLogLines;
        }, collect());
}

function remove_iip($text)
{
    // Ensure the input is valid UTF-8 before processing
    $text = sanitize_utf8_text($text);

    // Git access tokens
    $text = preg_replace('/x-access-token:.*?(?=@)/', 'x-access-token:'.REDACTED, $text);
    $text = preg_replace('/oauth2:.*?(?=@)/', 'oauth2:'.REDACTED, $text);

    // ANSI color codes
    $text = preg_replace('/\x1b\[[0-9;]*m/', '', $text);

    // Generic URLs with passwords (covers database URLs, ftp, amqp, ssh, git basic auth, etc.)
    // (protocol://user:password@host → protocol://user:<REDACTED>@host)
    $text = preg_replace('/((?:https?|postgres|mysql|mongodb|rediss?|mariadb|ftp|sftp|ssh|amqp|amqps|ldap|ldaps|s3):\/\/[^:]+:)[^@]+(@)/i', '$1'.REDACTED.'$2', $text);

    // Email addresses
    $text = preg_replace('/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/', REDACTED, $text);

    // Bearer/JWT tokens
    $text = preg_replace('/Bearer\s+[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+/i', 'Bearer '.REDACTED, $text);

    // GitHub tokens (ghp_ = personal, gho_ = OAuth, ghu_ = user-to-server, ghs_ = server-to-server, ghr_ = refresh)
    $text = preg_replace('/\b(gh[pousr]_[A-Za-z0-9_]{36,})\b/', REDACTED, $text);

    // GitLab tokens (glpat- = personal access token, glcbt- = CI build token, glrt- = runner token)
    $text = preg_replace('/\b(gl(?:pat|cbt|rt)-[A-Za-z0-9\-_]{20,})\b/', REDACTED, $text);

    // AWS credentials (Access Key ID starts with AKIA, ABIA, ACCA, ASIA)
    $text = preg_replace('/\b(A(?:KIA|BIA|CCA|SIA)[A-Z0-9]{16})\b/', REDACTED, $text);

    // AWS Secret Access Key (40 character base64-ish string, typically follows access key)
    $text = preg_replace('/(aws_secret_access_key|AWS_SECRET_ACCESS_KEY)[=:]\s*[\'"]?([A-Za-z0-9\/+=]{40})[\'"]?/i', '$1='.REDACTED, $text);

    // API keys (common patterns)
    $text = preg_replace('/(api[_-]?key|apikey|api[_-]?secret|secret[_-]?key)[=:]\s*[\'"]?[A-Za-z0-9\-_]{16,}[\'"]?/i', '$1='.REDACTED, $text);

    // Private key blocks
    $text = preg_replace('/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/', REDACTED, $text);

    return $text;
}

/**
 * Sanitizes text to ensure it contains valid UTF-8 encoding.
 *
 * This function is crucial for preventing "Malformed UTF-8 characters" errors
 * that can occur when Docker build output contains binary data mixed with text,
 * especially during image processing or builds with many assets.
 *
 * @param  string|null  $text  The text to sanitize
 * @return string Valid UTF-8 encoded text
 */
function sanitize_utf8_text(?string $text): string
{
    if (empty($text)) {
        return '';
    }

    // Convert to UTF-8, replacing invalid sequences
    $sanitized = mb_convert_encoding($text, 'UTF-8', 'UTF-8');

    // Additional fallback: use SUBSTITUTE flag to replace invalid sequences with substitution character
    if (! mb_check_encoding($sanitized, 'UTF-8')) {
        $sanitized = mb_convert_encoding($text, 'UTF-8', mb_detect_encoding($text, mb_detect_order(), true) ?: 'UTF-8');
    }

    return $sanitized;
}

function refresh_server_connection(?PrivateKey $private_key = null)
{
    if (is_null($private_key)) {
        return;
    }
    foreach ($private_key->servers as $server) {
        SshMultiplexingHelper::removeMuxFile($server);
    }
}

function checkRequiredCommands(Server $server)
{
    $commands = collect(['jq', 'jc']);
    foreach ($commands as $command) {
        $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
        if ($commandFound) {
            continue;
        }
        try {
            instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'apt update && apt install -y {$command}'"], $server);
        } catch (Throwable) {
            break;
        }
        $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
        if (! $commandFound) {
            break;
        }
    }
}