fix(deployment): prevent base deployments from being killed when PRs close (#7113)

- Fix container filtering to properly distinguish base deployments (pullRequestId=0) from PR deployments
- Add deployment cancellation when PR closes via webhook to prevent race conditions
- Prevent CleanupHelperContainersJob from killing active deployment containers
- Enhance error messages with exit codes and actual errors instead of vague "Oops" messages
- Protect status transitions in finally blocks to ensure proper job failure handling
This commit is contained in:
Andras Bacsai
2025-11-09 14:41:35 +01:00
parent 775216e7a5
commit 67605d50fc
8 changed files with 269 additions and 40 deletions

View File

@@ -246,6 +246,50 @@ class Github extends Controller
if ($action === 'closed') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
// Cancel any active deployments for this PR immediately
$activeDeployment = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->whereIn('status', [
\App\Enums\ApplicationDeploymentStatus::QUEUED->value,
\App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
])
->first();
if ($activeDeployment) {
try {
// Mark deployment as cancelled
$activeDeployment->update([
'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
// Add cancellation log entry
$activeDeployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
// Check if helper container exists and kill it
$deployment_uuid = $activeDeployment->deployment_uuid;
$server = $application->destination->server;
$checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
$containerExists = instant_remote_process([$checkCommand], $server);
if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
instant_remote_process(["docker rm -f {$deployment_uuid}"], $server);
$activeDeployment->addLogEntry('Deployment container stopped.');
}
// Kill running process if process ID exists
if ($activeDeployment->current_process_id) {
try {
$processKillCommand = "kill -9 {$activeDeployment->current_process_id}";
instant_remote_process([$processKillCommand], $server);
} catch (\Throwable $e) {
// Process might already be gone
}
}
} catch (\Throwable $e) {
// Silently handle errors during deployment cancellation
}
}
DeleteResourceJob::dispatch($found);
$return_payloads->push([
'application' => $application->name,
@@ -481,6 +525,51 @@ class Github extends Controller
if ($action === 'closed' || $action === 'close') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
// Cancel any active deployments for this PR immediately
$activeDeployment = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->whereIn('status', [
\App\Enums\ApplicationDeploymentStatus::QUEUED->value,
\App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
])
->first();
if ($activeDeployment) {
try {
// Mark deployment as cancelled
$activeDeployment->update([
'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
// Add cancellation log entry
$activeDeployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
// Check if helper container exists and kill it
$deployment_uuid = $activeDeployment->deployment_uuid;
$server = $application->destination->server;
$checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
$containerExists = instant_remote_process([$checkCommand], $server);
if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
instant_remote_process(["docker rm -f {$deployment_uuid}"], $server);
$activeDeployment->addLogEntry('Deployment container stopped.');
}
// Kill running process if process ID exists
if ($activeDeployment->current_process_id) {
try {
$processKillCommand = "kill -9 {$activeDeployment->current_process_id}";
instant_remote_process([$processKillCommand], $server);
} catch (\Throwable $e) {
// Process might already be gone
}
}
} catch (\Throwable $e) {
// Silently handle errors during deployment cancellation
}
}
// Clean up any deployed containers
$containers = getCurrentApplicationContainerStatus($application->destination->server, $application->id, $pull_request_id);
if ($containers->isNotEmpty()) {
$containers->each(function ($container) use ($application) {