fix(deployments): advance queue after cancellations (#11330)

app/Actions/Application/CleanupPreviewDeployment.php

@@ -54,6 +54,14 @@ class CleanupPreviewDeployment
             $server
         );
 
+        if ($result['cancelled_deployments'] > 0) {
+            try {
+                next_after_cancel($server);
+            } catch (\Throwable $e) {
+                \Log::warning("Failed to advance deployment queue after cleaning up preview for application {$application->id}: {$e->getMessage()}");
+            }
+        }
+
         // Step 2: Stop and remove all running PR containers
         $result['killed_containers'] = $this->stopRunningContainers(
             $application,
@@ -98,13 +106,13 @@ class CleanupPreviewDeployment
                 $deployment->update([
                     'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
                 ]);
+                $cancelled++;
 
                 // Add cancellation log entry
                 $deployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
 
                 // Try to kill helper container if it exists
                 $this->killHelperContainer($deployment->deployment_uuid, $server);
-                $cancelled++;
             } catch (\Throwable $e) {
                 \Log::warning("Failed to cancel deployment {$deployment->id}: {$e->getMessage()}");
             }

app/Http/Controllers/Api/DeployController.php

@@ -238,57 +238,71 @@ class DeployController extends Controller
             ApplicationDeploymentStatus::IN_PROGRESS->value,
         ];
 
-        if (! in_array($deployment->status, $cancellableStatuses)) {
+        if (! in_array($deployment->status, $cancellableStatuses, true)) {
             return response()->json([
                 'message' => "Deployment cannot be cancelled. Current status: {$deployment->status}",
             ], 400);
         }
 
         // Perform the cancellation
+        $cancelled = false;
+        $deploymentServer = Server::whereTeamId($teamId)->find($deployment->server_id);
+
         try {
             $deployment_uuid = $deployment->deployment_uuid;
             $kill_command = "docker rm -f {$deployment_uuid}";
             $build_server_id = $deployment->build_server_id ?? $deployment->server_id;
 
             // Mark deployment as cancelled
-            $deployment->update([
-                'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
-            ]);
+            $updated = ApplicationDeploymentQueue::whereKey($deployment->getKey())
+                ->whereIn('status', $cancellableStatuses)
+                ->update(['status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value]);
+
+            if ($updated !== 1) {
+                $deployment->refresh();
+
+                return response()->json([
+                    'message' => "Deployment cannot be cancelled. Current status: {$deployment->status}",
+                ], 400);
+            }
+
+            $deployment->status = ApplicationDeploymentStatus::CANCELLED_BY_USER->value;
+            $cancelled = true;
 
             // Get the server
             $server = Server::whereTeamId($teamId)->find($build_server_id);
 
-            if ($server) {
-                // Add cancellation log entry
-                $deployment->addLogEntry('Deployment cancelled by user via API.', 'stderr');
+            try {
+                if ($server) {
+                    // Add cancellation log entry
+                    $deployment->addLogEntry('Deployment cancelled by user via API.', 'stderr');
 
-                // Check if container exists and kill it
-                $checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
-                $containerExists = instant_remote_process([$checkCommand], $server);
+                    // Check if container exists and kill it
+                    $checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
+                    $containerExists = instant_remote_process([$checkCommand], $server);
 
-                if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
-                    instant_remote_process([$kill_command], $server);
-                    $deployment->addLogEntry('Deployment container stopped.');
-                } else {
-                    $deployment->addLogEntry('Deployment container not yet started. Will be cancelled when job checks status.');
-                }
+                    if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
+                        instant_remote_process([$kill_command], $server);
+                        $deployment->addLogEntry('Deployment container stopped.');
+                    } else {
+                        $deployment->addLogEntry('Deployment container not yet started. Will be cancelled when job checks status.');
+                    }
 
-                // Kill running process if process ID exists
-                if ($deployment->current_process_id) {
-                    try {
+                    // Kill running process if process ID exists
+                    if ($deployment->current_process_id) {
                         $processKillCommand = "kill -9 {$deployment->current_process_id}";
                         instant_remote_process([$processKillCommand], $server);
-                    } catch (\Throwable $e) {
-                        // Process might already be gone
                     }
                 }
+            } catch (\Throwable $e) {
+                \Log::warning("Failed to clean up cancelled deployment {$deployment->id}: {$e->getMessage()}");
             }
 
             auditLog('api.deployment.cancelled', [
                 'team_id' => $teamId,
                 'deployment_uuid' => $deployment->deployment_uuid,
-                'application_id' => $application?->id,
-                'application_uuid' => $application?->uuid,
+                'application_id' => $deployment->application_id,
+                'application_uuid' => $deployment->application?->uuid,
                 'server_id' => $deployment->server_id,
             ]);
 
@@ -301,6 +315,14 @@ class DeployController extends Controller
             return response()->json([
                 'message' => 'Failed to cancel deployment: '.$e->getMessage(),
             ], 500);
+        } finally {
+            if ($cancelled) {
+                try {
+                    next_after_cancel($deploymentServer);
+                } catch (\Throwable $e) {
+                    \Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}");
+                }
+            }
         }
     }
 

app/Jobs/DeleteResourceJob.php

@@ -158,12 +158,15 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
             ])
             ->get();
 
+        $cancelledDeployments = 0;
+
         foreach ($activeDeployments as $activeDeployment) {
             try {
                 // Mark deployment as cancelled
                 $activeDeployment->update([
                     'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
                 ]);
+                $cancelledDeployments++;
 
                 // Add cancellation log entry
                 $activeDeployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
@@ -186,6 +189,14 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue
             }
         }
 
+        if ($cancelledDeployments > 0) {
+            try {
+                next_after_cancel($server);
+            } catch (\Throwable $e) {
+                \Log::warning("Failed to advance deployment queue after deleting preview {$this->resource->id}: {$e->getMessage()}");
+            }
+        }
+
         try {
             if ($server->isSwarm()) {
                 $escapedStackName = escapeshellarg("{$application->uuid}-{$pull_request_id}");

app/Mcp/Tools/CancelDeployment.php

@@ -104,6 +104,13 @@ class CancelDeployment extends Tool
             'server_id' => $deployment->server_id,
         ]);
 
+        try {
+            $deploymentServer = Server::whereTeamId($teamId)->find($deployment->server_id);
+            next_after_cancel($deploymentServer);
+        } catch (\Throwable $e) {
+            \Log::warning("Failed to advance deployment queue after cancelling deployment {$deployment->id}: {$e->getMessage()}");
+        }
+
         return $this->mcpSuccess($request, $this->respond([
             'ok' => true,
             'message' => 'Deployment cancelled successfully.',

tests/Feature/Api/DeploymentCancellationApiTest.php

@@ -1,17 +1,32 @@
 <?php
 
 use App\Enums\ApplicationDeploymentStatus;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
+use App\Models\Environment;
 use App\Models\InstanceSettings;
+use App\Models\Project;
 use App\Models\Server;
+use App\Models\StandaloneDocker;
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
 
 uses(RefreshDatabase::class);
 
 beforeEach(function () {
-    InstanceSettings::updateOrCreate(['id' => 0]);
+    config([
+        'cache.default' => 'array',
+        'session.driver' => 'array',
+        'queue.default' => 'sync',
+        'app.maintenance.driver' => 'file',
+    ]);
+
+    InstanceSettings::unguarded(fn () => InstanceSettings::updateOrCreate(['id' => 0], ['is_api_enabled' => true]));
 
     // Create a team with owner
     $this->team = Team::factory()->create();
@@ -119,23 +134,104 @@ describe('POST /api/v1/deployments/{uuid}/cancel', function () {
     });
 
     test('cancels queued deployment and updates status in database', function () {
+        $otherTeam = Team::factory()->create();
+        $buildServer = Server::factory()->create(['team_id' => $otherTeam->id]);
         $deployment = ApplicationDeploymentQueue::create([
             'deployment_uuid' => 'queued-deployment-uuid',
             'application_id' => 1,
             'server_id' => $this->server->id,
+            'build_server_id' => $buildServer->id,
             'status' => ApplicationDeploymentStatus::QUEUED->value,
         ]);
 
-        $this->withHeaders([
+        $response = $this->withHeaders([
             'Authorization' => 'Bearer '.$this->bearerToken,
             'Content-Type' => 'application/json',
         ])->postJson("/api/v1/deployments/{$deployment->deployment_uuid}/cancel");
 
-        // The controller updates status before SSH calls, so DB state is always correct
+        $response->assertSuccessful()->assertJson([
+            'message' => 'Deployment cancelled successfully.',
+            'deployment_uuid' => $deployment->deployment_uuid,
+            'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
+        ]);
+
         $deployment->refresh();
         expect($deployment->status)->toBe(ApplicationDeploymentStatus::CANCELLED_BY_USER->value);
     });
 
+    test('starts the next queued deployment after cancellation', function () {
+        Queue::fake();
+
+        $otherTeam = Team::factory()->create();
+        $buildServer = Server::factory()->create(['team_id' => $otherTeam->id]);
+        $destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
+        $project = Project::factory()->create(['team_id' => $this->team->id]);
+        $environment = Environment::factory()->create(['project_id' => $project->id]);
+        $application = Application::factory()->create([
+            'environment_id' => $environment->id,
+            'destination_id' => $destination->id,
+            'destination_type' => $destination->getMorphClass(),
+        ]);
+        $deployment = ApplicationDeploymentQueue::create([
+            'deployment_uuid' => 'cancelled-queue-head-uuid',
+            'application_id' => $application->id,
+            'server_id' => $this->server->id,
+            'build_server_id' => $buildServer->id,
+            'destination_id' => $destination->id,
+            'commit' => 'first-commit',
+            'pull_request_id' => 0,
+            'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
+        ]);
+        $nextDeployment = ApplicationDeploymentQueue::create([
+            'deployment_uuid' => 'next-queued-deployment-uuid',
+            'application_id' => $application->id,
+            'server_id' => $this->server->id,
+            'destination_id' => $destination->id,
+            'commit' => 'second-commit',
+            'pull_request_id' => 0,
+            'status' => ApplicationDeploymentStatus::QUEUED->value,
+        ]);
+
+        $response = $this->withHeaders([
+            'Authorization' => 'Bearer '.$this->bearerToken,
+            'Content-Type' => 'application/json',
+        ])->postJson("/api/v1/deployments/{$deployment->deployment_uuid}/cancel");
+
+        $response->assertSuccessful();
+        expect($nextDeployment->fresh()->status)->toBe(ApplicationDeploymentStatus::IN_PROGRESS->value);
+        Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $nextDeployment->id);
+    });
+
+    test('updates only a still cancellable deployment and treats cleanup as best effort', function () {
+        Process::fake(fn () => throw new RuntimeException('SSH unavailable'));
+
+        $deployment = ApplicationDeploymentQueue::create([
+            'deployment_uuid' => 'atomic-cancellation-uuid',
+            'application_id' => 1,
+            'server_id' => $this->server->id,
+            'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
+        ]);
+        $updates = [];
+        DB::listen(function ($query) use (&$updates) {
+            if (str_starts_with(strtolower(ltrim($query->sql)), 'update')) {
+                $updates[] = strtolower($query->sql);
+            }
+        });
+
+        $response = $this->withHeaders([
+            'Authorization' => 'Bearer '.$this->bearerToken,
+            'Content-Type' => 'application/json',
+        ])->postJson("/api/v1/deployments/{$deployment->deployment_uuid}/cancel");
+
+        $response->assertSuccessful();
+        expect($deployment->fresh()->status)->toBe(ApplicationDeploymentStatus::CANCELLED_BY_USER->value)
+            ->and(collect($updates)->contains(
+                fn (string $sql) => str_contains($sql, 'application_deployment_queues')
+                    && str_contains($sql, 'status')
+                    && str_contains($sql, ' in '),
+            ))->toBeTrue();
+    });
+
     test('cancels in-progress deployment and updates status in database', function () {
         $deployment = ApplicationDeploymentQueue::create([
             'deployment_uuid' => 'in-progress-deployment-uuid',

tests/Feature/ApplicationPreviewQueueAdvancementTest.php

@@ -0,0 +1,94 @@
+<?php
+
+use App\Actions\Application\CleanupPreviewDeployment;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Jobs\DeleteResourceJob;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\ApplicationPreview;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+    $this->team = Team::factory()->create();
+    $this->server = Server::factory()->create(['team_id' => $this->team->id]);
+    $this->server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+        'force_disabled' => false,
+    ]);
+    $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
+    $project = Project::factory()->create(['team_id' => $this->team->id]);
+    $environment = $project->environments()->first()
+        ?? Environment::factory()->create(['project_id' => $project->id]);
+    $this->application = Application::factory()->create([
+        'environment_id' => $environment->id,
+        'destination_id' => $this->destination->id,
+        'destination_type' => $this->destination->getMorphClass(),
+    ]);
+    $this->preview = ApplicationPreview::create([
+        'application_id' => $this->application->id,
+        'pull_request_id' => 42,
+        'pull_request_html_url' => 'https://github.com/example/repository/pull/42',
+        'fqdn' => 'https://pr-42.example.com',
+    ]);
+
+    Process::fake(['*' => Process::result(output: '')]);
+    Queue::fake();
+});
+
+function createPreviewDeploymentsForQueueAdvancementTest(): array
+{
+    $activeDeployment = ApplicationDeploymentQueue::create([
+        'application_id' => test()->application->id,
+        'deployment_uuid' => 'preview-active-'.fake()->uuid(),
+        'status' => 'in_progress',
+        'server_id' => test()->server->id,
+        'destination_id' => test()->destination->id,
+        'commit' => 'preview-commit',
+        'pull_request_id' => 42,
+    ]);
+    $nextDeployment = ApplicationDeploymentQueue::create([
+        'application_id' => test()->application->id,
+        'deployment_uuid' => 'preview-next-'.fake()->uuid(),
+        'status' => 'queued',
+        'server_id' => test()->server->id,
+        'destination_id' => test()->destination->id,
+        'commit' => 'next-commit',
+        'pull_request_id' => 0,
+    ]);
+
+    return [$activeDeployment, $nextDeployment];
+}
+
+test('preview cleanup advances the deployment queue after cancelling active deployments', function () {
+    [$activeDeployment, $nextDeployment] = createPreviewDeploymentsForQueueAdvancementTest();
+
+    CleanupPreviewDeployment::run($this->application, 42, $this->preview);
+
+    expect($activeDeployment->fresh()->status)->toBe('cancelled-by-user')
+        ->and($nextDeployment->fresh()->status)->toBe('in_progress');
+    Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $nextDeployment->id);
+});
+
+test('deleting a preview advances the deployment queue after cancelling active deployments', function () {
+    [$activeDeployment, $nextDeployment] = createPreviewDeploymentsForQueueAdvancementTest();
+
+    (new DeleteResourceJob($this->preview))->handle();
+
+    expect($activeDeployment->fresh()->status)->toBe('cancelled-by-user')
+        ->and($nextDeployment->fresh()->status)->toBe('in_progress')
+        ->and(ApplicationPreview::withTrashed()->find($this->preview->id))->toBeNull();
+    Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $nextDeployment->id);
+});

tests/Feature/Mcp/McpReadToolsTest.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Jobs\ApplicationDeploymentJob;
 use App\Mcp\Concerns\BuildsResponse;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
@@ -30,11 +31,19 @@ use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
 use Illuminate\Support\Str;
 
 uses(RefreshDatabase::class);
 
 beforeEach(function () {
+    config([
+        'cache.default' => 'array',
+        'session.driver' => 'array',
+        'queue.default' => 'sync',
+        'app.maintenance.driver' => 'file',
+    ]);
+
     InstanceSettings::query()->where('id', 0)->delete();
     InstanceSettings::query()->delete();
     $settings = new InstanceSettings(['is_mcp_server_enabled' => true]);
@@ -1744,6 +1753,7 @@ test('cancel_deployment cancels team deployment and rejects other team', functio
     Process::fake([
         '*' => Process::result(output: ''),
     ]);
+    Queue::fake();
 
     $deployment = ApplicationDeploymentQueue::create([
         'application_id' => $this->application->id,
@@ -1755,6 +1765,17 @@ test('cancel_deployment cancels team deployment and rejects other team', functio
         'commit' => 'abc',
         'current_process_id' => '12345',
     ]);
+    $nextDeployment = ApplicationDeploymentQueue::create([
+        'application_id' => $this->application->id,
+        'deployment_uuid' => 'dep-next-'.fake()->uuid(),
+        'status' => 'queued',
+        'server_id' => $this->server->id,
+        'destination_id' => $this->destination->id,
+        'application_name' => $this->application->name,
+        'server_name' => $this->server->name,
+        'commit' => 'def',
+        'pull_request_id' => 0,
+    ]);
 
     $token = $this->user->createToken('mcp-cancel', ['read', 'deploy'])->plainTextToken;
     $ok = test()->withHeaders([
@@ -1777,6 +1798,8 @@ test('cancel_deployment cancels team deployment and rejects other team', functio
     expect($body['data']['ok'])->toBeTrue()
         ->and($body['data']['status'])->toBe('cancelled-by-user');
     expect($deployment->fresh()->status)->toBe('cancelled-by-user');
+    expect($nextDeployment->fresh()->status)->toBe('in_progress');
+    Queue::assertPushed(ApplicationDeploymentJob::class, fn (ApplicationDeploymentJob $job) => $job->application_deployment_queue_id === $nextDeployment->id);
 
     $otherTeam = Team::factory()->create();
     $otherServer = Server::factory()->create(['team_id' => $otherTeam->id]);