tiennm99/coolify
1
0
Fork 0
mirror of https://github.com/tiennm99/coolify.git synced 2026-06-23 23:39:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(jobs): add queue delay resilience to scheduled job execution

Browse Source 
Implement dedup key-based cron tracking to make scheduled jobs resilient to queue
delays. Even if a job is delayed by minutes, it will catch the missed cron window
by tracking previousRunDate in cache instead of relying on isDue() alone.

- Add dedupKey parameter to shouldRunNow() in ScheduledJobManager
  - When provided, uses getPreviousRunDate() + cache tracking for resilience
  - Falls back to isDue() for docker cleanups without dedup key
  - Prevents double-dispatch within same cron window

- Optimize ServerConnectionCheckJob dispatch
  - Skip SSH checks if Sentinel is healthy (enabled and live)
  - Reduces redundant checks when Sentinel heartbeat proves connectivity

- Remove hourly Sentinel update checks
  - Consolidate to daily CheckAndStartSentinelJob dispatch
  - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow

- Add logging for skipped database backups with context (backup_id, database_id, status)

- Refactor skip reason methods to accept server parameter, avoiding redundant queries

- Add comprehensive test suite for scheduling with various delay scenarios and timezones
This commit is contained in:
Andras Bacsai
2026-02-28 15:06:25 +01:00
parent f68793ed69
commit a0c177f6f2
5 changed files with 322 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Unit/ServerManagerJobSentinelCheckTest.php
+63 -80
View File
@@ -1,6 +1,7 @@
<?php
use App\Jobs\CheckAndStartSentinelJob;
use App\Jobs\ServerConnectionCheckJob;
use App\Jobs\ServerManagerJob;
use App\Models\InstanceSettings;
use App\Models\Server;
@@ -10,23 +11,22 @@ use Mockery;
beforeEach(function () {
Queue::fake();
Carbon::setTestNow('2025-01-15 12:00:00'); // Set to top of the hour for cron matching
Carbon::setTestNow('2025-01-15 12:00:00');
});
afterEach(function () {
Mockery::close();
Carbon::setTestNow(); // Reset frozen time
Carbon::setTestNow();
});
it('dispatches CheckAndStartSentinelJob hourly for sentinel-enabled servers', function () {
// Mock InstanceSettings
it('does not dispatch CheckAndStartSentinelJob hourly anymore', function () {
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
// Create a mock server with sentinel enabled
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
$server->shouldReceive('isSentinelLive')->andReturn(true);
$server->id = 1;
$server->name = 'test-server';
$server->ip = '192.168.1.100';
@@ -34,29 +34,76 @@ it('dispatches CheckAndStartSentinelJob hourly for sentinel-enabled servers', fu
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
// Mock the Server query
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
// Execute the job
$job = new ServerManagerJob;
$job->handle();
// Assert CheckAndStartSentinelJob was dispatched for the sentinel-enabled server
Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
// Hourly CheckAndStartSentinelJob dispatch was removed — ServerCheckJob handles it when Sentinel is out of sync
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
});
it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enabled', function () {
// Mock InstanceSettings
it('skips ServerConnectionCheckJob when sentinel is live', function () {
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
$server->shouldReceive('isSentinelLive')->andReturn(true);
$server->id = 1;
$server->name = 'test-server';
$server->ip = '192.168.1.100';
$server->sentinel_updated_at = Carbon::now();
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
$job = new ServerManagerJob;
$job->handle();
// Sentinel is healthy so SSH connection check is skipped
Queue::assertNotPushed(ServerConnectionCheckJob::class);
});
it('dispatches ServerConnectionCheckJob when sentinel is not live', function () {
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
$server->shouldReceive('isSentinelLive')->andReturn(false);
$server->id = 1;
$server->name = 'test-server';
$server->ip = '192.168.1.100';
$server->sentinel_updated_at = Carbon::now()->subMinutes(10);
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
$job = new ServerManagerJob;
$job->handle();
// Sentinel is out of sync so SSH connection check is needed
Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('dispatches ServerConnectionCheckJob when sentinel is not enabled', function () {
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
// Create a mock server with sentinel disabled
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(false);
$server->shouldReceive('isSentinelLive')->never();
$server->id = 2;
$server->name = 'test-server-no-sentinel';
$server->ip = '192.168.1.101';
@@ -64,78 +111,14 @@ it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enab
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
// Mock the Server query
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
// Execute the job
$job = new ServerManagerJob;
$job->handle();
// Assert CheckAndStartSentinelJob was NOT dispatched
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
});
it('respects server timezone when scheduling sentinel checks', function () {
// Mock InstanceSettings
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
// Set test time to top of hour in America/New_York (which is 17:00 UTC)
Carbon::setTestNow('2025-01-15 17:00:00'); // 12:00 PM EST (top of hour in EST)
// Create a mock server with sentinel enabled and America/New_York timezone
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
$server->id = 3;
$server->name = 'test-server-est';
$server->ip = '192.168.1.102';
$server->sentinel_updated_at = Carbon::now();
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'America/New_York']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
// Mock the Server query
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
// Execute the job
$job = new ServerManagerJob;
$job->handle();
// Assert CheckAndStartSentinelJob was dispatched (should run at top of hour in server's timezone)
Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
// Sentinel is not enabled so SSH connection check must run
Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('does not dispatch sentinel check when not at top of hour', function () {
// Mock InstanceSettings
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
// Set test time to middle of the hour (not top of hour)
Carbon::setTestNow('2025-01-15 12:30:00');
// Create a mock server with sentinel enabled
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
$server->id = 4;
$server->name = 'test-server-mid-hour';
$server->ip = '192.168.1.103';
$server->sentinel_updated_at = Carbon::now();
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
// Mock the Server query
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
// Execute the job
$job = new ServerManagerJob;
$job->handle();
// Assert CheckAndStartSentinelJob was NOT dispatched (not top of hour)
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
});