using System.Text.Json; using ItWorx.KnowledgeHub.Application.Identity; using ItWorx.KnowledgeHub.Application.Jobs; using ItWorx.KnowledgeHub.Application.Jobs.UseCases; using ItWorx.KnowledgeHub.Application.Knowledge; using ItWorx.KnowledgeHub.Application.Knowledge.UseCases; using ItWorx.KnowledgeHub.Application.Operations; using ItWorx.KnowledgeHub.Application.Operations.UseCases; using ItWorx.KnowledgeHub.Domain.Identity; using ItWorx.KnowledgeHub.Domain.Jobs; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace ItWorx.KnowledgeHub.Api.BackgroundServices; /// Executes durable jobs with bounded worker concurrency. PostgreSQL leases /// allow another process to resume work after a crash; each course page is handled as /// an independent, idempotent ETag-guarded update. public sealed partial class JobWorker( IServiceScopeFactory scopeFactory, BackgroundJobsOptions options, TimeProvider timeProvider, PlatformMetrics metrics, ILogger logger) : BackgroundService { private long lastMetricsRefreshMilliseconds; private int metricsRefreshInProgress; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { options.Validate(); if (!options.Enabled) { LogDisabled(); return; } var workers = Enumerable.Range(0, options.MaxConcurrency) .Select(_ => RunWorkerAsync(stoppingToken)); await Task.WhenAll(workers); } private async Task RunWorkerAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { using var scope = scopeFactory.CreateScope(); var repository = scope.ServiceProvider.GetRequiredService(); var leaseToken = Guid.NewGuid(); Guid? jobId; try { var now = timeProvider.GetUtcNow(); jobId = await repository.TryClaimNextAsync( leaseToken, now, now.AddSeconds(options.LeaseSeconds), stoppingToken); await RefreshMetricsAsync(repository, force: jobId is not null, stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } catch (Exception exception) { LogWorkerPollFailed(exception.GetType().Name); try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } continue; } if (jobId is null) { try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } continue; } var claimedJob = await repository.FindLeasedAsync(jobId.Value, leaseToken, stoppingToken); if (claimedJob is null) continue; using var leaseLost = new CancellationTokenSource(); using var userCancellation = new CancellationTokenSource(); using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource( stoppingToken, leaseLost.Token); using var monitorCancellation = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); var monitor = MonitorLeaseAsync( jobId.Value, leaseToken, leaseLost, userCancellation, monitorCancellation.Token); var measurement = metrics.Start("job"); try { await ExecuteLeasedJobAsync( scope.ServiceProvider, jobId.Value, leaseToken, operationCancellation.Token, userCancellation.Token, stoppingToken); var completed = await repository.FindAsync(claimedJob.WorkspaceId, jobId.Value, stoppingToken); if (completed?.Status == BackgroundJobStatus.Cancelled) measurement.Cancel(); else if (completed?.Status == BackgroundJobStatus.Succeeded) measurement.Complete(completed.Processed); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { // Leave the durable running row leased. A different worker will recover // it when the lease expires; explicit user cancellation is checked below. if (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested) { try { await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None); measurement.Cancel(); } catch (Exception exception) { LogJobStateWriteFailed(jobId.Value, exception.GetType().Name); } } } catch (OperationCanceledException) when (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested) { try { await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None); measurement.Cancel(); } catch (Exception exception) { LogJobStateWriteFailed(jobId.Value, exception.GetType().Name); } } catch (OperationCanceledException) when (leaseLost.IsCancellationRequested) { LogLeaseLost(jobId.Value); } catch (Exception exception) { var retry = await RetryTimeAsync(repository, jobId.Value, leaseToken, CancellationToken.None); try { await repository.RetryLeasedJobAsync( jobId.Value, leaseToken, timeProvider.GetUtcNow(), retry, "processing_retry", "De tijdelijke verwerking is mislukt; de taak wordt opnieuw geprobeerd.", CancellationToken.None); } catch (Exception stateException) { LogJobStateWriteFailed(jobId.Value, stateException.GetType().Name); } LogJobExecutionFailed(jobId.Value, exception.GetType().Name); } finally { measurement.Dispose(); monitorCancellation.Cancel(); try { await monitor; } catch (OperationCanceledException) { } } await RefreshMetricsAsync(repository, force: true, stoppingToken); } } private async Task ExecuteLeasedJobAsync( IServiceProvider services, Guid jobId, Guid leaseToken, CancellationToken operationToken, CancellationToken userCancellationToken, CancellationToken hostToken) { var jobs = services.GetRequiredService(); var job = await jobs.FindLeasedAsync(jobId, leaseToken, operationToken); if (job is null) return; if (!await IsActorAdministratorAsync(services, job, operationToken)) { await jobs.FailLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(), "actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken); return; } if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, operationToken)) { await jobs.CancelLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(), operationToken); return; } if (job.Type == BackgroundJobType.Backup) { using var backupCancellation = CancellationTokenSource.CreateLinkedTokenSource(operationToken, userCancellationToken); await ExecuteBackupAsync(services, job, leaseToken, backupCancellation.Token, operationToken); return; } if (job.Type == BackgroundJobType.CourseTagging) await ExecuteCourseTaggingAsync(services, job, leaseToken, operationToken); } private async Task ExecuteBackupAsync( IServiceProvider services, BackgroundJob job, Guid leaseToken, CancellationToken backupCancellationToken, CancellationToken stateCancellationToken) { var jobs = services.GetRequiredService(); var backups = services.GetRequiredService(); var summary = await backups.ExecuteAsync( new CreateBackupCommand(job.ActorId, job.CorrelationId), backupCancellationToken); var result = JsonSerializer.Serialize(new { backupId = summary.BackupId, createdAt = summary.CreatedAt, sizeBytes = summary.TotalSizeBytes, tableCount = summary.TableCount, attachmentCount = summary.AttachmentCount, schemaVersion = summary.SchemaVersion }); await jobs.RecordJobOutcomeAsync( job.Id, leaseToken, BackgroundJobItemState.Succeeded, null, null, stateCancellationToken); await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, result, timeProvider.GetUtcNow(), stateCancellationToken); } private async Task ExecuteCourseTaggingAsync( IServiceProvider services, BackgroundJob job, Guid leaseToken, CancellationToken operationToken) { using var payload = JsonDocument.Parse(job.PayloadJson); var rootPageId = payload.RootElement.GetProperty("rootPageId").GetGuid(); var tag = payload.RootElement.GetProperty("tag").GetString()!; var jobs = services.GetRequiredService(); var pages = services.GetRequiredService(); var updatePage = services.GetRequiredService(); while (true) { operationToken.ThrowIfCancellationRequested(); if (await jobs.IsCancellationRequestedAsync(job.Id, leaseToken, operationToken)) { await jobs.CancelLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(), operationToken); return; } var currentJob = await jobs.FindLeasedAsync(job.Id, leaseToken, operationToken); if (currentJob is null) return; if (currentJob.Processed >= currentJob.Total) { await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, JsonSerializer.Serialize(new { rootPageId, tag, total = currentJob.Total, processed = currentJob.Processed, succeeded = currentJob.Succeeded, skipped = currentJob.Skipped, failed = currentJob.Failed }), timeProvider.GetUtcNow(), operationToken); return; } if (!await IsActorAdministratorAsync(services, currentJob, operationToken)) { await jobs.FailLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(), "actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken); return; } var item = await jobs.FindNextPageItemAsync(job.Id, leaseToken, operationToken); if (item is null) throw new InvalidOperationException("The job snapshot is incomplete."); await jobs.MarkPageItemProcessingAsync(job.Id, leaseToken, item.Ordinal, timeProvider.GetUtcNow(), operationToken); var page = await pages.FindAsync(item.PageId, operationToken); if (page is null || page.WorkspaceId != job.WorkspaceId || page.IsTrashed) { await jobs.RecordPageItemOutcomeAsync( job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed, "page_unavailable", "Pagina is niet langer beschikbaar.", item.PageId, timeProvider.GetUtcNow(), operationToken); continue; } if (page.Tags.Any(existing => string.Equals(existing, tag, StringComparison.OrdinalIgnoreCase))) { await jobs.RecordPageItemOutcomeAsync( job.Id, leaseToken, item.Ordinal, item.InitiallyTagged ? BackgroundJobItemState.Skipped : BackgroundJobItemState.Succeeded, null, null, null, timeProvider.GetUtcNow(), operationToken); continue; } if (page.Tags.Count >= Domain.Knowledge.Page.MaxTagCount) { await jobs.RecordPageItemOutcomeAsync( job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Skipped, null, null, null, timeProvider.GetUtcNow(), operationToken); continue; } var etag = await pages.GetETagAsync(page.Id, operationToken); if (etag is null) { await jobs.RecordPageItemOutcomeAsync( job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed, "page_unavailable", "Pagina is niet langer beschikbaar.", page.Id, timeProvider.GetUtcNow(), operationToken); continue; } var tags = page.Tags.Append(tag).ToArray(); var result = await updatePage.ExecuteAsync( new UpdatePageCommand( page.Id, page.TitleValue, page.Markdown, page.DocumentType.ToString(), page.DocumentStatus.ToString(), $"Cursustag {tag} toegevoegd", tags, etag, job.ActorId, job.CorrelationId), operationToken); if (result.Status == PageOperationStatus.Success) { await jobs.RecordPageItemOutcomeAsync( job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Succeeded, null, null, null, timeProvider.GetUtcNow(), operationToken); } else { var conflict = result.Status == PageOperationStatus.ConcurrencyConflict; await jobs.RecordPageItemOutcomeAsync( job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed, conflict ? "concurrency_conflict" : "page_update_failed", conflict ? "Pagina is tijdens de taak gewijzigd; de wijziging is overgeslagen." : "De pagina kon niet worden bijgewerkt.", page.Id, timeProvider.GetUtcNow(), operationToken); } } } private static async Task IsActorAdministratorAsync( IServiceProvider services, BackgroundJob job, CancellationToken cancellationToken) { var accounts = services.GetRequiredService(); var memberships = services.GetRequiredService(); var account = await accounts.FindByIdAsync(job.ActorId, cancellationToken); if (account is not { Enabled: true }) return false; var role = await memberships.GetRoleAsync(job.WorkspaceId, job.ActorId, cancellationToken); return role == WorkspaceRole.Administrator; } private async Task MonitorLeaseAsync( Guid jobId, Guid leaseToken, CancellationTokenSource leaseLost, CancellationTokenSource userCancellation, CancellationToken cancellationToken) { var lastRenewal = timeProvider.GetUtcNow(); while (!cancellationToken.IsCancellationRequested) { try { await Task.Delay(TimeSpan.FromSeconds(1), timeProvider, cancellationToken); using var scope = scopeFactory.CreateScope(); var jobs = scope.ServiceProvider.GetRequiredService(); if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, cancellationToken)) userCancellation.Cancel(); var now = timeProvider.GetUtcNow(); if (now - lastRenewal >= TimeSpan.FromSeconds(options.LeaseRenewSeconds)) { if (!await jobs.RenewLeaseAsync(jobId, leaseToken, now.AddSeconds(options.LeaseSeconds), cancellationToken)) { leaseLost.Cancel(); return; } lastRenewal = now; } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return; } catch (Exception exception) { // A transient telemetry/lease read failure does not cancel work before // the lease can expire. The database remains authoritative on writes. LogLeaseMonitorFailed(jobId, exception.GetType().Name); } } } private async Task RefreshMetricsAsync(IBackgroundJobRepository repository, bool force, CancellationToken cancellationToken) { var nowMilliseconds = timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); if (!force && nowMilliseconds - Interlocked.Read(ref lastMetricsRefreshMilliseconds) < 10_000) return; if (Interlocked.CompareExchange(ref metricsRefreshInProgress, 1, 0) != 0) return; try { var statistics = await repository.GetQueueStatisticsAsync(cancellationToken); var oldestAge = statistics.OldestQueuedAt is { } oldest ? Math.Max(0, (timeProvider.GetUtcNow() - oldest).TotalSeconds) : (double?)null; metrics.SetQueueState(statistics.Queued, statistics.Running, statistics.Failed, oldestAge); Interlocked.Exchange(ref lastMetricsRefreshMilliseconds, nowMilliseconds); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { } catch (Exception exception) { LogJobMetricsRefreshFailed(exception.GetType().Name); } finally { Volatile.Write(ref metricsRefreshInProgress, 0); } } private async Task RetryTimeAsync( IBackgroundJobRepository repository, Guid jobId, Guid leaseToken, CancellationToken cancellationToken) { var attempt = (await repository.FindLeasedAsync(jobId, leaseToken, cancellationToken))?.AttemptCount ?? 1; var seconds = attempt switch { 1 => 2, 2 => 10, _ => 30 }; return timeProvider.GetUtcNow().AddSeconds(seconds); } [LoggerMessage(Level = LogLevel.Warning, Message = "Background job poll failed ({ErrorType}).")] private partial void LogWorkerPollFailed(string errorType); [LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} failed ({ErrorType}).")] private partial void LogJobExecutionFailed(Guid jobId, string errorType); [LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lost its processing lease.")] private partial void LogLeaseLost(Guid jobId); [LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} state write failed ({ErrorType}).")] private partial void LogJobStateWriteFailed(Guid jobId, string errorType); [LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lease monitoring failed ({ErrorType}).")] private partial void LogLeaseMonitorFailed(Guid jobId, string errorType); [LoggerMessage(Level = LogLevel.Warning, Message = "Background job metrics refresh failed ({ErrorType}).")] private partial void LogJobMetricsRefreshFailed(string errorType); [LoggerMessage(Level = LogLevel.Information, Message = "Background job worker is disabled by configuration.")] private partial void LogDisabled(); }