462 lines
20 KiB
C#
462 lines
20 KiB
C#
using System.Text.Json;
|
|
using ItWorx.KnowledgeHub.Application.Identity;
|
|
using ItWorx.KnowledgeHub.Application.Jobs;
|
|
using ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
|
using ItWorx.KnowledgeHub.Application.Knowledge;
|
|
using ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
|
using ItWorx.KnowledgeHub.Application.Operations;
|
|
using ItWorx.KnowledgeHub.Application.Operations.UseCases;
|
|
using ItWorx.KnowledgeHub.Domain.Identity;
|
|
using ItWorx.KnowledgeHub.Domain.Jobs;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace ItWorx.KnowledgeHub.Api.BackgroundServices;
|
|
|
|
/// <summary>Executes durable jobs with bounded worker concurrency. PostgreSQL leases
|
|
/// allow another process to resume work after a crash; each course page is handled as
|
|
/// an independent, idempotent ETag-guarded update.</summary>
|
|
public sealed partial class JobWorker(
|
|
IServiceScopeFactory scopeFactory,
|
|
BackgroundJobsOptions options,
|
|
TimeProvider timeProvider,
|
|
PlatformMetrics metrics,
|
|
ILogger<JobWorker> logger) : BackgroundService
|
|
{
|
|
private long lastMetricsRefreshMilliseconds;
|
|
private int metricsRefreshInProgress;
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
options.Validate();
|
|
if (!options.Enabled)
|
|
{
|
|
LogDisabled();
|
|
return;
|
|
}
|
|
var workers = Enumerable.Range(0, options.MaxConcurrency)
|
|
.Select(_ => RunWorkerAsync(stoppingToken));
|
|
await Task.WhenAll(workers);
|
|
}
|
|
|
|
private async Task RunWorkerAsync(CancellationToken stoppingToken)
|
|
{
|
|
while (!stoppingToken.IsCancellationRequested)
|
|
{
|
|
using var scope = scopeFactory.CreateScope();
|
|
var repository = scope.ServiceProvider.GetRequiredService<IBackgroundJobRepository>();
|
|
var leaseToken = Guid.NewGuid();
|
|
Guid? jobId;
|
|
try
|
|
{
|
|
var now = timeProvider.GetUtcNow();
|
|
jobId = await repository.TryClaimNextAsync(
|
|
leaseToken, now, now.AddSeconds(options.LeaseSeconds), stoppingToken);
|
|
await RefreshMetricsAsync(repository, force: jobId is not null, stoppingToken);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
LogWorkerPollFailed(exception.GetType().Name);
|
|
try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); }
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
|
continue;
|
|
}
|
|
|
|
if (jobId is null)
|
|
{
|
|
try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); }
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
|
continue;
|
|
}
|
|
|
|
var claimedJob = await repository.FindLeasedAsync(jobId.Value, leaseToken, stoppingToken);
|
|
if (claimedJob is null)
|
|
continue;
|
|
|
|
using var leaseLost = new CancellationTokenSource();
|
|
using var userCancellation = new CancellationTokenSource();
|
|
using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource(
|
|
stoppingToken, leaseLost.Token);
|
|
using var monitorCancellation = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
|
|
var monitor = MonitorLeaseAsync(
|
|
jobId.Value, leaseToken, leaseLost, userCancellation, monitorCancellation.Token);
|
|
var measurement = metrics.Start("job");
|
|
|
|
try
|
|
{
|
|
await ExecuteLeasedJobAsync(
|
|
scope.ServiceProvider, jobId.Value, leaseToken, operationCancellation.Token,
|
|
userCancellation.Token, stoppingToken);
|
|
var completed = await repository.FindAsync(claimedJob.WorkspaceId, jobId.Value, stoppingToken);
|
|
if (completed?.Status == BackgroundJobStatus.Cancelled)
|
|
measurement.Cancel();
|
|
else if (completed?.Status == BackgroundJobStatus.Succeeded)
|
|
measurement.Complete(completed.Processed);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
// Leave the durable running row leased. A different worker will recover
|
|
// it when the lease expires; explicit user cancellation is checked below.
|
|
if (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None);
|
|
measurement.Cancel();
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
LogJobStateWriteFailed(jobId.Value, exception.GetType().Name);
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None);
|
|
measurement.Cancel();
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
LogJobStateWriteFailed(jobId.Value, exception.GetType().Name);
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (leaseLost.IsCancellationRequested)
|
|
{
|
|
LogLeaseLost(jobId.Value);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
var retry = await RetryTimeAsync(repository, jobId.Value, leaseToken, CancellationToken.None);
|
|
try
|
|
{
|
|
await repository.RetryLeasedJobAsync(
|
|
jobId.Value, leaseToken, timeProvider.GetUtcNow(), retry,
|
|
"processing_retry", "De tijdelijke verwerking is mislukt; de taak wordt opnieuw geprobeerd.",
|
|
CancellationToken.None);
|
|
}
|
|
catch (Exception stateException)
|
|
{
|
|
LogJobStateWriteFailed(jobId.Value, stateException.GetType().Name);
|
|
}
|
|
LogJobExecutionFailed(jobId.Value, exception.GetType().Name);
|
|
}
|
|
finally
|
|
{
|
|
measurement.Dispose();
|
|
monitorCancellation.Cancel();
|
|
try { await monitor; }
|
|
catch (OperationCanceledException) { }
|
|
}
|
|
|
|
await RefreshMetricsAsync(repository, force: true, stoppingToken);
|
|
}
|
|
}
|
|
|
|
private async Task ExecuteLeasedJobAsync(
|
|
IServiceProvider services,
|
|
Guid jobId,
|
|
Guid leaseToken,
|
|
CancellationToken operationToken,
|
|
CancellationToken userCancellationToken,
|
|
CancellationToken hostToken)
|
|
{
|
|
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
|
var job = await jobs.FindLeasedAsync(jobId, leaseToken, operationToken);
|
|
if (job is null)
|
|
return;
|
|
|
|
if (!await IsActorAdministratorAsync(services, job, operationToken))
|
|
{
|
|
await jobs.FailLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(),
|
|
"actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken);
|
|
return;
|
|
}
|
|
|
|
if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, operationToken))
|
|
{
|
|
await jobs.CancelLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(), operationToken);
|
|
return;
|
|
}
|
|
|
|
if (job.Type == BackgroundJobType.Backup)
|
|
{
|
|
using var backupCancellation = CancellationTokenSource.CreateLinkedTokenSource(operationToken, userCancellationToken);
|
|
await ExecuteBackupAsync(services, job, leaseToken, backupCancellation.Token, operationToken);
|
|
return;
|
|
}
|
|
|
|
if (job.Type == BackgroundJobType.CourseTagging)
|
|
await ExecuteCourseTaggingAsync(services, job, leaseToken, operationToken);
|
|
}
|
|
|
|
private async Task ExecuteBackupAsync(
|
|
IServiceProvider services,
|
|
BackgroundJob job,
|
|
Guid leaseToken,
|
|
CancellationToken backupCancellationToken,
|
|
CancellationToken stateCancellationToken)
|
|
{
|
|
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
|
var backups = services.GetRequiredService<CreateBackupUseCase>();
|
|
var summary = await backups.ExecuteAsync(
|
|
new CreateBackupCommand(job.ActorId, job.CorrelationId), backupCancellationToken);
|
|
var result = JsonSerializer.Serialize(new
|
|
{
|
|
backupId = summary.BackupId,
|
|
createdAt = summary.CreatedAt,
|
|
sizeBytes = summary.TotalSizeBytes,
|
|
tableCount = summary.TableCount,
|
|
attachmentCount = summary.AttachmentCount,
|
|
schemaVersion = summary.SchemaVersion
|
|
});
|
|
await jobs.RecordJobOutcomeAsync(
|
|
job.Id, leaseToken, BackgroundJobItemState.Succeeded, null, null,
|
|
stateCancellationToken);
|
|
await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, result, timeProvider.GetUtcNow(), stateCancellationToken);
|
|
}
|
|
|
|
private async Task ExecuteCourseTaggingAsync(
|
|
IServiceProvider services,
|
|
BackgroundJob job,
|
|
Guid leaseToken,
|
|
CancellationToken operationToken)
|
|
{
|
|
using var payload = JsonDocument.Parse(job.PayloadJson);
|
|
var rootPageId = payload.RootElement.GetProperty("rootPageId").GetGuid();
|
|
var tag = payload.RootElement.GetProperty("tag").GetString()!;
|
|
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
|
var pages = services.GetRequiredService<IPageRepository>();
|
|
var updatePage = services.GetRequiredService<UpdatePageUseCase>();
|
|
|
|
while (true)
|
|
{
|
|
operationToken.ThrowIfCancellationRequested();
|
|
if (await jobs.IsCancellationRequestedAsync(job.Id, leaseToken, operationToken))
|
|
{
|
|
await jobs.CancelLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(), operationToken);
|
|
return;
|
|
}
|
|
|
|
var currentJob = await jobs.FindLeasedAsync(job.Id, leaseToken, operationToken);
|
|
if (currentJob is null)
|
|
return;
|
|
if (currentJob.Processed >= currentJob.Total)
|
|
{
|
|
await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, JsonSerializer.Serialize(new
|
|
{
|
|
rootPageId,
|
|
tag,
|
|
total = currentJob.Total,
|
|
processed = currentJob.Processed,
|
|
succeeded = currentJob.Succeeded,
|
|
skipped = currentJob.Skipped,
|
|
failed = currentJob.Failed
|
|
}), timeProvider.GetUtcNow(), operationToken);
|
|
return;
|
|
}
|
|
|
|
if (!await IsActorAdministratorAsync(services, currentJob, operationToken))
|
|
{
|
|
await jobs.FailLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(),
|
|
"actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken);
|
|
return;
|
|
}
|
|
|
|
var item = await jobs.FindNextPageItemAsync(job.Id, leaseToken, operationToken);
|
|
if (item is null)
|
|
throw new InvalidOperationException("The job snapshot is incomplete.");
|
|
|
|
await jobs.MarkPageItemProcessingAsync(job.Id, leaseToken, item.Ordinal, timeProvider.GetUtcNow(), operationToken);
|
|
var page = await pages.FindAsync(item.PageId, operationToken);
|
|
if (page is null || page.WorkspaceId != job.WorkspaceId || page.IsTrashed)
|
|
{
|
|
await jobs.RecordPageItemOutcomeAsync(
|
|
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
|
"page_unavailable", "Pagina is niet langer beschikbaar.", item.PageId,
|
|
timeProvider.GetUtcNow(), operationToken);
|
|
continue;
|
|
}
|
|
|
|
if (page.Tags.Any(existing => string.Equals(existing, tag, StringComparison.OrdinalIgnoreCase)))
|
|
{
|
|
await jobs.RecordPageItemOutcomeAsync(
|
|
job.Id, leaseToken, item.Ordinal,
|
|
item.InitiallyTagged ? BackgroundJobItemState.Skipped : BackgroundJobItemState.Succeeded,
|
|
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
|
continue;
|
|
}
|
|
|
|
if (page.Tags.Count >= Domain.Knowledge.Page.MaxTagCount)
|
|
{
|
|
await jobs.RecordPageItemOutcomeAsync(
|
|
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Skipped,
|
|
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
|
continue;
|
|
}
|
|
|
|
var etag = await pages.GetETagAsync(page.Id, operationToken);
|
|
if (etag is null)
|
|
{
|
|
await jobs.RecordPageItemOutcomeAsync(
|
|
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
|
"page_unavailable", "Pagina is niet langer beschikbaar.", page.Id,
|
|
timeProvider.GetUtcNow(), operationToken);
|
|
continue;
|
|
}
|
|
|
|
var tags = page.Tags.Append(tag).ToArray();
|
|
var result = await updatePage.ExecuteAsync(
|
|
new UpdatePageCommand(
|
|
page.Id,
|
|
page.TitleValue,
|
|
page.Markdown,
|
|
page.DocumentType.ToString(),
|
|
page.DocumentStatus.ToString(),
|
|
$"Cursustag {tag} toegevoegd",
|
|
tags,
|
|
etag,
|
|
job.ActorId,
|
|
job.CorrelationId),
|
|
operationToken);
|
|
|
|
if (result.Status == PageOperationStatus.Success)
|
|
{
|
|
await jobs.RecordPageItemOutcomeAsync(
|
|
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Succeeded,
|
|
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
|
}
|
|
else
|
|
{
|
|
var conflict = result.Status == PageOperationStatus.ConcurrencyConflict;
|
|
await jobs.RecordPageItemOutcomeAsync(
|
|
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
|
conflict ? "concurrency_conflict" : "page_update_failed",
|
|
conflict ? "Pagina is tijdens de taak gewijzigd; de wijziging is overgeslagen."
|
|
: "De pagina kon niet worden bijgewerkt.",
|
|
page.Id, timeProvider.GetUtcNow(), operationToken);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static async Task<bool> IsActorAdministratorAsync(
|
|
IServiceProvider services,
|
|
BackgroundJob job,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var accounts = services.GetRequiredService<IUserAccountManager>();
|
|
var memberships = services.GetRequiredService<IWorkspaceMembershipRepository>();
|
|
var account = await accounts.FindByIdAsync(job.ActorId, cancellationToken);
|
|
if (account is not { Enabled: true })
|
|
return false;
|
|
var role = await memberships.GetRoleAsync(job.WorkspaceId, job.ActorId, cancellationToken);
|
|
return role == WorkspaceRole.Administrator;
|
|
}
|
|
|
|
private async Task MonitorLeaseAsync(
|
|
Guid jobId,
|
|
Guid leaseToken,
|
|
CancellationTokenSource leaseLost,
|
|
CancellationTokenSource userCancellation,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var lastRenewal = timeProvider.GetUtcNow();
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(TimeSpan.FromSeconds(1), timeProvider, cancellationToken);
|
|
using var scope = scopeFactory.CreateScope();
|
|
var jobs = scope.ServiceProvider.GetRequiredService<IBackgroundJobRepository>();
|
|
if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, cancellationToken))
|
|
userCancellation.Cancel();
|
|
|
|
var now = timeProvider.GetUtcNow();
|
|
if (now - lastRenewal >= TimeSpan.FromSeconds(options.LeaseRenewSeconds))
|
|
{
|
|
if (!await jobs.RenewLeaseAsync(jobId, leaseToken, now.AddSeconds(options.LeaseSeconds), cancellationToken))
|
|
{
|
|
leaseLost.Cancel();
|
|
return;
|
|
}
|
|
lastRenewal = now;
|
|
}
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
return;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
// A transient telemetry/lease read failure does not cancel work before
|
|
// the lease can expire. The database remains authoritative on writes.
|
|
LogLeaseMonitorFailed(jobId, exception.GetType().Name);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task RefreshMetricsAsync(IBackgroundJobRepository repository, bool force, CancellationToken cancellationToken)
|
|
{
|
|
var nowMilliseconds = timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
|
|
if (!force && nowMilliseconds - Interlocked.Read(ref lastMetricsRefreshMilliseconds) < 10_000)
|
|
return;
|
|
if (Interlocked.CompareExchange(ref metricsRefreshInProgress, 1, 0) != 0)
|
|
return;
|
|
try
|
|
{
|
|
var statistics = await repository.GetQueueStatisticsAsync(cancellationToken);
|
|
var oldestAge = statistics.OldestQueuedAt is { } oldest
|
|
? Math.Max(0, (timeProvider.GetUtcNow() - oldest).TotalSeconds)
|
|
: (double?)null;
|
|
metrics.SetQueueState(statistics.Queued, statistics.Running, statistics.Failed, oldestAge);
|
|
Interlocked.Exchange(ref lastMetricsRefreshMilliseconds, nowMilliseconds);
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
LogJobMetricsRefreshFailed(exception.GetType().Name);
|
|
}
|
|
finally
|
|
{
|
|
Volatile.Write(ref metricsRefreshInProgress, 0);
|
|
}
|
|
}
|
|
|
|
private async Task<DateTimeOffset> RetryTimeAsync(
|
|
IBackgroundJobRepository repository, Guid jobId, Guid leaseToken, CancellationToken cancellationToken)
|
|
{
|
|
var attempt = (await repository.FindLeasedAsync(jobId, leaseToken, cancellationToken))?.AttemptCount ?? 1;
|
|
var seconds = attempt switch { 1 => 2, 2 => 10, _ => 30 };
|
|
return timeProvider.GetUtcNow().AddSeconds(seconds);
|
|
}
|
|
|
|
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job poll failed ({ErrorType}).")]
|
|
private partial void LogWorkerPollFailed(string errorType);
|
|
|
|
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} failed ({ErrorType}).")]
|
|
private partial void LogJobExecutionFailed(Guid jobId, string errorType);
|
|
|
|
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lost its processing lease.")]
|
|
private partial void LogLeaseLost(Guid jobId);
|
|
|
|
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} state write failed ({ErrorType}).")]
|
|
private partial void LogJobStateWriteFailed(Guid jobId, string errorType);
|
|
|
|
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lease monitoring failed ({ErrorType}).")]
|
|
private partial void LogLeaseMonitorFailed(Guid jobId, string errorType);
|
|
|
|
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job metrics refresh failed ({ErrorType}).")]
|
|
private partial void LogJobMetricsRefreshFailed(string errorType);
|
|
|
|
[LoggerMessage(Level = LogLevel.Information, Message = "Background job worker is disabled by configuration.")]
|
|
private partial void LogDisabled();
|
|
}
|