Publish curated Knowledge Hub source from 2ed833f290b439a7bc91e5fd19b042c269e3bee9
This commit is contained in:
Public Source Publisher committed
1 parent
10f2edcd29
commit
59f15b2d08
218 files changed
+23995
-1849
No files matched your search
@@ -0,0 +1,461 @@
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using ItWorx.KnowledgeHub.Application.Operations.UseCases;
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.BackgroundServices;
|
||||
|
||||
/// <summary>Executes durable jobs with bounded worker concurrency. PostgreSQL leases
|
||||
/// allow another process to resume work after a crash; each course page is handled as
|
||||
/// an independent, idempotent ETag-guarded update.</summary>
|
||||
public sealed partial class JobWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
BackgroundJobsOptions options,
|
||||
TimeProvider timeProvider,
|
||||
PlatformMetrics metrics,
|
||||
ILogger<JobWorker> logger) : BackgroundService
|
||||
{
|
||||
private long lastMetricsRefreshMilliseconds;
|
||||
private int metricsRefreshInProgress;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
options.Validate();
|
||||
if (!options.Enabled)
|
||||
{
|
||||
LogDisabled();
|
||||
return;
|
||||
}
|
||||
var workers = Enumerable.Range(0, options.MaxConcurrency)
|
||||
.Select(_ => RunWorkerAsync(stoppingToken));
|
||||
await Task.WhenAll(workers);
|
||||
}
|
||||
|
||||
private async Task RunWorkerAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
var repository = scope.ServiceProvider.GetRequiredService<IBackgroundJobRepository>();
|
||||
var leaseToken = Guid.NewGuid();
|
||||
Guid? jobId;
|
||||
try
|
||||
{
|
||||
var now = timeProvider.GetUtcNow();
|
||||
jobId = await repository.TryClaimNextAsync(
|
||||
leaseToken, now, now.AddSeconds(options.LeaseSeconds), stoppingToken);
|
||||
await RefreshMetricsAsync(repository, force: jobId is not null, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogWorkerPollFailed(exception.GetType().Name);
|
||||
try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
if (jobId is null)
|
||||
{
|
||||
try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
var claimedJob = await repository.FindLeasedAsync(jobId.Value, leaseToken, stoppingToken);
|
||||
if (claimedJob is null)
|
||||
continue;
|
||||
|
||||
using var leaseLost = new CancellationTokenSource();
|
||||
using var userCancellation = new CancellationTokenSource();
|
||||
using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
stoppingToken, leaseLost.Token);
|
||||
using var monitorCancellation = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
|
||||
var monitor = MonitorLeaseAsync(
|
||||
jobId.Value, leaseToken, leaseLost, userCancellation, monitorCancellation.Token);
|
||||
var measurement = metrics.Start("job");
|
||||
|
||||
try
|
||||
{
|
||||
await ExecuteLeasedJobAsync(
|
||||
scope.ServiceProvider, jobId.Value, leaseToken, operationCancellation.Token,
|
||||
userCancellation.Token, stoppingToken);
|
||||
var completed = await repository.FindAsync(claimedJob.WorkspaceId, jobId.Value, stoppingToken);
|
||||
if (completed?.Status == BackgroundJobStatus.Cancelled)
|
||||
measurement.Cancel();
|
||||
else if (completed?.Status == BackgroundJobStatus.Succeeded)
|
||||
measurement.Complete(completed.Processed);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// Leave the durable running row leased. A different worker will recover
|
||||
// it when the lease expires; explicit user cancellation is checked below.
|
||||
if (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None);
|
||||
measurement.Cancel();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogJobStateWriteFailed(jobId.Value, exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None);
|
||||
measurement.Cancel();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogJobStateWriteFailed(jobId.Value, exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (leaseLost.IsCancellationRequested)
|
||||
{
|
||||
LogLeaseLost(jobId.Value);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
var retry = await RetryTimeAsync(repository, jobId.Value, leaseToken, CancellationToken.None);
|
||||
try
|
||||
{
|
||||
await repository.RetryLeasedJobAsync(
|
||||
jobId.Value, leaseToken, timeProvider.GetUtcNow(), retry,
|
||||
"processing_retry", "De tijdelijke verwerking is mislukt; de taak wordt opnieuw geprobeerd.",
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception stateException)
|
||||
{
|
||||
LogJobStateWriteFailed(jobId.Value, stateException.GetType().Name);
|
||||
}
|
||||
LogJobExecutionFailed(jobId.Value, exception.GetType().Name);
|
||||
}
|
||||
finally
|
||||
{
|
||||
measurement.Dispose();
|
||||
monitorCancellation.Cancel();
|
||||
try { await monitor; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
|
||||
await RefreshMetricsAsync(repository, force: true, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ExecuteLeasedJobAsync(
|
||||
IServiceProvider services,
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
CancellationToken operationToken,
|
||||
CancellationToken userCancellationToken,
|
||||
CancellationToken hostToken)
|
||||
{
|
||||
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
||||
var job = await jobs.FindLeasedAsync(jobId, leaseToken, operationToken);
|
||||
if (job is null)
|
||||
return;
|
||||
|
||||
if (!await IsActorAdministratorAsync(services, job, operationToken))
|
||||
{
|
||||
await jobs.FailLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(),
|
||||
"actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, operationToken))
|
||||
{
|
||||
await jobs.CancelLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(), operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.Type == BackgroundJobType.Backup)
|
||||
{
|
||||
using var backupCancellation = CancellationTokenSource.CreateLinkedTokenSource(operationToken, userCancellationToken);
|
||||
await ExecuteBackupAsync(services, job, leaseToken, backupCancellation.Token, operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.Type == BackgroundJobType.CourseTagging)
|
||||
await ExecuteCourseTaggingAsync(services, job, leaseToken, operationToken);
|
||||
}
|
||||
|
||||
private async Task ExecuteBackupAsync(
|
||||
IServiceProvider services,
|
||||
BackgroundJob job,
|
||||
Guid leaseToken,
|
||||
CancellationToken backupCancellationToken,
|
||||
CancellationToken stateCancellationToken)
|
||||
{
|
||||
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
||||
var backups = services.GetRequiredService<CreateBackupUseCase>();
|
||||
var summary = await backups.ExecuteAsync(
|
||||
new CreateBackupCommand(job.ActorId, job.CorrelationId), backupCancellationToken);
|
||||
var result = JsonSerializer.Serialize(new
|
||||
{
|
||||
backupId = summary.BackupId,
|
||||
createdAt = summary.CreatedAt,
|
||||
sizeBytes = summary.TotalSizeBytes,
|
||||
tableCount = summary.TableCount,
|
||||
attachmentCount = summary.AttachmentCount,
|
||||
schemaVersion = summary.SchemaVersion
|
||||
});
|
||||
await jobs.RecordJobOutcomeAsync(
|
||||
job.Id, leaseToken, BackgroundJobItemState.Succeeded, null, null,
|
||||
stateCancellationToken);
|
||||
await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, result, timeProvider.GetUtcNow(), stateCancellationToken);
|
||||
}
|
||||
|
||||
private async Task ExecuteCourseTaggingAsync(
|
||||
IServiceProvider services,
|
||||
BackgroundJob job,
|
||||
Guid leaseToken,
|
||||
CancellationToken operationToken)
|
||||
{
|
||||
using var payload = JsonDocument.Parse(job.PayloadJson);
|
||||
var rootPageId = payload.RootElement.GetProperty("rootPageId").GetGuid();
|
||||
var tag = payload.RootElement.GetProperty("tag").GetString()!;
|
||||
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
||||
var pages = services.GetRequiredService<IPageRepository>();
|
||||
var updatePage = services.GetRequiredService<UpdatePageUseCase>();
|
||||
|
||||
while (true)
|
||||
{
|
||||
operationToken.ThrowIfCancellationRequested();
|
||||
if (await jobs.IsCancellationRequestedAsync(job.Id, leaseToken, operationToken))
|
||||
{
|
||||
await jobs.CancelLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(), operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var currentJob = await jobs.FindLeasedAsync(job.Id, leaseToken, operationToken);
|
||||
if (currentJob is null)
|
||||
return;
|
||||
if (currentJob.Processed >= currentJob.Total)
|
||||
{
|
||||
await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, JsonSerializer.Serialize(new
|
||||
{
|
||||
rootPageId,
|
||||
tag,
|
||||
total = currentJob.Total,
|
||||
processed = currentJob.Processed,
|
||||
succeeded = currentJob.Succeeded,
|
||||
skipped = currentJob.Skipped,
|
||||
failed = currentJob.Failed
|
||||
}), timeProvider.GetUtcNow(), operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!await IsActorAdministratorAsync(services, currentJob, operationToken))
|
||||
{
|
||||
await jobs.FailLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(),
|
||||
"actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var item = await jobs.FindNextPageItemAsync(job.Id, leaseToken, operationToken);
|
||||
if (item is null)
|
||||
throw new InvalidOperationException("The job snapshot is incomplete.");
|
||||
|
||||
await jobs.MarkPageItemProcessingAsync(job.Id, leaseToken, item.Ordinal, timeProvider.GetUtcNow(), operationToken);
|
||||
var page = await pages.FindAsync(item.PageId, operationToken);
|
||||
if (page is null || page.WorkspaceId != job.WorkspaceId || page.IsTrashed)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
||||
"page_unavailable", "Pagina is niet langer beschikbaar.", item.PageId,
|
||||
timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (page.Tags.Any(existing => string.Equals(existing, tag, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal,
|
||||
item.InitiallyTagged ? BackgroundJobItemState.Skipped : BackgroundJobItemState.Succeeded,
|
||||
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (page.Tags.Count >= Domain.Knowledge.Page.MaxTagCount)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Skipped,
|
||||
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var etag = await pages.GetETagAsync(page.Id, operationToken);
|
||||
if (etag is null)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
||||
"page_unavailable", "Pagina is niet langer beschikbaar.", page.Id,
|
||||
timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var tags = page.Tags.Append(tag).ToArray();
|
||||
var result = await updatePage.ExecuteAsync(
|
||||
new UpdatePageCommand(
|
||||
page.Id,
|
||||
page.TitleValue,
|
||||
page.Markdown,
|
||||
page.DocumentType.ToString(),
|
||||
page.DocumentStatus.ToString(),
|
||||
$"Cursustag {tag} toegevoegd",
|
||||
tags,
|
||||
etag,
|
||||
job.ActorId,
|
||||
job.CorrelationId),
|
||||
operationToken);
|
||||
|
||||
if (result.Status == PageOperationStatus.Success)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Succeeded,
|
||||
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
var conflict = result.Status == PageOperationStatus.ConcurrencyConflict;
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
||||
conflict ? "concurrency_conflict" : "page_update_failed",
|
||||
conflict ? "Pagina is tijdens de taak gewijzigd; de wijziging is overgeslagen."
|
||||
: "De pagina kon niet worden bijgewerkt.",
|
||||
page.Id, timeProvider.GetUtcNow(), operationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<bool> IsActorAdministratorAsync(
|
||||
IServiceProvider services,
|
||||
BackgroundJob job,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var accounts = services.GetRequiredService<IUserAccountManager>();
|
||||
var memberships = services.GetRequiredService<IWorkspaceMembershipRepository>();
|
||||
var account = await accounts.FindByIdAsync(job.ActorId, cancellationToken);
|
||||
if (account is not { Enabled: true })
|
||||
return false;
|
||||
var role = await memberships.GetRoleAsync(job.WorkspaceId, job.ActorId, cancellationToken);
|
||||
return role == WorkspaceRole.Administrator;
|
||||
}
|
||||
|
||||
private async Task MonitorLeaseAsync(
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
CancellationTokenSource leaseLost,
|
||||
CancellationTokenSource userCancellation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var lastRenewal = timeProvider.GetUtcNow();
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), timeProvider, cancellationToken);
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
var jobs = scope.ServiceProvider.GetRequiredService<IBackgroundJobRepository>();
|
||||
if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, cancellationToken))
|
||||
userCancellation.Cancel();
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
if (now - lastRenewal >= TimeSpan.FromSeconds(options.LeaseRenewSeconds))
|
||||
{
|
||||
if (!await jobs.RenewLeaseAsync(jobId, leaseToken, now.AddSeconds(options.LeaseSeconds), cancellationToken))
|
||||
{
|
||||
leaseLost.Cancel();
|
||||
return;
|
||||
}
|
||||
lastRenewal = now;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// A transient telemetry/lease read failure does not cancel work before
|
||||
// the lease can expire. The database remains authoritative on writes.
|
||||
LogLeaseMonitorFailed(jobId, exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RefreshMetricsAsync(IBackgroundJobRepository repository, bool force, CancellationToken cancellationToken)
|
||||
{
|
||||
var nowMilliseconds = timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
if (!force && nowMilliseconds - Interlocked.Read(ref lastMetricsRefreshMilliseconds) < 10_000)
|
||||
return;
|
||||
if (Interlocked.CompareExchange(ref metricsRefreshInProgress, 1, 0) != 0)
|
||||
return;
|
||||
try
|
||||
{
|
||||
var statistics = await repository.GetQueueStatisticsAsync(cancellationToken);
|
||||
var oldestAge = statistics.OldestQueuedAt is { } oldest
|
||||
? Math.Max(0, (timeProvider.GetUtcNow() - oldest).TotalSeconds)
|
||||
: (double?)null;
|
||||
metrics.SetQueueState(statistics.Queued, statistics.Running, statistics.Failed, oldestAge);
|
||||
Interlocked.Exchange(ref lastMetricsRefreshMilliseconds, nowMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogJobMetricsRefreshFailed(exception.GetType().Name);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Volatile.Write(ref metricsRefreshInProgress, 0);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DateTimeOffset> RetryTimeAsync(
|
||||
IBackgroundJobRepository repository, Guid jobId, Guid leaseToken, CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = (await repository.FindLeasedAsync(jobId, leaseToken, cancellationToken))?.AttemptCount ?? 1;
|
||||
var seconds = attempt switch { 1 => 2, 2 => 10, _ => 30 };
|
||||
return timeProvider.GetUtcNow().AddSeconds(seconds);
|
||||
}
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job poll failed ({ErrorType}).")]
|
||||
private partial void LogWorkerPollFailed(string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} failed ({ErrorType}).")]
|
||||
private partial void LogJobExecutionFailed(Guid jobId, string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lost its processing lease.")]
|
||||
private partial void LogLeaseLost(Guid jobId);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} state write failed ({ErrorType}).")]
|
||||
private partial void LogJobStateWriteFailed(Guid jobId, string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lease monitoring failed ({ErrorType}).")]
|
||||
private partial void LogLeaseMonitorFailed(Guid jobId, string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job metrics refresh failed ({ErrorType}).")]
|
||||
private partial void LogJobMetricsRefreshFailed(string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "Background job worker is disabled by configuration.")]
|
||||
private partial void LogDisabled();
|
||||
}
|
||||
Reference in new issue
Block a user