Initial curated itworx-knowledge-hub source from 3b1751d0d9dee551aa6994eee0093704dc334817
This commit is contained in:
Public Source Publisher committed
commit
ad8ad252a9
825 files changed
+124216
No files matched your search
+184
@@ -0,0 +1,184 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>These metric contract tests are Docker-free. They exercise the production
|
||||
/// metric surface directly, with synthetic dimensions only.</summary>
|
||||
public sealed class IngestionMetricContractTests
|
||||
{
|
||||
[Fact]
|
||||
public void Pilot_metrics_emit_only_bounded_labels_and_normalize_untrusted_dimensions()
|
||||
{
|
||||
using var meter = new Meter("ingestion-observability-test");
|
||||
var measurements = new List<ObservedMeasurement>();
|
||||
using var listener = new MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, currentListener) =>
|
||||
{
|
||||
if (instrument.Meter == meter)
|
||||
currentListener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
||||
measurements.Add(Observe(instrument.Name, value, tags)));
|
||||
listener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
||||
measurements.Add(Observe(instrument.Name, value, tags)));
|
||||
listener.Start();
|
||||
|
||||
var metrics = new IngestionMetrics(meter);
|
||||
metrics.RecordProposal("success", "none");
|
||||
metrics.RecordStageDuration("source_staging", "success", 1.25);
|
||||
metrics.RecordReviewDuration("success", 2.5);
|
||||
metrics.RecordPublication("success", "none");
|
||||
|
||||
const string sensitiveBody = "do not log me";
|
||||
const string sensitiveFileName = "secret-plan.pdf";
|
||||
const string sensitiveSourceUrl = "https://private.example/source?token=secret";
|
||||
metrics.RecordProposal(sensitiveBody, sensitiveSourceUrl);
|
||||
metrics.RecordStageDuration(sensitiveFileName, sensitiveBody, 3.75);
|
||||
metrics.RecordReviewDuration(sensitiveBody, 4.5);
|
||||
metrics.RecordPublication(sensitiveBody, sensitiveSourceUrl);
|
||||
|
||||
var expected = new[]
|
||||
{
|
||||
("ingestion_proposals_total", "outcome=other|problem_type=other|stage=proposal"),
|
||||
("ingestion_proposals_total", "outcome=success|problem_type=none|stage=proposal"),
|
||||
("ingestion_stage_duration_seconds", "outcome=other|problem_type=none|stage=other"),
|
||||
("ingestion_stage_duration_seconds", "outcome=success|problem_type=none|stage=source_staging"),
|
||||
("ingestion_review_duration_seconds", "outcome=other|problem_type=none|stage=review"),
|
||||
("ingestion_review_duration_seconds", "outcome=success|problem_type=none|stage=review"),
|
||||
("ingestion_publications_total", "outcome=other|problem_type=other|stage=publication"),
|
||||
("ingestion_publications_total", "outcome=success|problem_type=none|stage=publication")
|
||||
}
|
||||
.OrderBy(measurement => measurement.Item1, StringComparer.Ordinal)
|
||||
.ThenBy(measurement => measurement.Item2, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var actual = measurements
|
||||
.Select(measurement => (measurement.Name, Tags: SerializeTags(measurement.Tags)))
|
||||
.OrderBy(measurement => measurement.Name, StringComparer.Ordinal)
|
||||
.ThenBy(measurement => measurement.Tags, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
Assert.Equal(expected, actual);
|
||||
|
||||
var allowedLabelKeys = new HashSet<string>(["stage", "outcome", "problem_type"], StringComparer.Ordinal);
|
||||
Assert.All(measurements, measurement => Assert.Subset(allowedLabelKeys, measurement.Tags.Keys.ToHashSet(StringComparer.Ordinal)));
|
||||
Assert.All(measurements, measurement => Assert.True(measurement.Value > 0));
|
||||
var serializedMeasurements = string.Join('\n', actual.Select(measurement => measurement.Tags));
|
||||
Assert.DoesNotContain(sensitiveBody, serializedMeasurements, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveFileName, serializedMeasurements, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveSourceUrl, serializedMeasurements, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static ObservedMeasurement Observe(string name, double value, ReadOnlySpan<KeyValuePair<string, object?>> tags) =>
|
||||
new(name, value, tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value?.ToString() ?? string.Empty, StringComparer.Ordinal));
|
||||
|
||||
private static string SerializeTags(IReadOnlyDictionary<string, string> tags) =>
|
||||
string.Join('|', tags.OrderBy(tag => tag.Key, StringComparer.Ordinal).Select(tag => $"{tag.Key}={tag.Value}"));
|
||||
|
||||
private sealed record ObservedMeasurement(string Name, double Value, IReadOnlyDictionary<string, string> Tags);
|
||||
}
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionObservabilityTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
|
||||
postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
[Fact]
|
||||
public async Task Proposal_and_source_requests_never_write_source_content_or_file_metadata_to_logs()
|
||||
{
|
||||
const string sensitiveFileName = "secret-plan.pdf";
|
||||
const string sensitiveSource = "source bytes must stay out of logs";
|
||||
const string sensitiveUrl = "https://private.example/source?token=secret";
|
||||
var sensitiveManifest = $"do not log me {sensitiveUrl}";
|
||||
var captured = new ConcurrentQueue<string>();
|
||||
|
||||
using var loggingFactory = Factory.WithWebHostBuilder(builder =>
|
||||
builder.ConfigureLogging(logging => logging.AddProvider(new CapturingLoggerProvider(captured))));
|
||||
using var client = loggingFactory.CreateClient();
|
||||
var login = await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
login.EnsureSuccessStatusCode();
|
||||
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest(sensitiveManifest, "safe-observability-correlation"))
|
||||
};
|
||||
create.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", "safe-observability-create");
|
||||
create.Headers.Add("X-Correlation-ID", "safe-observability-correlation");
|
||||
var created = await client.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var sourceCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var sourceBody = new ByteArrayContent(System.Text.Encoding.UTF8.GetBytes(sensitiveSource));
|
||||
sourceBody.Headers.ContentType = new MediaTypeHeaderValue("application/pdf");
|
||||
multipart.Add(sourceBody, "file", sensitiveFileName);
|
||||
using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/sources")
|
||||
{
|
||||
Content = multipart
|
||||
};
|
||||
upload.Headers.Add(sourceCsrf.HeaderName, sourceCsrf.Value);
|
||||
upload.Headers.Add("Idempotency-Key", "safe-observability-source");
|
||||
upload.Headers.Add("If-Match", created.Headers.ETag!.Tag);
|
||||
upload.Headers.Add("X-Correlation-ID", "safe-observability-correlation");
|
||||
var staged = await client.SendAsync(upload);
|
||||
Assert.Equal(HttpStatusCode.OK, staged.StatusCode);
|
||||
|
||||
var logText = string.Join('\n', captured);
|
||||
Assert.Contains("safe-observability-correlation", logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveManifest, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveFileName, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveSource, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveUrl, logText, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private sealed class CapturingLoggerProvider(ConcurrentQueue<string> sink) : ILoggerProvider
|
||||
{
|
||||
public ILogger CreateLogger(string categoryName) => new CapturingLogger(categoryName, sink);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class CapturingLogger(string categoryName, ConcurrentQueue<string> sink) : ILogger
|
||||
{
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull
|
||||
{
|
||||
sink.Enqueue($"[{categoryName}] scope: {Format(state)}");
|
||||
return NoopDisposable.Instance;
|
||||
}
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel,
|
||||
EventId eventId,
|
||||
TState state,
|
||||
Exception? exception,
|
||||
Func<TState, Exception?, string> formatter) =>
|
||||
sink.Enqueue($"[{categoryName}] {formatter(state, exception)} {Format(state)} {exception}");
|
||||
|
||||
private static string Format<TState>(TState state) => state is IEnumerable<KeyValuePair<string, object?>> fields
|
||||
? string.Join(' ', fields.Select(field => $"{field.Key}={field.Value}"))
|
||||
: state?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
private sealed class NoopDisposable : IDisposable
|
||||
{
|
||||
public static NoopDisposable Instance { get; } = new();
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using ItWorx.KnowledgeHub.Contracts.Auth;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionPilotAvailabilityTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
|
||||
{
|
||||
[Fact]
|
||||
public async Task Disabled_intake_keeps_knowledge_routes_and_readiness_available_when_optional_mcp_is_unhealthy()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
var login = await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
login.EnsureSuccessStatusCode();
|
||||
|
||||
var session = await client.GetFromJsonAsync<SessionResponse>("/api/v1/auth/session");
|
||||
Assert.NotNull(session);
|
||||
Assert.False(session.Features?.IngestionProposals);
|
||||
|
||||
var intake = await client.GetAsync("/api/v1/ingestion-proposals");
|
||||
Assert.Equal(HttpStatusCode.NotFound, intake.StatusCode);
|
||||
var problem = await intake.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.Equal(IngestionProblemTypes.FeatureDisabled, problem?.Type);
|
||||
|
||||
var knowledge = await client.GetAsync("/api/v1/pages/roots");
|
||||
Assert.Equal(HttpStatusCode.OK, knowledge.StatusCode);
|
||||
|
||||
// Model an unavailable optional external integration. The readiness endpoint
|
||||
// must remain tied to the Hub's own database and storage requirements.
|
||||
var healthChecks = Factory.Services
|
||||
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
|
||||
.Value;
|
||||
healthChecks.Registrations.Add(new HealthCheckRegistration(
|
||||
"optional-mcp",
|
||||
_ => new UnavailableOptionalDependencyHealthCheck(),
|
||||
HealthStatus.Unhealthy,
|
||||
tags: null,
|
||||
timeout: TimeSpan.FromSeconds(1)));
|
||||
|
||||
var readiness = await client.GetAsync("/health/ready");
|
||||
Assert.Equal(HttpStatusCode.OK, readiness.StatusCode);
|
||||
Assert.Equal("Healthy", (await readiness.Content.ReadAsStringAsync()).Trim());
|
||||
|
||||
var liveness = await client.GetAsync("/health/live");
|
||||
Assert.Equal(HttpStatusCode.OK, liveness.StatusCode);
|
||||
}
|
||||
|
||||
private sealed class UnavailableOptionalDependencyHealthCheck : IHealthCheck
|
||||
{
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(HealthCheckResult.Unhealthy("Optional MCP dependency is unavailable."));
|
||||
}
|
||||
}
|
||||
+327
@@ -0,0 +1,327 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using ItWorx.KnowledgeHub.Contracts.Automation;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Contracts.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Domain.Automation;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionPilotContractTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
|
||||
postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
private readonly Dictionary<string, HttpClient> _matrixActors = new(StringComparer.Ordinal);
|
||||
|
||||
[Fact]
|
||||
public async Task Proposal_creation_requires_an_idempotency_key()
|
||||
{
|
||||
_matrixActors.Clear();
|
||||
using var reviewer = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(reviewer, OwnerEmail, OwnerPassword);
|
||||
_matrixActors.Add("reviewer", reviewer);
|
||||
|
||||
var response = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
"/api/v1/ingestion-proposals",
|
||||
new CreateIngestionProposalRequest("Synthetic request without a key", "pilot-no-key"));
|
||||
|
||||
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Synthetic_actor_matrix_enforces_review_and_publication_replay_is_exactly_once()
|
||||
{
|
||||
_matrixActors.Clear();
|
||||
using var reviewer = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(reviewer, OwnerEmail, OwnerPassword);
|
||||
_matrixActors.Add("reviewer", reviewer);
|
||||
|
||||
var writerToken = await CreateServiceAccountAsync(reviewer, "pilot-ingestion-writer", Scopes.IngestionWrite);
|
||||
using var writer = BearerClient(writerToken.RawToken);
|
||||
_matrixActors.Add(Scopes.IngestionWrite, writer);
|
||||
|
||||
var readOnlyToken = await CreateServiceAccountAsync(reviewer, "pilot-read-only", Scopes.PagesRead);
|
||||
using var readOnly = BearerClient(readOnlyToken.RawToken);
|
||||
_matrixActors.Add(Scopes.PagesRead, readOnly);
|
||||
|
||||
const string proposalPath = "/api/v1/ingestion-proposals";
|
||||
const string createKey = "pilot-contract-create-1";
|
||||
var publication = new IngestionPublicationRequest(
|
||||
"Synthetic pilot publication", "# Synthetic body", "WikiPage", "Active", "CreatePage");
|
||||
var created = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"POST",
|
||||
proposalPath,
|
||||
new CreateIngestionProposalRequest("Synthetic contract manifest", "pilot-contract-create", publication),
|
||||
idempotencyKey: createKey);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
Assert.NotNull(created.Headers.ETag);
|
||||
var proposal = Assert.IsType<IngestionProposalDetail>(await created.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
|
||||
var sameKeyDifferentBody = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"POST",
|
||||
proposalPath,
|
||||
new CreateIngestionProposalRequest("Different synthetic manifest", "pilot-contract-create", publication),
|
||||
idempotencyKey: createKey);
|
||||
await AssertTypedProblemAsync(sameKeyDifferentBody, HttpStatusCode.Conflict, ProblemTypes.IdempotencyKeyConflict);
|
||||
|
||||
var missingIngestionScope = await SendMatrixRequestAsync(
|
||||
Scopes.PagesRead,
|
||||
"POST",
|
||||
proposalPath,
|
||||
new CreateIngestionProposalRequest("Must be denied", "pilot-contract-denied"),
|
||||
idempotencyKey: "pilot-contract-denied-1");
|
||||
Assert.Equal(HttpStatusCode.Forbidden, missingIngestionScope.StatusCode);
|
||||
|
||||
var writerCannotReadReviewQueue = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite, "GET", proposalPath);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, writerCannotReadReviewQueue.StatusCode);
|
||||
|
||||
var analyze = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"PATCH",
|
||||
$"{proposalPath}/{proposal.Id}",
|
||||
new PatchIngestionProposalRequest(null, null, "Synthetic analysis is ready for review."),
|
||||
created.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, analyze.StatusCode);
|
||||
var ready = Assert.IsType<IngestionProposalDetail>(await analyze.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
Assert.Equal("ReadyForReview", ready.Status);
|
||||
|
||||
var staleProposalEdit = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"PATCH",
|
||||
$"{proposalPath}/{proposal.Id}",
|
||||
new PatchIngestionProposalRequest("Stale edit must not replace the latest proposal.", null),
|
||||
created.Headers.ETag!.Tag);
|
||||
await AssertTypedProblemAsync(staleProposalEdit, HttpStatusCode.PreconditionFailed, IngestionProblemTypes.ConcurrencyConflict);
|
||||
|
||||
var writerCannotApprove = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"POST",
|
||||
$"{proposalPath}/{proposal.Id}/approve",
|
||||
new IngestionProposalDecisionRequest(null),
|
||||
analyze.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, writerCannotApprove.StatusCode);
|
||||
|
||||
var reviewerCannotPublishBeforeApproval = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
$"{proposalPath}/{proposal.Id}/publish",
|
||||
ifMatch: analyze.Headers.ETag!.Tag,
|
||||
idempotencyKey: "pilot-contract-unapproved-publish");
|
||||
await AssertTypedProblemAsync(
|
||||
reviewerCannotPublishBeforeApproval,
|
||||
HttpStatusCode.Conflict,
|
||||
IngestionProblemTypes.InvalidTransition);
|
||||
|
||||
var approvedResponse = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
$"{proposalPath}/{proposal.Id}/approve",
|
||||
new IngestionProposalDecisionRequest(null),
|
||||
analyze.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, approvedResponse.StatusCode);
|
||||
var approved = Assert.IsType<IngestionProposalDetail>(await approvedResponse.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
Assert.Equal("Approved", approved.Status);
|
||||
|
||||
const string publishKey = "pilot-contract-publish-1";
|
||||
var writerCannotPublish = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"POST",
|
||||
$"{proposalPath}/{proposal.Id}/publish",
|
||||
ifMatch: approvedResponse.Headers.ETag!.Tag,
|
||||
idempotencyKey: publishKey);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, writerCannotPublish.StatusCode);
|
||||
|
||||
var publishedResponse = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
$"{proposalPath}/{proposal.Id}/publish",
|
||||
ifMatch: approvedResponse.Headers.ETag!.Tag,
|
||||
idempotencyKey: publishKey);
|
||||
Assert.Equal(HttpStatusCode.OK, publishedResponse.StatusCode);
|
||||
var published = Assert.IsType<IngestionProposalDetail>(await publishedResponse.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
Assert.Equal("Published", published.Status);
|
||||
Assert.NotNull(published.PublishedPageId);
|
||||
Assert.Equal(1, published.PublishedPageRevisionNumber);
|
||||
|
||||
var replay = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
$"{proposalPath}/{proposal.Id}/publish",
|
||||
ifMatch: approvedResponse.Headers.ETag!.Tag,
|
||||
idempotencyKey: publishKey);
|
||||
Assert.Equal(HttpStatusCode.OK, replay.StatusCode);
|
||||
var replayed = Assert.IsType<IngestionProposalDetail>(await replay.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
Assert.Equal(published.PublishedPageId, replayed.PublishedPageId);
|
||||
Assert.Equal(published.PublishedPageRevisionNumber, replayed.PublishedPageRevisionNumber);
|
||||
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
Assert.Equal(1, await db.Pages.CountAsync(page => page.Id == published.PublishedPageId));
|
||||
Assert.Equal(1, await db.PageRevisions.CountAsync(revision => revision.PageId == published.PublishedPageId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Stale_target_etag_returns_a_typed_problem_and_does_not_publish_a_revision()
|
||||
{
|
||||
_matrixActors.Clear();
|
||||
using var reviewer = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(reviewer, OwnerEmail, OwnerPassword);
|
||||
_matrixActors.Add("reviewer", reviewer);
|
||||
|
||||
var writerToken = await CreateServiceAccountAsync(reviewer, "pilot-stale-target-writer", Scopes.IngestionWrite);
|
||||
using var writer = BearerClient(writerToken.RawToken);
|
||||
_matrixActors.Add(Scopes.IngestionWrite, writer);
|
||||
|
||||
var pageResponse = await SetupHelper.PostWithCsrfAsync(
|
||||
reviewer,
|
||||
"/api/v1/pages",
|
||||
new CreatePageRequest(null, "Synthetic stale target", "WikiPage", "Initial page body.", null));
|
||||
Assert.Equal(HttpStatusCode.OK, pageResponse.StatusCode);
|
||||
var initialPage = Assert.IsType<PageDetail>(await pageResponse.Content.ReadFromJsonAsync<PageDetail>());
|
||||
|
||||
var editResponse = await SetupHelper.PutWithCsrfAsync(
|
||||
reviewer,
|
||||
$"/api/v1/pages/{initialPage.Id}",
|
||||
new UpdatePageRequest(
|
||||
initialPage.Title,
|
||||
"Target changed after the proposal snapshot.",
|
||||
initialPage.DocumentType,
|
||||
initialPage.DocumentStatus,
|
||||
"Synthetic target ETag advance"),
|
||||
initialPage.ETag);
|
||||
Assert.Equal(HttpStatusCode.OK, editResponse.StatusCode);
|
||||
var currentPage = Assert.IsType<PageDetail>(await editResponse.Content.ReadFromJsonAsync<PageDetail>());
|
||||
Assert.NotEqual(initialPage.ETag, currentPage.ETag);
|
||||
|
||||
var revisionsBeforePublish = await GetPageRevisionCountAsync(initialPage.Id);
|
||||
var createProposal = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"POST",
|
||||
"/api/v1/ingestion-proposals",
|
||||
new CreateIngestionProposalRequest(
|
||||
"Synthetic update proposal",
|
||||
"pilot-stale-target",
|
||||
new IngestionPublicationRequest(
|
||||
initialPage.Title,
|
||||
"Proposed replacement body.",
|
||||
initialPage.DocumentType,
|
||||
initialPage.DocumentStatus,
|
||||
"UpdatePage",
|
||||
TargetPageId: initialPage.Id,
|
||||
ExpectedTargetETag: initialPage.ETag)),
|
||||
idempotencyKey: "pilot-stale-target-create");
|
||||
Assert.Equal(HttpStatusCode.Created, createProposal.StatusCode);
|
||||
var proposal = Assert.IsType<IngestionProposalDetail>(await createProposal.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
|
||||
var analyzed = await SendMatrixRequestAsync(
|
||||
Scopes.IngestionWrite,
|
||||
"PATCH",
|
||||
$"/api/v1/ingestion-proposals/{proposal.Id}",
|
||||
new PatchIngestionProposalRequest(null, null, "Synthetic update analysis is ready."),
|
||||
createProposal.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, analyzed.StatusCode);
|
||||
|
||||
var approved = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
$"/api/v1/ingestion-proposals/{proposal.Id}/approve",
|
||||
new IngestionProposalDecisionRequest(null),
|
||||
analyzed.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, approved.StatusCode);
|
||||
|
||||
var stalePublish = await SendMatrixRequestAsync(
|
||||
"reviewer",
|
||||
"POST",
|
||||
$"/api/v1/ingestion-proposals/{proposal.Id}/publish",
|
||||
ifMatch: approved.Headers.ETag!.Tag,
|
||||
idempotencyKey: "pilot-stale-target-publish");
|
||||
await AssertTypedProblemAsync(stalePublish, HttpStatusCode.Conflict, IngestionProblemTypes.StaleTarget);
|
||||
|
||||
var proposalAfterConflict = await reviewer.GetFromJsonAsync<IngestionProposalDetail>(
|
||||
$"/api/v1/ingestion-proposals/{proposal.Id}");
|
||||
Assert.Equal("ReadyForReview", proposalAfterConflict?.Status);
|
||||
Assert.Equal(revisionsBeforePublish, await GetPageRevisionCountAsync(initialPage.Id));
|
||||
|
||||
var pageAfterConflict = await reviewer.GetFromJsonAsync<PageDetail>($"/api/v1/pages/{initialPage.Id}");
|
||||
Assert.Equal("Target changed after the proposal snapshot.", pageAfterConflict?.Markdown);
|
||||
Assert.Equal(currentPage.ETag, pageAfterConflict?.ETag);
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> SendMatrixRequestAsync(
|
||||
string actor,
|
||||
string method,
|
||||
string path,
|
||||
object? body = null,
|
||||
string? ifMatch = null,
|
||||
string? idempotencyKey = null)
|
||||
{
|
||||
var client = _matrixActors[actor];
|
||||
using var request = new HttpRequestMessage(new HttpMethod(method), path);
|
||||
if (body is not null)
|
||||
{
|
||||
request.Content = JsonContent.Create(body);
|
||||
}
|
||||
|
||||
if (ifMatch is not null)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation("If-Match", ifMatch);
|
||||
}
|
||||
|
||||
if (idempotencyKey is not null)
|
||||
{
|
||||
request.Headers.Add("Idempotency-Key", idempotencyKey);
|
||||
}
|
||||
|
||||
if (actor == "reviewer" && request.Method != HttpMethod.Get)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
}
|
||||
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static async Task<CreatedServiceAccountToken> CreateServiceAccountAsync(HttpClient reviewer, string name, params string[] scopes)
|
||||
{
|
||||
var response = await SetupHelper.PostWithCsrfAsync(
|
||||
reviewer,
|
||||
"/api/v1/service-accounts",
|
||||
new CreateServiceAccountRequest(name, "Synthetic pilot contract actor", scopes));
|
||||
response.EnsureSuccessStatusCode();
|
||||
return Assert.IsType<CreatedServiceAccountToken>(await response.Content.ReadFromJsonAsync<CreatedServiceAccountToken>());
|
||||
}
|
||||
|
||||
private HttpClient BearerClient(string rawToken)
|
||||
{
|
||||
var client = Factory.CreateClient();
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", rawToken);
|
||||
return client;
|
||||
}
|
||||
|
||||
private async Task<int> GetPageRevisionCountAsync(Guid pageId)
|
||||
{
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
return await db.PageRevisions.CountAsync(revision => revision.PageId == pageId);
|
||||
}
|
||||
|
||||
private static async Task AssertTypedProblemAsync(HttpResponseMessage response, HttpStatusCode statusCode, string type)
|
||||
{
|
||||
Assert.Equal(statusCode, response.StatusCode);
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.Equal(type, problem?.Type);
|
||||
}
|
||||
}
|
||||
+767
@@ -0,0 +1,767 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Contracts.Auth;
|
||||
using ItWorx.KnowledgeHub.Contracts.Automation;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Domain.Automation;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>HTTP contract for the bounded trusted-intake route group.</summary>
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalApiTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
|
||||
{
|
||||
[Fact]
|
||||
public async Task Disabled_feature_returns_the_typed_ingestion_problem_instead_of_exposing_a_route()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
|
||||
var response = await SetupHelper.PostWithCsrfAsync(
|
||||
client,
|
||||
"/api/v1/ingestion-proposals",
|
||||
new CreateIngestionProposalRequest("safe manifest", "correlation-1"));
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.Equal(IngestionProblemTypes.FeatureDisabled, problem?.Type);
|
||||
|
||||
var sourceCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var sourceContent = new MultipartFormDataContent();
|
||||
var sourceBody = new ByteArrayContent("disabled source"u8.ToArray());
|
||||
sourceBody.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
sourceContent.Add(sourceBody, "file", "disabled.txt");
|
||||
using var sourceRequest = new HttpRequestMessage(
|
||||
HttpMethod.Post,
|
||||
$"/api/v1/ingestion-proposals/{Guid.NewGuid()}/sources")
|
||||
{
|
||||
Content = sourceContent
|
||||
};
|
||||
sourceRequest.Headers.Add(sourceCsrf.HeaderName, sourceCsrf.Value);
|
||||
sourceRequest.Headers.Add("Idempotency-Key", "disabled-source-upload");
|
||||
sourceRequest.Headers.TryAddWithoutValidation("If-Match", "\"unused\"");
|
||||
|
||||
var sourceResponse = await client.SendAsync(sourceRequest);
|
||||
Assert.Equal(HttpStatusCode.NotFound, sourceResponse.StatusCode);
|
||||
var sourceProblem = await sourceResponse.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.Equal(IngestionProblemTypes.FeatureDisabled, sourceProblem?.Type);
|
||||
}
|
||||
}
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalSourceApiTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
|
||||
postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
[Fact]
|
||||
public async Task Upload_retry_with_a_new_multipart_boundary_replays_once_and_changed_bytes_conflict()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{ Content = JsonContent.Create(new CreateIngestionProposalRequest("Retry document", "retry-source")) };
|
||||
create.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", Guid.NewGuid().ToString());
|
||||
var created = await client.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
var key = Guid.NewGuid().ToString();
|
||||
|
||||
async Task<HttpResponseMessage> UploadAsync(string content)
|
||||
{
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var file = new ByteArrayContent(System.Text.Encoding.UTF8.GetBytes(content));
|
||||
file.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
multipart.Add(file, "file", "retry.txt");
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal.Id}/sources")
|
||||
{ Content = multipart };
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.Add("Idempotency-Key", key);
|
||||
request.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
using var first = await UploadAsync("same bytes");
|
||||
using var replay = await UploadAsync("same bytes");
|
||||
Assert.Equal(HttpStatusCode.OK, first.StatusCode);
|
||||
Assert.Equal(HttpStatusCode.OK, replay.StatusCode);
|
||||
var firstDetail = await first.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
var replayDetail = await replay.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal(Assert.Single(firstDetail!.Sources).AttachmentReferenceId, Assert.Single(replayDetail!.Sources).AttachmentReferenceId);
|
||||
using var conflict = await UploadAsync("changed bytes");
|
||||
Assert.Equal(HttpStatusCode.Conflict, conflict.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Human_document_upload_enters_review_only_after_source_and_explicit_submission()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest("Handmatig document", "manual-source",
|
||||
new IngestionPublicationRequest("Handboek", "Origineel als bijlage", "Reference", "Draft", "CreatePage")))
|
||||
};
|
||||
create.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", "manual-source-create");
|
||||
var created = await client.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var detail = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(detail);
|
||||
|
||||
using var tooEarly = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{detail.Id}/submit")
|
||||
{ Content = JsonContent.Create(new { }) };
|
||||
tooEarly.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
tooEarly.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.Conflict, (await client.SendAsync(tooEarly)).StatusCode);
|
||||
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var file = new ByteArrayContent("Original document"u8.ToArray());
|
||||
file.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
multipart.Add(file, "file", "document.txt");
|
||||
using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{detail.Id}/sources")
|
||||
{ Content = multipart };
|
||||
upload.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
upload.Headers.Add("Idempotency-Key", "manual-source-upload");
|
||||
upload.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
var uploaded = await client.SendAsync(upload);
|
||||
Assert.Equal(HttpStatusCode.OK, uploaded.StatusCode);
|
||||
|
||||
using var submit = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{detail.Id}/submit")
|
||||
{ Content = JsonContent.Create(new { }) };
|
||||
submit.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
submit.Headers.TryAddWithoutValidation("If-Match", uploaded.Headers.ETag!.Tag);
|
||||
var submitted = await client.SendAsync(submit);
|
||||
Assert.Equal(HttpStatusCode.OK, submitted.StatusCode);
|
||||
var reviewed = await submitted.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal("ReadyForReview", reviewed?.Status);
|
||||
Assert.Null(reviewed?.Analysis);
|
||||
Assert.Single(reviewed!.Sources);
|
||||
Assert.Contains(reviewed.Revisions, revision => revision.Operation == "manual-review-submitted");
|
||||
|
||||
using var lateMultipart = new MultipartFormDataContent();
|
||||
var lateFile = new ByteArrayContent("late content"u8.ToArray());
|
||||
lateFile.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
lateMultipart.Add(lateFile, "file", "late.txt");
|
||||
using var lateUpload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{detail.Id}/sources")
|
||||
{ Content = lateMultipart };
|
||||
lateUpload.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
lateUpload.Headers.Add("Idempotency-Key", "manual-source-late-upload");
|
||||
lateUpload.Headers.TryAddWithoutValidation("If-Match", submitted.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.Conflict, (await client.SendAsync(lateUpload)).StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Service_account_cannot_use_the_human_manual_submission_route()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var token = await CreateServiceAccountAsync(owner, "manual-submit-denied");
|
||||
using var service = BearerClient(token.RawToken);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{Guid.NewGuid()}/submit")
|
||||
{ Content = JsonContent.Create(new { }) };
|
||||
request.Headers.TryAddWithoutValidation("If-Match", "\"1\"");
|
||||
var response = await service.SendAsync(request);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Source_upload_rate_limit_is_per_remote_ip()
|
||||
{
|
||||
using var clientA = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(clientA, OwnerEmail, OwnerPassword);
|
||||
var created = await CreateProposalAsync(clientA, "rate-limited source", "source-rate-limit", "source-rate-create");
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var currentETag = created.Headers.ETag!.Tag;
|
||||
|
||||
clientA.DefaultRequestHeaders.Add("X-Test-Remote-Ip", "192.0.2.20");
|
||||
for (var attempt = 0; attempt < 5; attempt++)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(clientA);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var file = new ByteArrayContent("rate-limit source"u8.ToArray());
|
||||
file.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
multipart.Add(file, "file", $"source-{attempt}.txt");
|
||||
using var upload = new HttpRequestMessage(
|
||||
HttpMethod.Post,
|
||||
$"/api/v1/ingestion-proposals/{proposal!.Id}/sources")
|
||||
{
|
||||
Content = multipart
|
||||
};
|
||||
upload.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
upload.Headers.Add("Idempotency-Key", $"source-rate-upload-a-{attempt}");
|
||||
upload.Headers.TryAddWithoutValidation("If-Match", currentETag);
|
||||
|
||||
var response = await clientA.SendAsync(upload);
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
currentETag = response.Headers.ETag!.Tag;
|
||||
}
|
||||
|
||||
using var clientB = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(clientB, OwnerEmail, OwnerPassword);
|
||||
clientB.DefaultRequestHeaders.Add("X-Test-Remote-Ip", "192.0.2.21");
|
||||
var clientBCsrf = await SetupHelper.GetCsrfTokenAsync(clientB);
|
||||
using var clientBMultipart = new MultipartFormDataContent();
|
||||
var clientBFile = new ByteArrayContent("other source"u8.ToArray());
|
||||
clientBFile.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
clientBMultipart.Add(clientBFile, "file", "source-client-b.txt");
|
||||
using var clientBUpload = new HttpRequestMessage(
|
||||
HttpMethod.Post,
|
||||
$"/api/v1/ingestion-proposals/{proposal!.Id}/sources")
|
||||
{
|
||||
Content = clientBMultipart
|
||||
};
|
||||
clientBUpload.Headers.Add(clientBCsrf.HeaderName, clientBCsrf.Value);
|
||||
clientBUpload.Headers.Add("Idempotency-Key", "source-rate-upload-b");
|
||||
clientBUpload.Headers.TryAddWithoutValidation("If-Match", currentETag);
|
||||
var clientBResponse = await clientB.SendAsync(clientBUpload);
|
||||
Assert.Equal(HttpStatusCode.OK, clientBResponse.StatusCode);
|
||||
currentETag = clientBResponse.Headers.ETag!.Tag;
|
||||
|
||||
var clientACsrf = await SetupHelper.GetCsrfTokenAsync(clientA);
|
||||
using var exhaustedMultipart = new MultipartFormDataContent();
|
||||
var exhaustedFile = new ByteArrayContent("blocked source"u8.ToArray());
|
||||
exhaustedFile.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
exhaustedMultipart.Add(exhaustedFile, "file", "source-client-a-blocked.txt");
|
||||
using var exhaustedUpload = new HttpRequestMessage(
|
||||
HttpMethod.Post,
|
||||
$"/api/v1/ingestion-proposals/{proposal.Id}/sources")
|
||||
{
|
||||
Content = exhaustedMultipart
|
||||
};
|
||||
exhaustedUpload.Headers.Add(clientACsrf.HeaderName, clientACsrf.Value);
|
||||
exhaustedUpload.Headers.Add("Idempotency-Key", "source-rate-upload-a-exhausted");
|
||||
exhaustedUpload.Headers.TryAddWithoutValidation("If-Match", currentETag);
|
||||
var clientAResponse = await clientA.SendAsync(exhaustedUpload);
|
||||
Assert.Equal(HttpStatusCode.TooManyRequests, clientAResponse.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Patch_requires_an_IfMatch_header_and_rejects_a_stale_ETag()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var created = await CreateProposalAsync(client, "etag manifest", "etag-correlation", "etag-create");
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var missingIfMatch = await SetupHelper.PatchWithCsrfAsync(
|
||||
client, $"/api/v1/ingestion-proposals/{proposal!.Id}",
|
||||
new PatchIngestionProposalRequest("without concurrency precondition", null));
|
||||
Assert.Equal((HttpStatusCode)428, missingIfMatch.StatusCode);
|
||||
|
||||
var firstCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var firstPatch = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest("updated manifest", null))
|
||||
};
|
||||
firstPatch.Headers.Add(firstCsrf.HeaderName, firstCsrf.Value);
|
||||
firstPatch.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
var firstUpdated = await client.SendAsync(firstPatch);
|
||||
Assert.Equal(HttpStatusCode.OK, firstUpdated.StatusCode);
|
||||
|
||||
var staleCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var stalePatch = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest("stale manifest", null))
|
||||
};
|
||||
stalePatch.Headers.Add(staleCsrf.HeaderName, staleCsrf.Value);
|
||||
stalePatch.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag.Tag);
|
||||
var stale = await client.SendAsync(stalePatch);
|
||||
|
||||
Assert.Equal(HttpStatusCode.PreconditionFailed, stale.StatusCode);
|
||||
var problem = await stale.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.Equal(IngestionProblemTypes.ConcurrencyConflict, problem?.Type);
|
||||
var current = await client.GetFromJsonAsync<IngestionProposalDetail>($"/api/v1/ingestion-proposals/{proposal.Id}");
|
||||
Assert.Equal("updated manifest", current?.Manifest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reusing_an_idempotency_key_with_a_different_proposal_body_returns_conflict()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
|
||||
var first = await CreateProposalAsync(client, "original manifest", "idempotency-correlation", "same-key");
|
||||
Assert.Equal(HttpStatusCode.Created, first.StatusCode);
|
||||
|
||||
var second = await CreateProposalAsync(client, "different manifest", "idempotency-correlation", "same-key");
|
||||
Assert.Equal(HttpStatusCode.Conflict, second.StatusCode);
|
||||
var problem = await second.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.Equal(ProblemTypes.IdempotencyKeyConflict, problem?.Type);
|
||||
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
Assert.Equal(1, await db.IngestionProposals.CountAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task List_filters_by_status_and_uses_cursor_pagination_without_duplicates()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var receivedOne = await CreateProposalAsync(owner, "received one", "queue-1", "queue-1");
|
||||
var receivedTwo = await CreateProposalAsync(owner, "received two", "queue-2", "queue-2");
|
||||
var receivedOneDetail = await receivedOne.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
var receivedTwoDetail = await receivedTwo.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(receivedOneDetail);
|
||||
Assert.NotNull(receivedTwoDetail);
|
||||
|
||||
var service = await CreateServiceAccountAsync(owner, "queue-analyzer");
|
||||
using var serviceClient = BearerClient(service.RawToken);
|
||||
var ready = await CreateProposalAsync(serviceClient, "ready manifest", "queue-ready", "queue-ready");
|
||||
var readyDetail = await ready.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(readyDetail);
|
||||
using var analysis = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{readyDetail!.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest(null, null, "safe queue analysis"))
|
||||
};
|
||||
analysis.Headers.TryAddWithoutValidation("If-Match", ready.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, (await serviceClient.SendAsync(analysis)).StatusCode);
|
||||
|
||||
var receivedIds = new List<Guid>();
|
||||
Guid? cursor = null;
|
||||
do
|
||||
{
|
||||
var path = cursor is null
|
||||
? "/api/v1/ingestion-proposals?status=Received&limit=1"
|
||||
: $"/api/v1/ingestion-proposals?status=Received&limit=1&cursor={cursor}";
|
||||
var page = await owner.GetFromJsonAsync<IngestionProposalListResponse>(path);
|
||||
Assert.NotNull(page);
|
||||
Assert.All(page.Items, item => Assert.Equal("Received", item.Status));
|
||||
receivedIds.AddRange(page.Items.Select(item => item.Id));
|
||||
cursor = page.NextCursor;
|
||||
} while (cursor is not null);
|
||||
|
||||
Assert.Equal(2, receivedIds.Count);
|
||||
Assert.Equal(2, receivedIds.Distinct().Count());
|
||||
Assert.Contains(receivedOneDetail!.Id, receivedIds);
|
||||
Assert.Contains(receivedTwoDetail!.Id, receivedIds);
|
||||
|
||||
var readyPage = await owner.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=ReadyForReview");
|
||||
var readySummary = Assert.Single(readyPage!.Items);
|
||||
Assert.Equal(readyDetail.Id, readySummary.Id);
|
||||
Assert.Equal("ReadyForReview", readySummary.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Numbered_queue_pages_are_stable_for_equal_timestamps_and_unknown_cursors_do_not_change_order()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var olderResponse = await CreateProposalAsync(owner, "older queue item", "queue-older", "queue-older");
|
||||
var newerResponse = await CreateProposalAsync(owner, "newer queue item", "queue-newer", "queue-newer");
|
||||
var tiedResponse = await CreateProposalAsync(owner, "same-time queue item", "queue-tied", "queue-tied");
|
||||
var older = await olderResponse.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
var newer = await newerResponse.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
var tied = await tiedResponse.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(older);
|
||||
Assert.NotNull(newer);
|
||||
Assert.NotNull(tied);
|
||||
|
||||
// Pin the immutable creation time so this regression is deterministic even
|
||||
// on a host whose clock has coarse timestamp precision.
|
||||
using (var scope = Factory.Services.CreateScope())
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
var createdAt = DateTimeOffset.UtcNow;
|
||||
await db.IngestionProposalRevisions
|
||||
.Where(revision => EF.Property<Guid>(revision, "IngestionProposalId") == older!.Id)
|
||||
.ExecuteUpdateAsync(update => update.SetProperty(revision => revision.OccurredAt, createdAt.AddMinutes(-5)));
|
||||
await db.IngestionProposalRevisions
|
||||
.Where(revision => EF.Property<Guid>(revision, "IngestionProposalId") == newer!.Id)
|
||||
.ExecuteUpdateAsync(update => update.SetProperty(revision => revision.OccurredAt, createdAt));
|
||||
await db.IngestionProposalRevisions
|
||||
.Where(revision => EF.Property<Guid>(revision, "IngestionProposalId") == tied!.Id)
|
||||
.ExecuteUpdateAsync(update => update.SetProperty(revision => revision.OccurredAt, createdAt));
|
||||
}
|
||||
|
||||
var expectedIds = new[] { newer!.Id, tied!.Id }
|
||||
.OrderByDescending(id => id)
|
||||
.Append(older!.Id)
|
||||
.ToArray();
|
||||
var actualIds = new List<Guid>();
|
||||
Guid? cursor = null;
|
||||
do
|
||||
{
|
||||
var path = cursor is null
|
||||
? "/api/v1/ingestion-proposals?page=1&pageSize=1"
|
||||
: $"/api/v1/ingestion-proposals?cursor={cursor}&limit=1";
|
||||
var page = await owner.GetFromJsonAsync<IngestionProposalListResponse>(path);
|
||||
Assert.NotNull(page);
|
||||
actualIds.AddRange(page.Items.Select(item => item.Id));
|
||||
cursor = page.NextCursor;
|
||||
} while (cursor is not null);
|
||||
|
||||
Assert.Equal(expectedIds, actualIds);
|
||||
|
||||
var unknownCursor = await owner.GetFromJsonAsync<IngestionProposalListResponse>(
|
||||
$"/api/v1/ingestion-proposals?cursor={Guid.NewGuid()}&limit=1");
|
||||
Assert.NotNull(unknownCursor);
|
||||
Assert.Empty(unknownCursor.Items);
|
||||
Assert.Null(unknownCursor.NextCursor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Queue_returns_a_bounded_page_total_and_truthful_publication_dossier()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var created = await CreateProposalAsync(client, "paged", "queue-contract", "queue-contract-key");
|
||||
var session = await client.GetFromJsonAsync<SessionResponse>("/api/v1/auth/session");
|
||||
Assert.NotNull(session?.UserId);
|
||||
var response = await client.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?page=1&pageSize=25&submitterId=" + session.UserId);
|
||||
|
||||
Assert.NotNull(response);
|
||||
Assert.Equal(1, response.Page);
|
||||
Assert.Equal(25, response.PageSize);
|
||||
Assert.True(response.Total >= 1);
|
||||
var item = Assert.Single(response.Items);
|
||||
Assert.Equal(session.UserId, item.SubmitterId);
|
||||
Assert.Null(item.Destination);
|
||||
|
||||
var createdDetail = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(createdDetail);
|
||||
Assert.NotNull(createdDetail.Permissions);
|
||||
Assert.Empty(createdDetail.Warnings);
|
||||
Assert.Empty(createdDetail.Citations);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Create_persists_its_correlation_and_echoes_the_request_correlation_header()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
const string requestCorrelation = "ingestion-http-correlation";
|
||||
var response = await CreateProposalAsync(client, "correlated manifest", "proposal-correlation", "correlation-key", requestCorrelation);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
||||
Assert.Equal(requestCorrelation, Assert.Single(response.Headers.GetValues("X-Correlation-ID")));
|
||||
var detail = await response.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal("proposal-correlation", detail?.CorrelationId);
|
||||
Assert.NotNull(response.Headers.ETag);
|
||||
Assert.Equal($"\"{detail!.ETag}\"", response.Headers.ETag!.Tag);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Accepted_source_uses_the_stored_digest_and_a_proposal_only_attachment_usage()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest("safe manifest", "source-correlation"))
|
||||
};
|
||||
create.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", "source-create-1");
|
||||
var created = await client.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var sourceCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var sourceBody = new ByteArrayContent("trusted source bytes"u8.ToArray());
|
||||
sourceBody.Headers.ContentType = new MediaTypeHeaderValue("application/pdf");
|
||||
multipart.Add(sourceBody, "file", "evidence.pdf");
|
||||
using var sourceRequest = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal.Id}/sources") { Content = multipart };
|
||||
sourceRequest.Headers.Add(sourceCsrf.HeaderName, sourceCsrf.Value);
|
||||
sourceRequest.Headers.Add("Idempotency-Key", "source-upload-1");
|
||||
sourceRequest.Headers.Add("If-Match", created.Headers.ETag!.Tag);
|
||||
var response = await client.SendAsync(sourceRequest);
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
var detail = await response.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
var staged = Assert.Single(detail!.Sources);
|
||||
Assert.Equal("evidence.pdf", staged.FileName);
|
||||
Assert.NotEqual(new string('0', 64), staged.DigestSha256);
|
||||
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
var usage = await db.AttachmentUsages.SingleAsync(item => item.AttachmentReferenceId == staged.AttachmentReferenceId);
|
||||
var reference = await db.AttachmentReferences.SingleAsync(item => item.Id == staged.AttachmentReferenceId);
|
||||
Assert.Equal(proposal.Id, usage.IngestionProposalId);
|
||||
Assert.Null(usage.PageId);
|
||||
Assert.Null(usage.StagingSessionId);
|
||||
Assert.Equal(reference.BlobSha256, staged.DigestSha256);
|
||||
var audit = await db.AuditEvents.SingleAsync(item => item.Action == "ingestion-proposal.source-staged" && item.TargetId == proposal.Id.ToString());
|
||||
Assert.Equal("User", audit.ActorKind);
|
||||
Assert.Equal("web", audit.Source);
|
||||
Assert.DoesNotContain("trusted source bytes", audit.Summary, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rejected_source_does_not_create_a_proposal_attachment_usage_or_source_descriptor()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest("safe manifest", "rejected-source"))
|
||||
};
|
||||
create.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", "rejected-source-create");
|
||||
var created = await client.SendAsync(create);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var sourceCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var executable = new ByteArrayContent([0x4D, 0x5A, 0x90, 0x00]);
|
||||
executable.Headers.ContentType = new MediaTypeHeaderValue("application/pdf");
|
||||
multipart.Add(executable, "file", "disguised.pdf");
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/sources") { Content = multipart };
|
||||
request.Headers.Add(sourceCsrf.HeaderName, sourceCsrf.Value);
|
||||
request.Headers.Add("Idempotency-Key", "rejected-source-upload");
|
||||
request.Headers.Add("If-Match", created.Headers.ETag!.Tag);
|
||||
var response = await client.SendAsync(request);
|
||||
|
||||
Assert.Equal(HttpStatusCode.UnprocessableEntity, response.StatusCode);
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
Assert.Empty(await db.IngestionProposalSources.Where(source => EF.Property<Guid>(source, "IngestionProposalId") == proposal.Id).ToListAsync());
|
||||
Assert.Empty(await db.AttachmentUsages.Where(usage => usage.IngestionProposalId == proposal.Id).ToListAsync());
|
||||
Assert.Empty(await db.AttachmentReferences.ToListAsync());
|
||||
Assert.Empty(await db.AttachmentBlobs.ToListAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Service_account_can_complete_only_its_own_analysis_and_read_its_metadata_only_status()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var first = await CreateServiceAccountAsync(owner, "ingestion-first");
|
||||
var second = await CreateServiceAccountAsync(owner, "ingestion-second");
|
||||
using var firstClient = BearerClient(first.RawToken);
|
||||
using var secondClient = BearerClient(second.RawToken);
|
||||
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest("service manifest", "service-correlation"))
|
||||
};
|
||||
create.Headers.Add("Idempotency-Key", "service-proposal");
|
||||
var created = await firstClient.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var foreignDetail = await secondClient.GetAsync($"/api/v1/ingestion-proposals/{proposal!.Id}");
|
||||
Assert.Equal(HttpStatusCode.NotFound, foreignDetail.StatusCode);
|
||||
|
||||
using var foreignAnalysis = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest(null, null, "foreign analysis"))
|
||||
};
|
||||
foreignAnalysis.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await secondClient.SendAsync(foreignAnalysis)).StatusCode);
|
||||
|
||||
var ownStatus = await firstClient.GetAsync($"/api/v1/ingestion-proposals/{proposal.Id}");
|
||||
Assert.Equal(HttpStatusCode.OK, ownStatus.StatusCode);
|
||||
var safeStatus = await ownStatus.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal("service manifest", safeStatus?.Manifest);
|
||||
Assert.Empty(safeStatus!.Sources);
|
||||
|
||||
using var forbiddenPatch = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest("forbidden", null, null))
|
||||
};
|
||||
forbiddenPatch.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, (await firstClient.SendAsync(forbiddenPatch)).StatusCode);
|
||||
|
||||
using var analysisPatch = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest(null, null, "Safe analysis outcome."))
|
||||
};
|
||||
analysisPatch.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
var analyzed = await firstClient.SendAsync(analysisPatch);
|
||||
Assert.Equal(HttpStatusCode.OK, analyzed.StatusCode);
|
||||
var detail = await analyzed.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(detail);
|
||||
Assert.Equal("ReadyForReview", detail.Status);
|
||||
Assert.Equal("service manifest", detail.Manifest);
|
||||
Assert.Equal("Safe analysis outcome.", detail.Analysis);
|
||||
Assert.Null(detail.ReviewerComment);
|
||||
Assert.Null(detail.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Service_account_with_ingestion_write_cannot_make_a_human_review_decision()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var service = await CreateServiceAccountAsync(owner, "review-denied");
|
||||
using var serviceClient = BearerClient(service.RawToken);
|
||||
var created = await CreateProposalAsync(serviceClient, "review-bound manifest", "review-bound", "review-bound-key");
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Forbidden, (await serviceClient.GetAsync("/api/v1/ingestion-proposals")).StatusCode);
|
||||
|
||||
foreach (var (route, request) in new[]
|
||||
{
|
||||
("request-changes", new IngestionProposalDecisionRequest("review requested")),
|
||||
("approve", new IngestionProposalDecisionRequest(null)),
|
||||
("reject", new IngestionProposalDecisionRequest("review rejected"))
|
||||
})
|
||||
{
|
||||
using var decision = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/{route}")
|
||||
{
|
||||
Content = JsonContent.Create(request)
|
||||
};
|
||||
decision.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, (await serviceClient.SendAsync(decision)).StatusCode);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cookie_reviewer_can_approve_a_ready_for_review_proposal()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var service = await CreateServiceAccountAsync(owner, "review-positive");
|
||||
using var serviceClient = BearerClient(service.RawToken);
|
||||
var created = await CreateProposalAsync(serviceClient, "reviewable manifest", "review-positive", "review-positive-key");
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
using var analysis = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal!.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest(null, null, "safe review analysis"))
|
||||
};
|
||||
analysis.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
var analyzed = await serviceClient.SendAsync(analysis);
|
||||
var ready = await analyzed.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal(HttpStatusCode.OK, analyzed.StatusCode);
|
||||
Assert.NotNull(ready);
|
||||
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(owner);
|
||||
using var approve = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal.Id}/approve")
|
||||
{
|
||||
Content = JsonContent.Create(new IngestionProposalDecisionRequest(null))
|
||||
};
|
||||
approve.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
approve.Headers.TryAddWithoutValidation("If-Match", $"\"{ready!.ETag}\"");
|
||||
var approved = await owner.SendAsync(approve);
|
||||
var responseBody = await approved.Content.ReadAsStringAsync();
|
||||
Assert.True(approved.IsSuccessStatusCode,
|
||||
$"Expected approval to succeed, received {(int)approved.StatusCode} {approved.StatusCode}: {responseBody}");
|
||||
using var responseJson = JsonDocument.Parse(responseBody);
|
||||
var status = responseJson.RootElement.GetProperty("status");
|
||||
Assert.Equal(JsonValueKind.String, status.ValueKind);
|
||||
Assert.Equal("Approved", status.GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Service_source_upload_audits_the_service_account_with_api_source()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var service = await CreateServiceAccountAsync(owner, "service-source-audit");
|
||||
using var serviceClient = BearerClient(service.RawToken);
|
||||
var created = await CreateProposalAsync(serviceClient, "source audit manifest", "service-source-audit", "service-source-create");
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var sourceBody = new ByteArrayContent("service evidence"u8.ToArray());
|
||||
sourceBody.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
multipart.Add(sourceBody, "file", "service-evidence.txt");
|
||||
using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/sources") { Content = multipart };
|
||||
upload.Headers.Add("Idempotency-Key", "service-source-upload");
|
||||
upload.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, (await serviceClient.SendAsync(upload)).StatusCode);
|
||||
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
||||
var audit = await db.AuditEvents.SingleAsync(item => item.Action == "ingestion-proposal.source-staged" && item.TargetId == proposal.Id.ToString());
|
||||
Assert.Equal("ServiceAccount", audit.ActorKind);
|
||||
Assert.Equal("api", audit.Source);
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> CreateProposalAsync(HttpClient client, string manifest, string correlationId, string idempotencyKey,
|
||||
string? requestCorrelation = null)
|
||||
{
|
||||
var csrf = client.DefaultRequestHeaders.Authorization is null ? await SetupHelper.GetCsrfTokenAsync(client) : null;
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest(manifest, correlationId))
|
||||
};
|
||||
if (csrf is not null) request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.Add("Idempotency-Key", idempotencyKey);
|
||||
if (requestCorrelation is not null) request.Headers.Add("X-Correlation-ID", requestCorrelation);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static async Task<CreatedServiceAccountToken> CreateServiceAccountAsync(HttpClient owner, string name)
|
||||
{
|
||||
var response = await SetupHelper.PostWithCsrfAsync(owner, "/api/v1/service-accounts",
|
||||
new CreateServiceAccountRequest(name, "trusted-intake test", [Scopes.IngestionWrite]));
|
||||
response.EnsureSuccessStatusCode();
|
||||
var token = await response.Content.ReadFromJsonAsync<CreatedServiceAccountToken>();
|
||||
return Assert.IsType<CreatedServiceAccountToken>(token);
|
||||
}
|
||||
|
||||
private HttpClient BearerClient(string token)
|
||||
{
|
||||
var client = Factory.CreateClient();
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
|
||||
return client;
|
||||
}
|
||||
}
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalSourceLimitApiTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
|
||||
postgres, new Dictionary<string, string?>
|
||||
{
|
||||
["IngestionProposal:Enabled"] = "true",
|
||||
["IngestionProposal:MaximumSourceBytes"] = "4"
|
||||
})
|
||||
{
|
||||
[Fact]
|
||||
public async Task Source_larger_than_the_configured_limit_is_rejected_before_staging()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
var created = await CreateProposalAsync(client, "limit manifest", "limit-correlation", "limit-create");
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var sourceBody = new ByteArrayContent("12345"u8.ToArray());
|
||||
sourceBody.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
multipart.Add(sourceBody, "file", "too-large.txt");
|
||||
using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/sources") { Content = multipart };
|
||||
upload.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
upload.Headers.Add("Idempotency-Key", "limit-source");
|
||||
upload.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
|
||||
Assert.Equal(HttpStatusCode.RequestEntityTooLarge, (await client.SendAsync(upload)).StatusCode);
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> CreateProposalAsync(HttpClient client, string manifest, string correlationId, string idempotencyKey)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest(manifest, correlationId))
|
||||
};
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.Add("Idempotency-Key", idempotencyKey);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
}
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Security.Claims;
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Contracts.Admin;
|
||||
using ItWorx.KnowledgeHub.Contracts.Auth;
|
||||
using ItWorx.KnowledgeHub.Contracts.Automation;
|
||||
using ItWorx.KnowledgeHub.Domain.Automation;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>
|
||||
/// Covers the bounded Task 3 surface without registering proposal routes that Task 4
|
||||
/// owns. The policy probe invokes the application's registered authorization policies
|
||||
/// against the same claims emitted by the service-token handler.
|
||||
/// </summary>
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalAuthorizationTests : OwnerSetupTestBase
|
||||
{
|
||||
private readonly PostgresContainerFixture _postgres;
|
||||
|
||||
public IngestionProposalAuthorizationTests(PostgresContainerFixture postgres) : base(postgres)
|
||||
{
|
||||
_postgres = postgres;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ingestion_write_scope_satisfies_submission_but_not_human_review()
|
||||
{
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var authorization = scope.ServiceProvider.GetRequiredService<IAuthorizationService>();
|
||||
|
||||
var ingestingService = ServicePrincipal(Scopes.IngestionWrite);
|
||||
|
||||
Assert.True((await authorization.AuthorizeAsync(
|
||||
ingestingService, resource: null, AuthorizationPolicies.IngestionWrite)).Succeeded);
|
||||
Assert.False((await authorization.AuthorizeAsync(
|
||||
ingestingService, resource: null, AuthorizationPolicies.IngestionReview)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pages_write_scope_cannot_satisfy_ingestion_submission()
|
||||
{
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var authorization = scope.ServiceProvider.GetRequiredService<IAuthorizationService>();
|
||||
|
||||
var pagesService = ServicePrincipal(Scopes.PagesWrite);
|
||||
|
||||
Assert.False((await authorization.AuthorizeAsync(
|
||||
pagesService, resource: null, AuthorizationPolicies.IngestionWrite)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_delegated_principal_with_ingestion_write_can_submit_but_cannot_review()
|
||||
{
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var authorization = scope.ServiceProvider.GetRequiredService<IAuthorizationService>();
|
||||
|
||||
var delegatedIdentity = new ClaimsIdentity(
|
||||
[
|
||||
new Claim(ClaimTypes.NameIdentifier, Guid.NewGuid().ToString()),
|
||||
new Claim(ClaimTypes.Name, "delegated-human")
|
||||
], authenticationType: "delegated-cookie");
|
||||
var serviceIdentity = new ClaimsIdentity(
|
||||
[
|
||||
new Claim(ClaimTypes.NameIdentifier, Guid.NewGuid().ToString()),
|
||||
new Claim(ServiceTokenAuthenticationHandler.ServiceAccountClaimType, Guid.NewGuid().ToString()),
|
||||
new Claim(ServiceTokenAuthenticationHandler.ScopeClaimType, Scopes.IngestionWrite)
|
||||
], ServiceTokenAuthenticationDefaults.Scheme);
|
||||
|
||||
var delegatedService = new ClaimsPrincipal([delegatedIdentity, serviceIdentity]);
|
||||
|
||||
Assert.True((await authorization.AuthorizeAsync(
|
||||
delegatedService, resource: null, AuthorizationPolicies.IngestionWrite)).Succeeded);
|
||||
Assert.False((await authorization.AuthorizeAsync(
|
||||
delegatedService, resource: null, AuthorizationPolicies.IngestionReview)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Default_options_disable_ingestion_and_anonymous_session_has_no_features_or_workspace()
|
||||
{
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var options = scope.ServiceProvider.GetRequiredService<IOptions<IngestionProposalOptions>>().Value;
|
||||
Assert.False(options.Enabled);
|
||||
|
||||
using var client = Factory.CreateClient();
|
||||
var session = await client.GetFromJsonAsync<SessionResponse>("/api/v1/auth/session");
|
||||
|
||||
Assert.NotNull(session);
|
||||
Assert.False(session.Authenticated);
|
||||
Assert.Null(session.Features);
|
||||
Assert.Null(session.WorkspaceId);
|
||||
Assert.Null(session.Role);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Authenticated_default_configuration_session_exposes_disabled_ingestion_feature()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
var login = await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
login.EnsureSuccessStatusCode();
|
||||
var session = await login.Content.ReadFromJsonAsync<SessionResponse>();
|
||||
|
||||
Assert.NotNull(session);
|
||||
Assert.True(session.Authenticated);
|
||||
Assert.NotNull(session.Features);
|
||||
Assert.False(session.Features.IngestionProposals);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_authenticated_human_editor_can_submit_and_review()
|
||||
{
|
||||
using var administratorClient = Factory.CreateClient();
|
||||
var administratorLogin = await SetupHelper.LoginAsync(administratorClient, OwnerEmail, OwnerPassword);
|
||||
administratorLogin.EnsureSuccessStatusCode();
|
||||
|
||||
const string editorEmail = "ingestion-editor@itworx.test";
|
||||
const string editorPassword = "editor-strong-password";
|
||||
var createEditor = await SetupHelper.PostWithCsrfAsync(
|
||||
administratorClient,
|
||||
"/api/v1/admin/users",
|
||||
new CreateUserRequest("Ingestion Editor", editorEmail, editorPassword, "Editor"));
|
||||
createEditor.EnsureSuccessStatusCode();
|
||||
|
||||
using var editorClient = Factory.CreateClient();
|
||||
var editorLogin = await SetupHelper.LoginAsync(editorClient, editorEmail, editorPassword);
|
||||
editorLogin.EnsureSuccessStatusCode();
|
||||
var session = await editorLogin.Content.ReadFromJsonAsync<SessionResponse>();
|
||||
Assert.NotNull(session?.UserId);
|
||||
Assert.Equal("Editor", session.Role);
|
||||
|
||||
using var scope = Factory.Services.CreateScope();
|
||||
var accessor = scope.ServiceProvider.GetRequiredService<IHttpContextAccessor>();
|
||||
accessor.HttpContext = new DefaultHttpContext { User = HumanPrincipal(session.UserId.Value) };
|
||||
|
||||
try
|
||||
{
|
||||
var authorization = scope.ServiceProvider.GetRequiredService<IAuthorizationService>();
|
||||
var human = HumanPrincipal(session.UserId.Value);
|
||||
|
||||
Assert.True((await authorization.AuthorizeAsync(
|
||||
human, resource: null, AuthorizationPolicies.IngestionWrite)).Succeeded);
|
||||
Assert.True((await authorization.AuthorizeAsync(
|
||||
human, resource: null, AuthorizationPolicies.IngestionReview)).Succeeded);
|
||||
}
|
||||
finally
|
||||
{
|
||||
accessor.HttpContext = null;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Authenticated_session_exposes_only_the_server_configured_ingestion_feature()
|
||||
{
|
||||
var configuredFactory = new KnowledgeHubIntegrationFactory(_postgres,
|
||||
new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" });
|
||||
await ((IAsyncLifetime)configuredFactory).InitializeAsync();
|
||||
|
||||
try
|
||||
{
|
||||
using var client = configuredFactory.CreateClient();
|
||||
var setup = await SetupHelper.CompleteFirstRunSetupAsync(
|
||||
client, "Ingestion test", "Owner", "ingestion-owner@itworx.test", "correct-horse-battery-staple");
|
||||
setup.EnsureSuccessStatusCode();
|
||||
|
||||
var login = await SetupHelper.LoginAsync(client, "ingestion-owner@itworx.test", "correct-horse-battery-staple");
|
||||
login.EnsureSuccessStatusCode();
|
||||
var session = await login.Content.ReadFromJsonAsync<SessionResponse>();
|
||||
|
||||
Assert.NotNull(session);
|
||||
Assert.True(session.Authenticated);
|
||||
Assert.NotNull(session.Features);
|
||||
Assert.True(session.Features.IngestionProposals);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await ((IAsyncLifetime)configuredFactory).DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ingestion_write_is_an_accepted_service_account_scope()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
var login = await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
login.EnsureSuccessStatusCode();
|
||||
|
||||
var response = await SetupHelper.PostWithCsrfAsync(
|
||||
client,
|
||||
"/api/v1/service-accounts",
|
||||
new CreateServiceAccountRequest("ingestion", "trusted intake", [Scopes.IngestionWrite]));
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
private static ClaimsPrincipal ServicePrincipal(string scope)
|
||||
{
|
||||
var identity = new ClaimsIdentity(
|
||||
[
|
||||
new Claim(ClaimTypes.NameIdentifier, Guid.NewGuid().ToString()),
|
||||
new Claim(ServiceTokenAuthenticationHandler.ServiceAccountClaimType, Guid.NewGuid().ToString()),
|
||||
new Claim(ServiceTokenAuthenticationHandler.ScopeClaimType, scope)
|
||||
], ServiceTokenAuthenticationDefaults.Scheme);
|
||||
|
||||
return new ClaimsPrincipal(identity);
|
||||
}
|
||||
|
||||
private static ClaimsPrincipal HumanPrincipal(Guid userId) => new(new ClaimsIdentity(
|
||||
[
|
||||
new Claim(ClaimTypes.NameIdentifier, userId.ToString()),
|
||||
new Claim(ClaimTypes.Name, "human-editor")
|
||||
], authenticationType: "cookie"));
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalEtagContractTests(PostgresContainerFixture postgres)
|
||||
: OwnerSetupTestBase(postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
[Fact]
|
||||
public async Task Manual_create_upload_submit_uses_the_lowercase_etag_json_field()
|
||||
{
|
||||
using var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
|
||||
var createCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest(
|
||||
"JSON ETag document", Guid.NewGuid().ToString(),
|
||||
new IngestionPublicationRequest("ETag contract document", "Original document in attachment", "Reference", "Draft", "CreatePage")))
|
||||
};
|
||||
create.Headers.Add(createCsrf.HeaderName, createCsrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", Guid.NewGuid().ToString());
|
||||
using var created = await client.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
|
||||
var proposalId = await ReadProposalIdAsync(created);
|
||||
var createETag = await ReadFrontendETagAsync(created);
|
||||
|
||||
var uploadCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var file = new ByteArrayContent("Actual uploaded document bytes"u8.ToArray());
|
||||
file.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
multipart.Add(file, "file", "etag-document.txt");
|
||||
using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposalId}/sources")
|
||||
{
|
||||
Content = multipart
|
||||
};
|
||||
upload.Headers.Add(uploadCsrf.HeaderName, uploadCsrf.Value);
|
||||
upload.Headers.Add("Idempotency-Key", Guid.NewGuid().ToString());
|
||||
upload.Headers.TryAddWithoutValidation("If-Match", $"\"{createETag.Value}\"");
|
||||
|
||||
// The browser reads `etag` from JSON and sends that value as If-Match. If
|
||||
// the API only emits `eTag`, this request reproduces the observed 412.
|
||||
using var uploaded = await client.SendAsync(upload);
|
||||
Assert.Equal(HttpStatusCode.OK, uploaded.StatusCode);
|
||||
Assert.True(createETag.Found, "Create JSON must expose the frontend contract field `etag`.");
|
||||
var uploadETag = await ReadFrontendETagAsync(uploaded);
|
||||
Assert.True(uploadETag.Found, "Upload JSON must expose the frontend contract field `etag`.");
|
||||
|
||||
var submitCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var submit = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposalId}/submit")
|
||||
{
|
||||
Content = JsonContent.Create(new { })
|
||||
};
|
||||
submit.Headers.Add(submitCsrf.HeaderName, submitCsrf.Value);
|
||||
submit.Headers.TryAddWithoutValidation("If-Match", $"\"{uploadETag.Value}\"");
|
||||
using var submitted = await client.SendAsync(submit);
|
||||
Assert.Equal(HttpStatusCode.OK, submitted.StatusCode);
|
||||
Assert.True((await ReadFrontendETagAsync(submitted)).Found, "Submit JSON must expose the frontend contract field `etag`.");
|
||||
using var submittedJson = JsonDocument.Parse(await submitted.Content.ReadAsStringAsync());
|
||||
Assert.Equal("ReadyForReview", submittedJson.RootElement.GetProperty("status").GetString());
|
||||
}
|
||||
|
||||
private static async Task<(bool Found, string Value)> ReadFrontendETagAsync(HttpResponseMessage response)
|
||||
{
|
||||
using var json = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
if (json.RootElement.TryGetProperty("etag", out var etag) && etag.ValueKind == JsonValueKind.String
|
||||
&& !string.IsNullOrWhiteSpace(etag.GetString()))
|
||||
{
|
||||
return (true, etag.GetString()!);
|
||||
}
|
||||
|
||||
return (false, "undefined");
|
||||
}
|
||||
|
||||
private static async Task<Guid> ReadProposalIdAsync(HttpResponseMessage response)
|
||||
{
|
||||
using var json = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
return json.RootElement.GetProperty("id").GetGuid();
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
using ItWorx.KnowledgeHub.Application.Common;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Operations;
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalPersistenceTests(PostgresContainerFixture postgres)
|
||||
{
|
||||
private static readonly Guid WorkspaceId = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||||
private static readonly Guid ActorId = Guid.Parse("22222222-2222-2222-2222-222222222222");
|
||||
private const string Manifest = "# Proposed knowledge\n\nSafe manifest metadata.";
|
||||
private const string Digest = "c8c86d360bfb7bb0d036b7352bf89714946fb527c4074340752259109371f429";
|
||||
|
||||
[Fact]
|
||||
public async Task Proposal_preserves_digest_and_unique_key()
|
||||
{
|
||||
await using var target = await RestoreTargetDatabase.CreateAsync(postgres);
|
||||
var options = new DbContextOptionsBuilder<KnowledgeHubDbContext>()
|
||||
.UseNpgsql(target.ConnectionString, npgsql => npgsql.MigrationsAssembly(typeof(KnowledgeHubDbContext).Assembly.GetName().Name))
|
||||
.UseSnakeCaseNamingConvention()
|
||||
.Options;
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var repository = new IngestionProposalRepository(db);
|
||||
var proposal = IngestionProposal.Create(WorkspaceId, ActorId, "correlation-2", Manifest);
|
||||
proposal.AddSource("proposal.pdf", "application/pdf", 42, Digest, "clean", Guid.NewGuid());
|
||||
proposal.AssignIdempotencyKey("key-2");
|
||||
repository.Add(proposal);
|
||||
await repository.SaveChangesAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var repository = new IngestionProposalRepository(db);
|
||||
var roundTripped = await repository.FindByIdempotencyKeyAsync(WorkspaceId, "key-2", CancellationToken.None);
|
||||
|
||||
Assert.NotNull(roundTripped);
|
||||
var source = Assert.Single(roundTripped.Sources);
|
||||
Assert.Equal(Digest, source.DigestSha256);
|
||||
Assert.Equal("proposal.pdf", source.FileName);
|
||||
Assert.Equal("application/pdf", source.MimeType);
|
||||
Assert.Equal(42, source.ByteSize);
|
||||
Assert.Equal("clean", source.ScanResult);
|
||||
Assert.Single(roundTripped.Revisions);
|
||||
}
|
||||
|
||||
await Assert.ThrowsAsync<DbUpdateException>(async () =>
|
||||
{
|
||||
await using var db = new KnowledgeHubDbContext(options);
|
||||
var repository = new IngestionProposalRepository(db);
|
||||
var duplicate = IngestionProposal.Create(WorkspaceId, ActorId, "correlation-duplicate", Manifest);
|
||||
duplicate.AssignIdempotencyKey("key-2");
|
||||
repository.Add(duplicate);
|
||||
await repository.SaveChangesAsync(CancellationToken.None);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Repository_scopes_reads_etags_updates_and_bounded_lists_to_workspace()
|
||||
{
|
||||
await using var target = await RestoreTargetDatabase.CreateAsync(postgres);
|
||||
var options = new DbContextOptionsBuilder<KnowledgeHubDbContext>()
|
||||
.UseNpgsql(target.ConnectionString, npgsql => npgsql.MigrationsAssembly(typeof(KnowledgeHubDbContext).Assembly.GetName().Name))
|
||||
.UseSnakeCaseNamingConvention()
|
||||
.Options;
|
||||
var otherWorkspaceId = Guid.NewGuid();
|
||||
Guid proposalId;
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var repository = new IngestionProposalRepository(db);
|
||||
var first = IngestionProposal.Create(WorkspaceId, ActorId, "first", Manifest);
|
||||
first.AssignIdempotencyKey("queue-1");
|
||||
repository.Add(first);
|
||||
var second = IngestionProposal.Create(WorkspaceId, ActorId, "second", Manifest);
|
||||
second.AssignIdempotencyKey("queue-2");
|
||||
repository.Add(second);
|
||||
var other = IngestionProposal.Create(otherWorkspaceId, ActorId, "other", Manifest);
|
||||
other.AssignIdempotencyKey("other-1");
|
||||
repository.Add(other);
|
||||
await repository.SaveChangesAsync(CancellationToken.None);
|
||||
proposalId = first.Id;
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var repository = new IngestionProposalRepository(db);
|
||||
var page = await repository.ListAsync(new IngestionProposalQuery(WorkspaceId, Limit: 1), CancellationToken.None);
|
||||
|
||||
Assert.Single(page.Items);
|
||||
Assert.NotNull(page.NextCursor);
|
||||
Assert.Null(await repository.FindAsync(otherWorkspaceId, proposalId, CancellationToken.None));
|
||||
Assert.Null(await repository.GetETagAsync(otherWorkspaceId, proposalId, CancellationToken.None));
|
||||
Assert.Null(await repository.FindForUpdateAsync(otherWorkspaceId, proposalId, "0", CancellationToken.None));
|
||||
Assert.DoesNotContain((await repository.ListAsync(new IngestionProposalQuery(otherWorkspaceId, Limit: 100), CancellationToken.None)).Items,
|
||||
proposal => proposal.WorkspaceId == WorkspaceId);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Repository_translates_a_stale_xmin_to_an_application_concurrency_conflict()
|
||||
{
|
||||
await using var target = await RestoreTargetDatabase.CreateAsync(postgres);
|
||||
var options = new DbContextOptionsBuilder<KnowledgeHubDbContext>()
|
||||
.UseNpgsql(target.ConnectionString, npgsql => npgsql.MigrationsAssembly(typeof(KnowledgeHubDbContext).Assembly.GetName().Name))
|
||||
.UseSnakeCaseNamingConvention()
|
||||
.Options;
|
||||
Guid proposalId;
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var repository = new IngestionProposalRepository(db);
|
||||
var proposal = IngestionProposal.Create(WorkspaceId, ActorId, "stale-xmin", Manifest);
|
||||
proposal.AssignIdempotencyKey("stale-xmin-key");
|
||||
repository.Add(proposal);
|
||||
await repository.SaveChangesAsync(CancellationToken.None);
|
||||
proposalId = proposal.Id;
|
||||
}
|
||||
|
||||
await using var staleDb = new KnowledgeHubDbContext(options);
|
||||
var staleRepository = new IngestionProposalRepository(staleDb);
|
||||
var staleEtag = await staleRepository.GetETagAsync(WorkspaceId, proposalId, CancellationToken.None);
|
||||
|
||||
await using (var currentDb = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var currentRepository = new IngestionProposalRepository(currentDb);
|
||||
var currentEtag = await currentRepository.GetETagAsync(WorkspaceId, proposalId, CancellationToken.None);
|
||||
var current = await currentRepository.FindForUpdateAsync(WorkspaceId, proposalId, currentEtag!, CancellationToken.None);
|
||||
Assert.NotNull(current);
|
||||
current.AddSource("current-source.pdf", "application/pdf", 42, Digest, "clean", Guid.NewGuid());
|
||||
current.BeginAnalysis(ActorId, DateTimeOffset.UtcNow);
|
||||
await currentRepository.SaveChangesAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
await using (var verificationDb = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var verificationRepository = new IngestionProposalRepository(verificationDb);
|
||||
var updated = await verificationRepository.FindAsync(WorkspaceId, proposalId, CancellationToken.None);
|
||||
Assert.NotNull(updated);
|
||||
Assert.Equal(IngestionProposalStatus.Analyzing, updated.Status);
|
||||
Assert.Contains(updated.Revisions, revision => revision.Operation == "analysis-started");
|
||||
Assert.Contains(updated.Sources, source => source.FileName == "current-source.pdf");
|
||||
}
|
||||
|
||||
var stale = await staleRepository.FindForUpdateAsync(WorkspaceId, proposalId, staleEtag!, CancellationToken.None);
|
||||
Assert.NotNull(stale);
|
||||
stale.ApplyAnalysis("stale analysis", ActorId, DateTimeOffset.UtcNow);
|
||||
|
||||
var conflict = await Assert.ThrowsAsync<ConcurrencyConflictException>(
|
||||
() => staleRepository.SaveChangesAsync(CancellationToken.None));
|
||||
Assert.Equal(proposalId, conflict.EntityId);
|
||||
Assert.Empty(staleDb.ChangeTracker.Entries());
|
||||
}
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using ItWorx.KnowledgeHub.Contracts.Admin;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionProposalPrivacyTests(PostgresContainerFixture postgres)
|
||||
: OwnerSetupTestBase(postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
private const string EditorPassword = "strong-editor-password";
|
||||
private const string ReaderPassword = "strong-reader-password";
|
||||
|
||||
[Fact]
|
||||
public async Task Proposal_details_changes_and_source_bytes_follow_submitter_and_review_roles()
|
||||
{
|
||||
using var administrator = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(administrator, OwnerEmail, OwnerPassword);
|
||||
using var submitter = await CreateUserAsync(administrator, "indiener@itworx.test", "Editor", EditorPassword);
|
||||
using var otherEditor = await CreateUserAsync(administrator, "reviewer@itworx.test", "Editor", EditorPassword);
|
||||
using var reader = await CreateUserAsync(administrator, "lezer@itworx.test", "Reader", ReaderPassword);
|
||||
|
||||
var created = await CreateProposalAsync(submitter);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, (await GetProposalAsync(submitter, proposal.Id)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.OK, (await GetProposalAsync(administrator, proposal.Id)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await GetProposalAsync(otherEditor, proposal.Id)).StatusCode);
|
||||
var submitterReceivedQueue = await submitter.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=Received");
|
||||
Assert.Equal(proposal.Id, Assert.Single(submitterReceivedQueue!.Items).Id);
|
||||
var administratorReceivedQueue = await administrator.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=Received");
|
||||
Assert.Equal(proposal.Id, Assert.Single(administratorReceivedQueue!.Items).Id);
|
||||
var foreignReceivedQueue = await otherEditor.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=Received");
|
||||
Assert.Empty(foreignReceivedQueue!.Items);
|
||||
Assert.Equal(0, foreignReceivedQueue.Total);
|
||||
|
||||
using (var foreignPatch = await PatchProposalAsync(otherEditor, proposal.Id, created.Headers.ETag!.Tag, "not allowed"))
|
||||
Assert.Equal(HttpStatusCode.NotFound, foreignPatch.StatusCode);
|
||||
|
||||
using var ownerPatch = await PatchProposalAsync(submitter, proposal.Id, created.Headers.ETag.Tag, "owner edit");
|
||||
Assert.Equal(HttpStatusCode.OK, ownerPatch.StatusCode);
|
||||
using var adminPatch = await PatchProposalAsync(administrator, proposal.Id, ownerPatch.Headers.ETag!.Tag, "admin edit");
|
||||
Assert.Equal(HttpStatusCode.OK, adminPatch.StatusCode);
|
||||
|
||||
var staged = await UploadSourceAsync(submitter, proposal.Id, adminPatch.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, staged.StatusCode);
|
||||
var stagedDetail = await staged.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
var source = Assert.Single(stagedDetail!.Sources);
|
||||
var sourceUrl = $"/api/v1/attachments/{source.AttachmentReferenceId}/content";
|
||||
|
||||
// A received proposal and its original bytes remain private to the submitter
|
||||
// and administrators, even from another workspace Editor.
|
||||
Assert.Equal(HttpStatusCode.OK, (await submitter.GetAsync(sourceUrl)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.OK, (await administrator.GetAsync(sourceUrl)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await otherEditor.GetAsync(sourceUrl)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await reader.GetAsync(sourceUrl)).StatusCode);
|
||||
|
||||
using var submitted = await SubmitAsync(submitter, proposal.Id, staged.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, submitted.StatusCode);
|
||||
var ready = await submitted.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal("ReadyForReview", ready?.Status);
|
||||
|
||||
// During review, workspace Editors retain the established reviewer access.
|
||||
Assert.Equal(HttpStatusCode.OK, (await GetProposalAsync(otherEditor, proposal.Id)).StatusCode);
|
||||
var reviewerQueue = await otherEditor.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=ReadyForReview");
|
||||
Assert.Contains(reviewerQueue!.Items, item => item.Id == proposal.Id);
|
||||
Assert.Equal(HttpStatusCode.OK, (await otherEditor.GetAsync(sourceUrl)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await reader.GetAsync(sourceUrl)).StatusCode);
|
||||
using var reviewerPatch = await PatchProposalAsync(otherEditor, proposal.Id, submitted.Headers.ETag!.Tag, "reviewer edit");
|
||||
Assert.Equal(HttpStatusCode.OK, reviewerPatch.StatusCode);
|
||||
|
||||
using var changesRequested = await RequestChangesAsync(otherEditor, proposal.Id, reviewerPatch.Headers.ETag!.Tag);
|
||||
Assert.Equal(HttpStatusCode.OK, changesRequested.StatusCode);
|
||||
var changed = await changesRequested.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal("ChangesRequested", changed?.Status);
|
||||
|
||||
// Once returned for changes, proposal metadata and source bytes are private
|
||||
// to the submitter and administrators again.
|
||||
Assert.Equal(HttpStatusCode.OK, (await GetProposalAsync(submitter, proposal.Id)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.OK, (await GetProposalAsync(administrator, proposal.Id)).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await GetProposalAsync(otherEditor, proposal.Id)).StatusCode);
|
||||
var submitterChangesQueue = await submitter.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=ChangesRequested");
|
||||
Assert.Contains(submitterChangesQueue!.Items, item => item.Id == proposal.Id);
|
||||
var administratorChangesQueue = await administrator.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=ChangesRequested");
|
||||
Assert.Contains(administratorChangesQueue!.Items, item => item.Id == proposal.Id);
|
||||
var foreignChangesQueue = await otherEditor.GetFromJsonAsync<IngestionProposalListResponse>("/api/v1/ingestion-proposals?status=ChangesRequested");
|
||||
Assert.DoesNotContain(foreignChangesQueue!.Items, item => item.Id == proposal.Id);
|
||||
Assert.Equal(0, foreignChangesQueue.Total);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await otherEditor.GetAsync(sourceUrl)).StatusCode);
|
||||
using (var foreignChangesPatch = await PatchProposalAsync(otherEditor, proposal.Id, changesRequested.Headers.ETag!.Tag, "not allowed"))
|
||||
Assert.Equal(HttpStatusCode.NotFound, foreignChangesPatch.StatusCode);
|
||||
|
||||
using var submitterChangesPatch = await PatchProposalAsync(submitter, proposal.Id, changesRequested.Headers.ETag.Tag, "submitter update");
|
||||
Assert.Equal(HttpStatusCode.OK, submitterChangesPatch.StatusCode);
|
||||
using var adminChangesPatch = await PatchProposalAsync(administrator, proposal.Id, submitterChangesPatch.Headers.ETag!.Tag, "admin update");
|
||||
Assert.Equal(HttpStatusCode.OK, adminChangesPatch.StatusCode);
|
||||
Assert.Equal(HttpStatusCode.OK, (await administrator.GetAsync(sourceUrl)).StatusCode);
|
||||
}
|
||||
|
||||
private async Task<HttpClient> CreateUserAsync(HttpClient administrator, string email, string role, string password)
|
||||
{
|
||||
var response = await SetupHelper.PostWithCsrfAsync(administrator, "/api/v1/admin/users",
|
||||
new CreateUserRequest(email, email, password, role));
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
var client = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(client, email, password);
|
||||
return client;
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> CreateProposalAsync(HttpClient client)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest(
|
||||
"Private source manifest", Guid.NewGuid().ToString(),
|
||||
new IngestionPublicationRequest("Private source", "Source summary", "Reference", "Draft", "CreatePage")))
|
||||
};
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.Add("Idempotency-Key", Guid.NewGuid().ToString());
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static Task<HttpResponseMessage> GetProposalAsync(HttpClient client, Guid proposalId) =>
|
||||
client.GetAsync($"/api/v1/ingestion-proposals/{proposalId}");
|
||||
|
||||
private static async Task<HttpResponseMessage> PatchProposalAsync(HttpClient client, Guid proposalId, string etag, string manifest)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposalId}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest(manifest, null))
|
||||
};
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.TryAddWithoutValidation("If-Match", etag);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> UploadSourceAsync(HttpClient client, Guid proposalId, string etag)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var form = new MultipartFormDataContent();
|
||||
var source = new ByteArrayContent("unpublished document bytes"u8.ToArray());
|
||||
source.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
|
||||
form.Add(source, "file", "private-source.txt");
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposalId}/sources")
|
||||
{
|
||||
Content = form
|
||||
};
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.Add("Idempotency-Key", Guid.NewGuid().ToString());
|
||||
request.Headers.TryAddWithoutValidation("If-Match", etag);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> SubmitAsync(HttpClient client, Guid proposalId, string etag)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposalId}/submit")
|
||||
{
|
||||
Content = JsonContent.Create(new { })
|
||||
};
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.TryAddWithoutValidation("If-Match", etag);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> RequestChangesAsync(HttpClient client, Guid proposalId, string etag)
|
||||
{
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposalId}/request-changes")
|
||||
{
|
||||
Content = JsonContent.Create(new IngestionProposalDecisionRequest("Please revise."))
|
||||
};
|
||||
request.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
request.Headers.TryAddWithoutValidation("If-Match", etag);
|
||||
return await client.SendAsync(request);
|
||||
}
|
||||
}
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
using System.Security.Cryptography;
|
||||
using ItWorx.KnowledgeHub.Application.Attachments;
|
||||
using ItWorx.KnowledgeHub.Application.Attachments.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
using ItWorx.KnowledgeHub.Domain.Attachments;
|
||||
using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>
|
||||
/// Does not use the PostgreSQL fixture: the repository port deliberately provides a
|
||||
/// controllable save-failure seam around the real upload pipeline. It models the
|
||||
/// database rollback that happens when the shared DbContext save fails after the
|
||||
/// physical blob commit.
|
||||
/// </summary>
|
||||
public sealed class IngestionProposalSourceCompensationTests
|
||||
{
|
||||
private static readonly Guid WorkspaceId = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||||
private static readonly Guid ActorId = Guid.Parse("22222222-2222-2222-2222-222222222222");
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_shared_database_save_after_a_new_blob_commit_compensates_every_staged_artifact()
|
||||
{
|
||||
var proposal = IngestionProposal.Create(WorkspaceId, ActorId, "compensation", "safe manifest");
|
||||
var attachments = new TransactionalAttachmentRepository();
|
||||
var audits = new TransactionalAuditRecorder();
|
||||
var proposals = new FailingSaveIngestionProposalRepository(proposal, attachments, audits);
|
||||
var blobs = new MemoryBlobStore();
|
||||
var uploads = new UploadAttachmentUseCase(
|
||||
pages: null!,
|
||||
stagingSessions: null!,
|
||||
ingestionProposals: proposals,
|
||||
projects: null!,
|
||||
attachments,
|
||||
blobs,
|
||||
new PlainTextSignatureDetector(),
|
||||
audits,
|
||||
new AttachmentContentLock(),
|
||||
TimeProvider.System);
|
||||
var useCase = new StageIngestionProposalSourceUseCase(
|
||||
proposals,
|
||||
attachments,
|
||||
blobs,
|
||||
uploads,
|
||||
audits,
|
||||
new AttachmentContentLock(),
|
||||
TimeProvider.System);
|
||||
var content = "post-blob save failure"u8.ToArray();
|
||||
var digest = Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant();
|
||||
|
||||
await using var stream = new MemoryStream(content);
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => useCase.ExecuteAsync(
|
||||
new StageIngestionProposalSourceCommand(
|
||||
WorkspaceId,
|
||||
proposal.Id,
|
||||
ActorId,
|
||||
"evidence.txt",
|
||||
"text/plain",
|
||||
stream,
|
||||
"compensation-test",
|
||||
MaximumSourceCount: 5,
|
||||
ExpectedETag: "1",
|
||||
AuditActorKind.ServiceAccount,
|
||||
AuditSource: "api"),
|
||||
CancellationToken.None));
|
||||
|
||||
Assert.Equal(1, proposals.SaveAttempts);
|
||||
Assert.Equal(1, proposals.PendingSourceCountAtFailedSave);
|
||||
Assert.Equal(0, proposals.PersistedSourceCount);
|
||||
Assert.Empty(attachments.Blobs);
|
||||
Assert.Empty(attachments.References);
|
||||
Assert.Empty(attachments.Usages);
|
||||
Assert.Empty(audits.Events);
|
||||
Assert.False(blobs.Exists(digest));
|
||||
Assert.Equal([digest], blobs.DeletedHashes);
|
||||
}
|
||||
|
||||
private sealed class FailingSaveIngestionProposalRepository(
|
||||
IngestionProposal proposal,
|
||||
TransactionalAttachmentRepository attachments,
|
||||
TransactionalAuditRecorder audits) : IIngestionProposalRepository
|
||||
{
|
||||
public int SaveAttempts { get; private set; }
|
||||
|
||||
public int PendingSourceCountAtFailedSave { get; private set; }
|
||||
|
||||
// This is the durable snapshot; it changes only after a successful shared
|
||||
// save, exactly like a fresh DbContext query after the database rollback.
|
||||
public int PersistedSourceCount { get; private set; }
|
||||
|
||||
public Task<IngestionProposalPage> ListAsync(IngestionProposalQuery query, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new IngestionProposalPage([], null));
|
||||
|
||||
public Task<IngestionProposal?> FindAsync(Guid workspaceId, Guid proposalId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IngestionProposal?>(workspaceId == proposal.WorkspaceId && proposalId == proposal.Id ? proposal : null);
|
||||
|
||||
public Task<IngestionProposal?> FindByIdempotencyKeyAsync(Guid workspaceId, string idempotencyKey, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IngestionProposal?>(null);
|
||||
|
||||
public Task<string?> GetETagAsync(Guid workspaceId, Guid proposalId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<string?>(workspaceId == proposal.WorkspaceId && proposalId == proposal.Id ? "1" : null);
|
||||
|
||||
public Task<IngestionProposal?> FindForUpdateAsync(Guid workspaceId, Guid proposalId, string expectedETag, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IngestionProposal?>(workspaceId == proposal.WorkspaceId && proposalId == proposal.Id && expectedETag == "1" ? proposal : null);
|
||||
|
||||
public void Add(IngestionProposal item) => throw new NotSupportedException();
|
||||
|
||||
public Task<int> SaveChangesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
SaveAttempts += 1;
|
||||
PendingSourceCountAtFailedSave = proposal.Sources.Count;
|
||||
attachments.RollBack();
|
||||
audits.RollBack();
|
||||
return Task.FromException<int>(new InvalidOperationException("Forced shared DbContext save failure."));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TransactionalAttachmentRepository : IAttachmentRepository
|
||||
{
|
||||
private readonly List<AttachmentBlob> blobs = [];
|
||||
private readonly List<AttachmentReference> references = [];
|
||||
private readonly List<AttachmentUsage> usages = [];
|
||||
|
||||
public IReadOnlyList<AttachmentBlob> Blobs => blobs;
|
||||
public IReadOnlyList<AttachmentReference> References => references;
|
||||
public IReadOnlyList<AttachmentUsage> Usages => usages;
|
||||
|
||||
public Task<AttachmentBlob?> FindBlobAsync(string sha256, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(blobs.SingleOrDefault(blob => blob.Sha256 == sha256));
|
||||
|
||||
public void AddBlob(AttachmentBlob blob) => blobs.Add(blob);
|
||||
|
||||
public Task<AttachmentReference?> FindReferenceAsync(Guid referenceId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(references.SingleOrDefault(reference => reference.Id == referenceId));
|
||||
|
||||
public void AddReference(AttachmentReference reference) => references.Add(reference);
|
||||
|
||||
public void AddUsage(AttachmentUsage usage) => usages.Add(usage);
|
||||
|
||||
public Task<IReadOnlyList<AttachmentReference>> GetReferencesForPageAsync(Guid pageId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyList<AttachmentReference>>([]);
|
||||
|
||||
public Task<IReadOnlyDictionary<string, AttachmentBlob>> FindBlobsAsync(IReadOnlyCollection<string> sha256Hashes, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyDictionary<string, AttachmentBlob>>(new Dictionary<string, AttachmentBlob>());
|
||||
|
||||
public Task<AttachmentUsage?> FindUsageForReferenceAsync(Guid attachmentReferenceId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(usages.SingleOrDefault(usage => usage.AttachmentReferenceId == attachmentReferenceId));
|
||||
|
||||
public Task<IReadOnlyList<string>> FindUnreferencedBlobHashesAsync(IReadOnlyCollection<string> candidateHashes, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyList<string>>(candidateHashes.Where(hash => references.All(reference => reference.BlobSha256 != hash)).ToArray());
|
||||
|
||||
public Task<int> RemoveBlobRecordsAsync(IReadOnlyCollection<string> sha256Hashes, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(0);
|
||||
|
||||
public Task<IReadOnlyList<AttachmentUsage>> GetUsagesForStagingSessionAsync(Guid stagingSessionId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyList<AttachmentUsage>>([]);
|
||||
|
||||
public Task TransferProposalUsagesToPageAsync(Guid ingestionProposalId, Guid pageId, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var usage in usages.Where(usage => usage.IngestionProposalId == ingestionProposalId))
|
||||
{
|
||||
usage.TransferProposalToPage(pageId);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<StagingAttachmentCleanupResult> RemoveOrphanedStagingAttachmentsAsync(Guid stagingSessionId, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(StagingAttachmentCleanupResult.None);
|
||||
|
||||
public Task SaveChangesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void RollBack()
|
||||
{
|
||||
blobs.Clear();
|
||||
references.Clear();
|
||||
usages.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TransactionalAuditRecorder : IAuditEventRecorder
|
||||
{
|
||||
private readonly List<AuditEventRecord> events = [];
|
||||
|
||||
public IReadOnlyList<AuditEventRecord> Events => events;
|
||||
|
||||
public void Add(AuditEventRecord auditEvent) => events.Add(auditEvent);
|
||||
|
||||
public Task RecordAsync(AuditEventRecord auditEvent, CancellationToken cancellationToken)
|
||||
{
|
||||
events.Add(auditEvent);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void RollBack() => events.Clear();
|
||||
}
|
||||
|
||||
private sealed class MemoryBlobStore : IAttachmentBlobStore
|
||||
{
|
||||
private readonly Dictionary<string, byte[]> blobs = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, byte[]> quarantined = new(StringComparer.Ordinal);
|
||||
|
||||
public IReadOnlyList<string> DeletedHashes => deletedHashes;
|
||||
|
||||
private readonly List<string> deletedHashes = [];
|
||||
|
||||
public bool Exists(string sha256) => blobs.ContainsKey(sha256);
|
||||
|
||||
public async Task<QuarantinedUpload> WriteToQuarantineAsync(Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
using var bytes = new MemoryStream();
|
||||
await content.CopyToAsync(bytes, cancellationToken);
|
||||
var value = bytes.ToArray();
|
||||
var sha256 = Convert.ToHexString(SHA256.HashData(value)).ToLowerInvariant();
|
||||
var token = Guid.NewGuid().ToString("N");
|
||||
quarantined.Add(token, value);
|
||||
return new QuarantinedUpload(token, sha256, value.LongLength);
|
||||
}
|
||||
|
||||
public Task<byte[]> ReadQuarantinePrefixAsync(string quarantineToken, int maxBytes, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(quarantined[quarantineToken].Take(maxBytes).ToArray());
|
||||
|
||||
public Task<bool> BlobExistsAsync(string sha256, CancellationToken cancellationToken) => Task.FromResult(Exists(sha256));
|
||||
|
||||
public Task CommitAsync(string quarantineToken, string sha256, CancellationToken cancellationToken)
|
||||
{
|
||||
blobs.Add(sha256, quarantined[quarantineToken]);
|
||||
quarantined.Remove(quarantineToken);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DiscardQuarantineAsync(string quarantineToken, CancellationToken cancellationToken)
|
||||
{
|
||||
quarantined.Remove(quarantineToken);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<Stream> OpenReadAsync(string sha256, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<Stream>(new MemoryStream(blobs[sha256], writable: false));
|
||||
|
||||
public Task<bool> DeleteVerifiedAsync(string sha256, CancellationToken cancellationToken)
|
||||
{
|
||||
var removed = blobs.Remove(sha256);
|
||||
if (removed)
|
||||
{
|
||||
deletedHashes.Add(sha256);
|
||||
}
|
||||
|
||||
return Task.FromResult(removed);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PlainTextSignatureDetector : IContentSignatureDetector
|
||||
{
|
||||
public SignatureDetectionResult Detect(ReadOnlySpan<byte> prefix, string? declaredContentType, string? extension) =>
|
||||
new("text/plain", AttachmentDeliveryPolicy.DownloadOnly, DeclaredTypeMismatch: false);
|
||||
}
|
||||
}
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Attachments;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Automation;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Contracts.Automation;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Domain.Attachments;
|
||||
using ItWorx.KnowledgeHub.Domain.Automation;
|
||||
using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Domain.Knowledge;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>Atomic-publication use-case tests using ports that model one shared
|
||||
/// DbContext. They are deliberately Docker-free; HTTP authorization stays covered by
|
||||
/// the PostgreSQL API suite.</summary>
|
||||
public sealed class IngestionPublicationTests
|
||||
{
|
||||
private static readonly Guid WorkspaceId = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||||
private static readonly Guid ActorId = Guid.Parse("22222222-2222-2222-2222-222222222222");
|
||||
|
||||
[Fact]
|
||||
public async Task Create_publication_stages_page_revision_attachment_audit_and_outbox_in_one_save()
|
||||
{
|
||||
var fixture = Fixture.Create();
|
||||
var result = await fixture.PublishAsync();
|
||||
|
||||
Assert.Equal(IngestionProposalResultStatus.Success, result.Status);
|
||||
var page = Assert.Single(fixture.Pages.Added);
|
||||
Assert.Equal("Trusted article", page.TitleValue);
|
||||
Assert.Equal(DocumentStatus.Active, page.DocumentStatus);
|
||||
Assert.Equal(1, page.CurrentRevisionNumber);
|
||||
Assert.Equal(page.Id, Assert.Single(fixture.Pages.Revisions).PageId);
|
||||
Assert.Equal(page.Id, fixture.Attachments.TransferredPageId);
|
||||
Assert.Equal("ingestion-proposal.published", Assert.Single(fixture.Audits.Events).Action);
|
||||
Assert.Equal("ingestion-proposal.published.v1", Assert.Single(fixture.Outbox.Messages).EventType);
|
||||
Assert.Equal(IngestionProposalStatus.Published, fixture.Proposal.Status);
|
||||
Assert.Equal(page.Id, fixture.Proposal.PublishedPageId);
|
||||
Assert.Equal(1, fixture.Proposals.SaveCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Stale_update_returns_proposal_to_review_without_page_attachment_audit_or_outbox_side_effects()
|
||||
{
|
||||
var fixture = Fixture.Update("newer-target-etag");
|
||||
var result = await fixture.PublishAsync();
|
||||
|
||||
Assert.Equal(IngestionProposalResultStatus.StaleTarget, result.Status);
|
||||
Assert.Equal(IngestionProposalStatus.ReadyForReview, fixture.Proposal.Status);
|
||||
Assert.Empty(fixture.Pages.Added);
|
||||
Assert.Empty(fixture.Pages.Revisions);
|
||||
Assert.Null(fixture.Attachments.TransferredPageId);
|
||||
Assert.Empty(fixture.Audits.Events);
|
||||
Assert.Empty(fixture.Outbox.Messages);
|
||||
Assert.Equal(1, fixture.Proposals.SaveCalls); // ReadyForReview only.
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Replay_with_same_key_is_exactly_once_even_when_original_etag_is_retried()
|
||||
{
|
||||
var fixture = Fixture.Create();
|
||||
var first = await fixture.PublishAsync();
|
||||
var replay = await fixture.PublishAsync();
|
||||
|
||||
Assert.Equal(IngestionProposalResultStatus.Success, first.Status);
|
||||
Assert.Equal(IngestionProposalResultStatus.Success, replay.Status);
|
||||
Assert.Equal(first.PageId, replay.PageId);
|
||||
Assert.Single(fixture.Pages.Added);
|
||||
Assert.Single(fixture.Pages.Revisions);
|
||||
Assert.Single(fixture.Outbox.Messages);
|
||||
Assert.Equal(1, fixture.Proposals.SaveCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Shared_save_failure_rolls_back_every_staged_side_effect()
|
||||
{
|
||||
var fixture = Fixture.Create(failSave: true);
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => fixture.PublishAsync());
|
||||
|
||||
Assert.Empty(fixture.Pages.Added);
|
||||
Assert.Empty(fixture.Pages.Revisions);
|
||||
Assert.Null(fixture.Attachments.TransferredPageId);
|
||||
Assert.Empty(fixture.Audits.Events);
|
||||
Assert.Empty(fixture.Outbox.Messages);
|
||||
}
|
||||
|
||||
private sealed class Fixture
|
||||
{
|
||||
private const string ProposalETag = "proposal-etag";
|
||||
private const string PublishKey = "publish-once";
|
||||
private Fixture(IngestionProposal proposal, Proposals proposals, Pages pages, Attachments attachments, Audits audits, Outbox outbox)
|
||||
=> (Proposal, Proposals, Pages, Attachments, Audits, Outbox) = (proposal, proposals, pages, attachments, audits, outbox);
|
||||
public IngestionProposal Proposal { get; }
|
||||
public Proposals Proposals { get; }
|
||||
public Pages Pages { get; }
|
||||
public Attachments Attachments { get; }
|
||||
public Audits Audits { get; }
|
||||
public Outbox Outbox { get; }
|
||||
|
||||
public static Fixture Create(bool failSave = false) => Build(Approved(IngestionMutationKind.CreatePage), new Pages(), failSave);
|
||||
|
||||
public static Fixture Update(string targetEtag)
|
||||
{
|
||||
var target = Page.Create(WorkspaceId, null, SortKey.First(), PageTitle.Create("Existing"), DocumentType.WikiPage,
|
||||
"original", ActorId, DateTimeOffset.UtcNow);
|
||||
return Build(Approved(IngestionMutationKind.UpdatePage, target.Id), new Pages(target, targetEtag));
|
||||
}
|
||||
|
||||
public Task<PublishIngestionProposalResult> PublishAsync() => new PublishIngestionProposalUseCase(
|
||||
Proposals, Pages, new LinkResolutionService(Pages), Attachments, Audits, Outbox, TimeProvider.System).ExecuteAsync(
|
||||
new(WorkspaceId, Proposal.Id, ProposalETag, PublishKey, ActorId, "publication-test"), CancellationToken.None);
|
||||
|
||||
private static Fixture Build(IngestionProposal proposal, Pages pages, bool failSave = false)
|
||||
{
|
||||
var attachments = new Attachments(); var audits = new Audits(); var outbox = new Outbox();
|
||||
var proposals = new Proposals(proposal, failSave, () => { pages.RollBack(); attachments.RollBack(); audits.RollBack(); outbox.RollBack(); });
|
||||
return new Fixture(proposal, proposals, pages, attachments, audits, outbox);
|
||||
}
|
||||
|
||||
private static IngestionProposal Approved(IngestionMutationKind kind, Guid? targetPageId = null)
|
||||
{
|
||||
var proposal = IngestionProposal.Create(WorkspaceId, ActorId, "publication", "safe manifest");
|
||||
proposal.SetPublication(IngestionPublication.Create(kind, "Trusted article", "# Published body", "WikiPage", "Active",
|
||||
targetPageId: targetPageId, expectedTargetETag: kind == IngestionMutationKind.UpdatePage ? "target-etag" : null), ActorId, DateTimeOffset.UtcNow);
|
||||
proposal.AddSource("evidence.txt", "text/plain", 1, "c8c86d360bfb7bb0d036b7352bf89714946fb527c4074340752259109371f429", "clean", Guid.NewGuid());
|
||||
proposal.BeginAnalysis(ActorId, DateTimeOffset.UtcNow);
|
||||
proposal.ApplyAnalysis("Reviewed analysis", ActorId, DateTimeOffset.UtcNow);
|
||||
proposal.Approve(ActorId, DateTimeOffset.UtcNow);
|
||||
return proposal;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Proposals(IngestionProposal proposal, bool failSave, Action rollback) : IIngestionProposalRepository
|
||||
{
|
||||
private string etag = "proposal-etag";
|
||||
public int SaveCalls { get; private set; }
|
||||
public Task<IngestionProposalPage> ListAsync(IngestionProposalQuery query, CancellationToken cancellationToken) => Task.FromResult(new IngestionProposalPage([], null));
|
||||
public Task<IngestionProposal?> FindAsync(Guid workspaceId, Guid proposalId, CancellationToken cancellationToken) => Task.FromResult<IngestionProposal?>(workspaceId == proposal.WorkspaceId && proposalId == proposal.Id ? proposal : null);
|
||||
public Task<IngestionProposal?> FindByIdempotencyKeyAsync(Guid workspaceId, string idempotencyKey, CancellationToken cancellationToken) => Task.FromResult<IngestionProposal?>(null);
|
||||
public Task<string?> GetETagAsync(Guid workspaceId, Guid proposalId, CancellationToken cancellationToken) => Task.FromResult<string?>(workspaceId == proposal.WorkspaceId && proposalId == proposal.Id ? etag : null);
|
||||
public Task<IngestionProposal?> FindForUpdateAsync(Guid workspaceId, Guid proposalId, string expectedETag, CancellationToken cancellationToken) => Task.FromResult<IngestionProposal?>(workspaceId == proposal.WorkspaceId && proposalId == proposal.Id && expectedETag == etag ? proposal : null);
|
||||
public void Add(IngestionProposal item) => throw new NotSupportedException();
|
||||
public Task<int> SaveChangesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
SaveCalls++;
|
||||
if (failSave) { rollback(); return Task.FromException<int>(new InvalidOperationException("Forced shared DbContext save failure.")); }
|
||||
etag = "proposal-published-etag";
|
||||
return Task.FromResult(1);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Pages(Page? target = null, string? targetEtag = null) : IPageRepository
|
||||
{
|
||||
public List<Page> Added { get; } = [];
|
||||
public List<PageRevision> Revisions { get; } = [];
|
||||
public Task<Page?> FindAsync(Guid pageId, CancellationToken cancellationToken) => Task.FromResult(target?.Id == pageId ? target : null);
|
||||
public Task<string?> GetETagAsync(Guid pageId, CancellationToken cancellationToken) => Task.FromResult(target?.Id == pageId ? targetEtag : null);
|
||||
public Task<Page?> FindForUpdateAsync(Guid pageId, string expectedETag, CancellationToken cancellationToken) => Task.FromResult(target?.Id == pageId && targetEtag == expectedETag ? target : null);
|
||||
public Task<PageChildrenPage> GetChildrenAsync(Guid workspaceId, Guid? parentPageId, bool includeTrashed, string? cursor, int limit, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<RagPublicationManifestPage> GetPublicationManifestAsync(Guid workspaceId, string? spaceTag, string? cursor, int limit, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<bool> HasChildrenAsync(Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<Page>> GetBreadcrumbAsync(Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<Page>> FindAllByNormalizedTitleAsync(Guid workspaceId, string normalizedTitle, CancellationToken cancellationToken) => Task.FromResult<IReadOnlyList<Page>>([]);
|
||||
public Task<bool> IsSameOrDescendantAsync(Guid pageId, Guid candidateId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<Page>> GetSiblingsAsync(Guid workspaceId, Guid? parentPageId, bool includeTrashed, CancellationToken cancellationToken) => Task.FromResult<IReadOnlyList<Page>>([]);
|
||||
public Task<IReadOnlyList<PageLink>> GetLinksForPageAsync(Guid sourcePageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<Page>> GetBacklinkSourcesAsync(Guid targetPageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<PageRevision>> GetRevisionsAsync(Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<PageRevision?> GetRevisionAsync(Guid pageId, long revisionNumber, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<int> PurgeEligibleTrashedPagesAsync(Guid workspaceId, DateTimeOffset now, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public void Add(Page page) => Added.Add(page);
|
||||
public void ReplaceLinks(Guid sourcePageId, IReadOnlyList<PageLink> links) { }
|
||||
public void AddRevision(PageRevision revision) => Revisions.Add(revision);
|
||||
public Task SaveChangesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public void RollBack() { Added.Clear(); Revisions.Clear(); }
|
||||
}
|
||||
|
||||
private sealed class Attachments : IAttachmentRepository
|
||||
{
|
||||
public Guid? TransferredPageId { get; private set; }
|
||||
public Task<AttachmentBlob?> FindBlobAsync(string sha256, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public void AddBlob(AttachmentBlob blob) => throw new NotSupportedException();
|
||||
public Task<AttachmentReference?> FindReferenceAsync(Guid referenceId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public void AddReference(AttachmentReference reference) => throw new NotSupportedException();
|
||||
public void AddUsage(AttachmentUsage usage) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<AttachmentReference>> GetReferencesForPageAsync(Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyDictionary<string, AttachmentBlob>> FindBlobsAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<AttachmentUsage?> FindUsageForReferenceAsync(Guid attachmentReferenceId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<string>> FindUnreferencedBlobHashesAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<int> RemoveBlobRecordsAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<AttachmentUsage>> GetUsagesForStagingSessionAsync(Guid stagingSessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task TransferProposalUsagesToPageAsync(Guid ingestionProposalId, Guid pageId, CancellationToken cancellationToken) { TransferredPageId = pageId; return Task.CompletedTask; }
|
||||
public Task<StagingAttachmentCleanupResult> RemoveOrphanedStagingAttachmentsAsync(Guid stagingSessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task SaveChangesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public void RollBack() => TransferredPageId = null;
|
||||
}
|
||||
|
||||
private sealed class Audits : IAuditEventRecorder
|
||||
{
|
||||
public List<AuditEventRecord> Events { get; } = [];
|
||||
public void Add(AuditEventRecord auditEvent) => Events.Add(auditEvent);
|
||||
public Task RecordAsync(AuditEventRecord auditEvent, CancellationToken cancellationToken) { Events.Add(auditEvent); return Task.CompletedTask; }
|
||||
public void RollBack() => Events.Clear();
|
||||
}
|
||||
|
||||
private sealed class Outbox : IOutboxRepository
|
||||
{
|
||||
public List<OutboxMessage> Messages { get; } = [];
|
||||
public void Add(OutboxMessage message) => Messages.Add(message);
|
||||
public Task<IReadOnlyList<OutboxMessage>> GetUnprocessedAsync(int limit, CancellationToken cancellationToken) => Task.FromResult<IReadOnlyList<OutboxMessage>>([]);
|
||||
public void RollBack() => Messages.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Route-level guard for the non-delegable human-review boundary. This is
|
||||
/// fixture-backed because it intentionally exercises the real cookie and bearer auth
|
||||
/// handlers.</summary>
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionPublicationApiTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
|
||||
postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
[Fact]
|
||||
public async Task Publish_rejects_a_service_account_and_accepts_the_human_reviewer()
|
||||
{
|
||||
using var owner = Factory.CreateClient();
|
||||
await SetupHelper.LoginAsync(owner, OwnerEmail, OwnerPassword);
|
||||
var serviceToken = await CreateServiceAccountAsync(owner, "publication-analyzer");
|
||||
using var service = Factory.CreateClient();
|
||||
service.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", serviceToken.RawToken);
|
||||
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest("safe manifest", "publication-api",
|
||||
new IngestionPublicationRequest("Published page", "# Body", "WikiPage", "Active", "CreatePage")))
|
||||
};
|
||||
create.Headers.Add("Idempotency-Key", "publication-api-create");
|
||||
var created = await service.SendAsync(create);
|
||||
created.EnsureSuccessStatusCode();
|
||||
var proposal = Assert.IsType<IngestionProposalDetail>(await created.Content.ReadFromJsonAsync<IngestionProposalDetail>());
|
||||
Assert.Equal("CreatePage", proposal.MutationKind);
|
||||
Assert.Null(proposal.Destination?.PageId);
|
||||
Assert.Equal("Published page", proposal.Destination?.Title);
|
||||
Assert.Equal("WikiPage", proposal.Metadata?.DocumentType);
|
||||
Assert.Equal("Active", proposal.Metadata?.DocumentStatus);
|
||||
Assert.Null(proposal.ExpectedTargetETag);
|
||||
Assert.Empty(proposal.Attachments);
|
||||
Assert.Empty(proposal.Citations);
|
||||
Assert.Empty(proposal.Warnings);
|
||||
Assert.Empty(proposal.DuplicateCandidates);
|
||||
Assert.Empty(proposal.ConflictCandidates);
|
||||
Assert.NotNull(proposal.Diff);
|
||||
Assert.Null(proposal.Diff.Before);
|
||||
Assert.Equal("# Body", proposal.Diff.After);
|
||||
|
||||
using var analyze = new HttpRequestMessage(HttpMethod.Patch, $"/api/v1/ingestion-proposals/{proposal.Id}")
|
||||
{
|
||||
Content = JsonContent.Create(new PatchIngestionProposalRequest(null, null, "Safe review analysis."))
|
||||
};
|
||||
analyze.Headers.TryAddWithoutValidation("If-Match", created.Headers.ETag!.Tag);
|
||||
var analyzed = await service.SendAsync(analyze);
|
||||
analyzed.EnsureSuccessStatusCode();
|
||||
|
||||
using var forbiddenPublish = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal.Id}/publish");
|
||||
forbiddenPublish.Headers.TryAddWithoutValidation("If-Match", analyzed.Headers.ETag!.Tag);
|
||||
forbiddenPublish.Headers.Add("Idempotency-Key", "service-cannot-publish");
|
||||
Assert.Equal(HttpStatusCode.Forbidden, (await service.SendAsync(forbiddenPublish)).StatusCode);
|
||||
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(owner);
|
||||
using var approve = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal.Id}/approve")
|
||||
{
|
||||
Content = JsonContent.Create(new IngestionProposalDecisionRequest(null))
|
||||
};
|
||||
approve.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
approve.Headers.TryAddWithoutValidation("If-Match", analyzed.Headers.ETag!.Tag);
|
||||
var approved = await owner.SendAsync(approve);
|
||||
approved.EnsureSuccessStatusCode();
|
||||
|
||||
csrf = await SetupHelper.GetCsrfTokenAsync(owner);
|
||||
using var publish = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal.Id}/publish");
|
||||
publish.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
publish.Headers.TryAddWithoutValidation("If-Match", approved.Headers.ETag!.Tag);
|
||||
publish.Headers.Add("Idempotency-Key", "human-publish-once");
|
||||
var published = await owner.SendAsync(publish);
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, published.StatusCode);
|
||||
var detail = await published.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.Equal("Published", detail?.Status);
|
||||
Assert.NotNull(detail?.PublishedPageId);
|
||||
}
|
||||
|
||||
private static async Task<CreatedServiceAccountToken> CreateServiceAccountAsync(HttpClient owner, string name)
|
||||
{
|
||||
var response = await SetupHelper.PostWithCsrfAsync(owner, "/api/v1/service-accounts",
|
||||
new CreateServiceAccountRequest(name, "trusted-intake publication test", [Scopes.IngestionWrite]));
|
||||
response.EnsureSuccessStatusCode();
|
||||
return Assert.IsType<CreatedServiceAccountToken>(await response.Content.ReadFromJsonAsync<CreatedServiceAccountToken>());
|
||||
}
|
||||
}
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using ItWorx.KnowledgeHub.Application.Attachments;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
using ItWorx.KnowledgeHub.Domain.Attachments;
|
||||
using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Domain.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Attachments;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Operations;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL persistence coverage for source retention. These tests deliberately use
|
||||
/// separate contexts around the candidate query, so the repository's deletion-time
|
||||
/// ownership recheck is proved against committed state rather than a port fake.
|
||||
/// </summary>
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionSourceRetentionPersistenceTests(PostgresContainerFixture postgres)
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 9, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
private static readonly Guid WorkspaceId = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||||
private static readonly Guid ActorId = Guid.Parse("22222222-2222-2222-2222-222222222222");
|
||||
private const string Digest = "c8c86d360bfb7bb0d036b7352bf89714946fb527c4074340752259109371f429";
|
||||
|
||||
[Fact]
|
||||
public async Task Cleanup_removes_persisted_expired_usage_and_reference_but_retains_the_physical_blob_record()
|
||||
{
|
||||
await using var target = await RestoreTargetDatabase.CreateAsync(postgres);
|
||||
var options = CreateOptions(target.ConnectionString);
|
||||
var seeded = await SeedExpiredSourceAsync(options);
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var cleanup = CreateCleanup(new IngestionProposalRepository(db), new AttachmentRepository(db));
|
||||
|
||||
var result = await cleanup.ExecuteAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(new IngestionSourceRetentionCleanupResult(1, 1), result);
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
Assert.Null(await db.AttachmentUsages.SingleOrDefaultAsync(usage => usage.Id == seeded.UsageId));
|
||||
Assert.Null(await db.AttachmentReferences.SingleOrDefaultAsync(reference => reference.Id == seeded.ReferenceId));
|
||||
Assert.NotNull(await db.AttachmentBlobs.SingleOrDefaultAsync(blob => blob.Sha256 == Digest));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cleanup_staged_usage_and_reference_removal_rolls_back_without_a_committed_transaction()
|
||||
{
|
||||
await using var target = await RestoreTargetDatabase.CreateAsync(postgres);
|
||||
var options = CreateOptions(target.ConnectionString);
|
||||
var seeded = await SeedExpiredSourceAsync(options);
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var retention = new IngestionProposalRepository(db);
|
||||
var candidate = Assert.Single(await retention.FindExpiredProposalSourcesAsync(Now.AddDays(-30), 100, CancellationToken.None));
|
||||
var attachments = new AttachmentRepository(db);
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(new ProposalSourceAttachmentCleanupResult(true, true),
|
||||
await attachments.RemoveExpiredProposalSourceUsageAsync(candidate, CancellationToken.None));
|
||||
await db.SaveChangesAsync(CancellationToken.None);
|
||||
await transaction.RollbackAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
Assert.NotNull(await db.AttachmentUsages.SingleOrDefaultAsync(usage => usage.Id == seeded.UsageId));
|
||||
Assert.NotNull(await db.AttachmentReferences.SingleOrDefaultAsync(reference => reference.Id == seeded.ReferenceId));
|
||||
Assert.NotNull(await db.AttachmentBlobs.SingleOrDefaultAsync(blob => blob.Sha256 == Digest));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cleanup_rechecks_committed_ownership_and_preserves_a_candidate_published_and_transferred_to_a_page()
|
||||
{
|
||||
await using var target = await RestoreTargetDatabase.CreateAsync(postgres);
|
||||
var options = CreateOptions(target.ConnectionString);
|
||||
var seeded = await SeedExpiredSourceAsync(options);
|
||||
IngestionSourceRetentionCandidate candidate;
|
||||
Guid publishedPageId;
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var retention = new IngestionProposalRepository(db);
|
||||
candidate = Assert.Single(await retention.FindExpiredProposalSourcesAsync(Now.AddDays(-30), 100, CancellationToken.None));
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var proposal = await db.IngestionProposals.SingleAsync(item => item.Id == seeded.ProposalId);
|
||||
var page = Page.Create(WorkspaceId, null, SortKey.First(), PageTitle.Create("Published evidence"),
|
||||
DocumentType.WikiPage, DocumentStatus.Active, "# Published evidence", ActorId, Now);
|
||||
var usage = await db.AttachmentUsages.SingleAsync(item => item.Id == seeded.UsageId);
|
||||
|
||||
proposal.SetPublication(IngestionPublication.Create(IngestionMutationKind.CreatePage, "Published evidence",
|
||||
"# Published evidence", "WikiPage", "Active"), ActorId, Now);
|
||||
proposal.BeginAnalysis(ActorId, Now);
|
||||
proposal.ApplyAnalysis("Reviewed analysis", ActorId, Now);
|
||||
proposal.Approve(ActorId, Now);
|
||||
proposal.BeginPublishing(ActorId, Now);
|
||||
proposal.MarkPublished(ActorId, page.Id, page.CurrentRevisionNumber, "retention-race", Now);
|
||||
usage.TransferProposalToPage(page.Id);
|
||||
db.Pages.Add(page);
|
||||
await db.SaveChangesAsync(CancellationToken.None);
|
||||
publishedPageId = page.Id;
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var cleanup = new AttachmentRepository(db);
|
||||
|
||||
var result = await cleanup.RemoveExpiredProposalSourceUsageAsync(candidate, CancellationToken.None);
|
||||
|
||||
Assert.Equal(ProposalSourceAttachmentCleanupResult.None, result);
|
||||
await db.SaveChangesAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
await using (var db = new KnowledgeHubDbContext(options))
|
||||
{
|
||||
var usage = await db.AttachmentUsages.SingleAsync(item => item.Id == seeded.UsageId);
|
||||
Assert.Equal(publishedPageId, usage.PageId);
|
||||
Assert.Null(usage.IngestionProposalId);
|
||||
Assert.NotNull(await db.AttachmentReferences.SingleOrDefaultAsync(reference => reference.Id == seeded.ReferenceId));
|
||||
Assert.NotNull(await db.AttachmentBlobs.SingleOrDefaultAsync(blob => blob.Sha256 == Digest));
|
||||
}
|
||||
}
|
||||
|
||||
private static DbContextOptions<KnowledgeHubDbContext> CreateOptions(string connectionString) =>
|
||||
new DbContextOptionsBuilder<KnowledgeHubDbContext>()
|
||||
.UseNpgsql(connectionString, npgsql => npgsql.MigrationsAssembly(typeof(KnowledgeHubDbContext).Assembly.GetName().Name))
|
||||
.UseSnakeCaseNamingConvention()
|
||||
.Options;
|
||||
|
||||
private static CleanupExpiredIngestionSourcesUseCase CreateCleanup(
|
||||
IIngestionSourceRetentionRepository retention,
|
||||
IAttachmentRepository attachments) =>
|
||||
new(retention, attachments, new AuditRecorder(), new IngestionMetrics(new Meter("ingestion-retention-persistence")),
|
||||
new FixedTimeProvider(Now));
|
||||
|
||||
private static async Task<SeededSource> SeedExpiredSourceAsync(DbContextOptions<KnowledgeHubDbContext> options)
|
||||
{
|
||||
await using var db = new KnowledgeHubDbContext(options);
|
||||
var reference = AttachmentReference.Create(WorkspaceId, Digest, "evidence.txt", ActorId, Now.AddDays(-31));
|
||||
var proposal = IngestionProposal.Create(WorkspaceId, ActorId, "retention-persistence", "safe manifest");
|
||||
proposal.AddSource("evidence.txt", "text/plain", 42, Digest, "clean", reference.Id);
|
||||
var usage = AttachmentUsage.ForIngestionProposal(reference.Id, proposal.Id, Now.AddDays(-31));
|
||||
|
||||
db.AttachmentBlobs.Add(AttachmentBlob.Create(Digest, 42, "text/plain", AttachmentDeliveryPolicy.DownloadOnly, Now.AddDays(-31)));
|
||||
db.AttachmentReferences.Add(reference);
|
||||
db.IngestionProposals.Add(proposal);
|
||||
db.AttachmentUsages.Add(usage);
|
||||
await db.SaveChangesAsync(CancellationToken.None);
|
||||
|
||||
return new SeededSource(proposal.Id, proposal.Sources.Single().Id, reference.Id, usage.Id);
|
||||
}
|
||||
|
||||
private sealed record SeededSource(Guid ProposalId, Guid SourceId, Guid ReferenceId, Guid UsageId);
|
||||
|
||||
private sealed class AuditRecorder : IAuditEventRecorder
|
||||
{
|
||||
public void Add(AuditEventRecord auditEvent) { }
|
||||
public Task RecordAsync(AuditEventRecord auditEvent, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => now;
|
||||
}
|
||||
}
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using ItWorx.KnowledgeHub.Application.Attachments;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
using ItWorx.KnowledgeHub.Domain.Attachments;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>Docker-free retention tests. The in-memory ports model the same shared
|
||||
/// scoped unit of work used by the EF repositories, while making the safety boundary
|
||||
/// and audit/metric payloads independently observable.</summary>
|
||||
public sealed class IngestionSourceRetentionTests
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 9, 20, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
[Fact]
|
||||
public async Task Cleanup_removes_only_expired_proposal_source_usage_and_keeps_safe_audit_metadata()
|
||||
{
|
||||
var expired = Source(Now.AddDays(-31));
|
||||
var active = Source(Now);
|
||||
var published = Source(Now.AddDays(-31));
|
||||
var retained = Source(Now.AddDays(-31));
|
||||
var retention = new RetentionRepository([expired, active, published, retained]);
|
||||
var attachments = new Attachments([
|
||||
new(expired, isProposalOwned: true, hasOtherUsage: false),
|
||||
new(active, isProposalOwned: true, hasOtherUsage: false),
|
||||
new(published, isProposalOwned: false, hasOtherUsage: true),
|
||||
new(retained, isProposalOwned: true, hasOtherUsage: true)
|
||||
]);
|
||||
var audits = new Audits();
|
||||
var metrics = new IngestionMetrics(new Meter("retention-test-metrics"));
|
||||
var cleanup = new CleanupExpiredIngestionSourcesUseCase(retention, attachments, audits, metrics, new FixedTimeProvider(Now));
|
||||
|
||||
var result = await cleanup.ExecuteAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.SourceUsagesRemoved);
|
||||
Assert.Equal(1, result.AttachmentReferencesRemoved);
|
||||
Assert.False(attachments.Exists(expired.AttachmentReferenceId));
|
||||
Assert.True(attachments.Exists(active.AttachmentReferenceId));
|
||||
Assert.True(attachments.Exists(published.AttachmentReferenceId));
|
||||
Assert.True(attachments.Exists(retained.AttachmentReferenceId));
|
||||
|
||||
Assert.Equal(2, audits.Events.Count);
|
||||
Assert.All(audits.Events, audit =>
|
||||
{
|
||||
Assert.Equal("ingestion-source.retained-reference-removed", audit.Action);
|
||||
Assert.Equal("Expired proposal source reference removed.", audit.Summary);
|
||||
var serializedAudit = string.Join(' ', audit.Action, audit.TargetType, audit.TargetId, audit.Summary, audit.Source, audit.CorrelationId);
|
||||
Assert.DoesNotContain("source-body", serializedAudit, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("C:\\private\\intake", serializedAudit, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("https://private.example", serializedAudit, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("token=secret", serializedAudit, StringComparison.OrdinalIgnoreCase);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cleanup_emits_only_bounded_ingestion_metric_labels()
|
||||
{
|
||||
var expired = Source(Now.AddDays(-31));
|
||||
var retention = new RetentionRepository([expired]);
|
||||
var attachments = new Attachments([new(expired, isProposalOwned: true, hasOtherUsage: false)]);
|
||||
var audits = new Audits();
|
||||
var meter = new Meter("retention-metrics-contract");
|
||||
var metrics = new IngestionMetrics(meter);
|
||||
var observedTags = new List<KeyValuePair<string, object?>>();
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, currentListener) =>
|
||||
{
|
||||
if (instrument.Meter == meter)
|
||||
currentListener.EnableMeasurementEvents(instrument);
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((instrument, _, tags, _) =>
|
||||
{
|
||||
if (instrument.Meter == meter)
|
||||
observedTags.AddRange(tags.ToArray());
|
||||
});
|
||||
listener.Start();
|
||||
var cleanup = new CleanupExpiredIngestionSourcesUseCase(retention, attachments, audits, metrics, new FixedTimeProvider(Now));
|
||||
|
||||
await cleanup.ExecuteAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(["outcome", "problem_type", "stage"], observedTags.Select(tag => tag.Key).OrderBy(key => key).ToArray());
|
||||
Assert.All(observedTags, tag => Assert.DoesNotContain("source-body-and-token=secret", tag.Value?.ToString() ?? string.Empty, StringComparison.Ordinal));
|
||||
Assert.Contains(observedTags, tag => tag.Key == "stage" && Equals(tag.Value, "retention"));
|
||||
Assert.Contains(observedTags, tag => tag.Key == "outcome" && Equals(tag.Value, "success"));
|
||||
Assert.Contains(observedTags, tag => tag.Key == "problem_type" && Equals(tag.Value, "none"));
|
||||
}
|
||||
|
||||
private static IngestionSourceRetentionCandidate Source(DateTimeOffset createdAt) => new(
|
||||
Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), createdAt);
|
||||
|
||||
private sealed class RetentionRepository(IReadOnlyList<IngestionSourceRetentionCandidate> sources) : IIngestionSourceRetentionRepository
|
||||
{
|
||||
public int SaveCalls { get; private set; }
|
||||
|
||||
public Task<IReadOnlyList<IngestionSourceRetentionCandidate>> FindExpiredProposalSourcesAsync(
|
||||
DateTimeOffset createdOnOrBefore, int batchSize, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyList<IngestionSourceRetentionCandidate>>(
|
||||
sources.Where(source => source.CreatedAt <= createdOnOrBefore).Take(batchSize).ToArray());
|
||||
|
||||
public Task<int> SaveChangesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
SaveCalls++;
|
||||
return Task.FromResult(1);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Attachments(IEnumerable<Row> rows) : IAttachmentRepository
|
||||
{
|
||||
private readonly List<Row> rows = rows.ToList();
|
||||
|
||||
public bool Exists(Guid referenceId) => rows.Any(row => row.Source.AttachmentReferenceId == referenceId && row.ReferenceExists);
|
||||
|
||||
public Task<ProposalSourceAttachmentCleanupResult> RemoveExpiredProposalSourceUsageAsync(
|
||||
IngestionSourceRetentionCandidate source, CancellationToken cancellationToken)
|
||||
{
|
||||
var row = rows.Single(row => row.Source.SourceId == source.SourceId);
|
||||
if (!row.IsProposalOwned || !row.UsageExists)
|
||||
return Task.FromResult(ProposalSourceAttachmentCleanupResult.None);
|
||||
|
||||
row.UsageExists = false;
|
||||
if (!row.HasOtherUsage)
|
||||
{
|
||||
row.ReferenceExists = false;
|
||||
return Task.FromResult(new ProposalSourceAttachmentCleanupResult(true, true));
|
||||
}
|
||||
|
||||
return Task.FromResult(new ProposalSourceAttachmentCleanupResult(true, false));
|
||||
}
|
||||
|
||||
public Task<AttachmentBlob?> FindBlobAsync(string sha256, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public void AddBlob(AttachmentBlob blob) => throw new NotSupportedException();
|
||||
public Task<AttachmentReference?> FindReferenceAsync(Guid referenceId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public void AddReference(AttachmentReference reference) => throw new NotSupportedException();
|
||||
public void AddUsage(AttachmentUsage usage) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<AttachmentReference>> GetReferencesForPageAsync(Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyDictionary<string, AttachmentBlob>> FindBlobsAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<AttachmentUsage?> FindUsageForReferenceAsync(Guid attachmentReferenceId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<string>> FindUnreferencedBlobHashesAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<int> RemoveBlobRecordsAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<AttachmentUsage>> GetUsagesForStagingSessionAsync(Guid stagingSessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task TransferProposalUsagesToPageAsync(Guid ingestionProposalId, Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task<StagingAttachmentCleanupResult> RemoveOrphanedStagingAttachmentsAsync(Guid stagingSessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
public Task SaveChangesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class Audits : IAuditEventRecorder
|
||||
{
|
||||
public List<AuditEventRecord> Events { get; } = [];
|
||||
public void Add(AuditEventRecord auditEvent) => Events.Add(auditEvent);
|
||||
public Task RecordAsync(AuditEventRecord auditEvent, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class Row(IngestionSourceRetentionCandidate source, bool isProposalOwned, bool hasOtherUsage)
|
||||
{
|
||||
public IngestionSourceRetentionCandidate Source { get; } = source;
|
||||
public bool IsProposalOwned { get; } = isProposalOwned;
|
||||
public bool HasOtherUsage { get; } = hasOtherUsage;
|
||||
public bool UsageExists { get; set; } = true;
|
||||
public bool ReferenceExists { get; set; } = true;
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => now;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user