Initial curated itworx-knowledge-hub source from 3b1751d0d9dee551aa6994eee0093704dc334817
This commit is contained in:
Public Source Publisher committed
commit
ad8ad252a9
825 files changed
+124216
No files matched your search
+184
@@ -0,0 +1,184 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;
|
||||
|
||||
/// <summary>These metric contract tests are Docker-free. They exercise the production
|
||||
/// metric surface directly, with synthetic dimensions only.</summary>
|
||||
public sealed class IngestionMetricContractTests
|
||||
{
|
||||
[Fact]
|
||||
public void Pilot_metrics_emit_only_bounded_labels_and_normalize_untrusted_dimensions()
|
||||
{
|
||||
using var meter = new Meter("ingestion-observability-test");
|
||||
var measurements = new List<ObservedMeasurement>();
|
||||
using var listener = new MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, currentListener) =>
|
||||
{
|
||||
if (instrument.Meter == meter)
|
||||
currentListener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
||||
measurements.Add(Observe(instrument.Name, value, tags)));
|
||||
listener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
||||
measurements.Add(Observe(instrument.Name, value, tags)));
|
||||
listener.Start();
|
||||
|
||||
var metrics = new IngestionMetrics(meter);
|
||||
metrics.RecordProposal("success", "none");
|
||||
metrics.RecordStageDuration("source_staging", "success", 1.25);
|
||||
metrics.RecordReviewDuration("success", 2.5);
|
||||
metrics.RecordPublication("success", "none");
|
||||
|
||||
const string sensitiveBody = "do not log me";
|
||||
const string sensitiveFileName = "secret-plan.pdf";
|
||||
const string sensitiveSourceUrl = "https://private.example/source?token=secret";
|
||||
metrics.RecordProposal(sensitiveBody, sensitiveSourceUrl);
|
||||
metrics.RecordStageDuration(sensitiveFileName, sensitiveBody, 3.75);
|
||||
metrics.RecordReviewDuration(sensitiveBody, 4.5);
|
||||
metrics.RecordPublication(sensitiveBody, sensitiveSourceUrl);
|
||||
|
||||
var expected = new[]
|
||||
{
|
||||
("ingestion_proposals_total", "outcome=other|problem_type=other|stage=proposal"),
|
||||
("ingestion_proposals_total", "outcome=success|problem_type=none|stage=proposal"),
|
||||
("ingestion_stage_duration_seconds", "outcome=other|problem_type=none|stage=other"),
|
||||
("ingestion_stage_duration_seconds", "outcome=success|problem_type=none|stage=source_staging"),
|
||||
("ingestion_review_duration_seconds", "outcome=other|problem_type=none|stage=review"),
|
||||
("ingestion_review_duration_seconds", "outcome=success|problem_type=none|stage=review"),
|
||||
("ingestion_publications_total", "outcome=other|problem_type=other|stage=publication"),
|
||||
("ingestion_publications_total", "outcome=success|problem_type=none|stage=publication")
|
||||
}
|
||||
.OrderBy(measurement => measurement.Item1, StringComparer.Ordinal)
|
||||
.ThenBy(measurement => measurement.Item2, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var actual = measurements
|
||||
.Select(measurement => (measurement.Name, Tags: SerializeTags(measurement.Tags)))
|
||||
.OrderBy(measurement => measurement.Name, StringComparer.Ordinal)
|
||||
.ThenBy(measurement => measurement.Tags, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
Assert.Equal(expected, actual);
|
||||
|
||||
var allowedLabelKeys = new HashSet<string>(["stage", "outcome", "problem_type"], StringComparer.Ordinal);
|
||||
Assert.All(measurements, measurement => Assert.Subset(allowedLabelKeys, measurement.Tags.Keys.ToHashSet(StringComparer.Ordinal)));
|
||||
Assert.All(measurements, measurement => Assert.True(measurement.Value > 0));
|
||||
var serializedMeasurements = string.Join('\n', actual.Select(measurement => measurement.Tags));
|
||||
Assert.DoesNotContain(sensitiveBody, serializedMeasurements, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveFileName, serializedMeasurements, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveSourceUrl, serializedMeasurements, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static ObservedMeasurement Observe(string name, double value, ReadOnlySpan<KeyValuePair<string, object?>> tags) =>
|
||||
new(name, value, tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value?.ToString() ?? string.Empty, StringComparer.Ordinal));
|
||||
|
||||
private static string SerializeTags(IReadOnlyDictionary<string, string> tags) =>
|
||||
string.Join('|', tags.OrderBy(tag => tag.Key, StringComparer.Ordinal).Select(tag => $"{tag.Key}={tag.Value}"));
|
||||
|
||||
private sealed record ObservedMeasurement(string Name, double Value, IReadOnlyDictionary<string, string> Tags);
|
||||
}
|
||||
|
||||
[Collection(PostgresTestGroup.Name)]
|
||||
public sealed class IngestionObservabilityTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
|
||||
postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
|
||||
{
|
||||
[Fact]
|
||||
public async Task Proposal_and_source_requests_never_write_source_content_or_file_metadata_to_logs()
|
||||
{
|
||||
const string sensitiveFileName = "secret-plan.pdf";
|
||||
const string sensitiveSource = "source bytes must stay out of logs";
|
||||
const string sensitiveUrl = "https://private.example/source?token=secret";
|
||||
var sensitiveManifest = $"do not log me {sensitiveUrl}";
|
||||
var captured = new ConcurrentQueue<string>();
|
||||
|
||||
using var loggingFactory = Factory.WithWebHostBuilder(builder =>
|
||||
builder.ConfigureLogging(logging => logging.AddProvider(new CapturingLoggerProvider(captured))));
|
||||
using var client = loggingFactory.CreateClient();
|
||||
var login = await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
||||
login.EnsureSuccessStatusCode();
|
||||
|
||||
var csrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
|
||||
{
|
||||
Content = JsonContent.Create(new CreateIngestionProposalRequest(sensitiveManifest, "safe-observability-correlation"))
|
||||
};
|
||||
create.Headers.Add(csrf.HeaderName, csrf.Value);
|
||||
create.Headers.Add("Idempotency-Key", "safe-observability-create");
|
||||
create.Headers.Add("X-Correlation-ID", "safe-observability-correlation");
|
||||
var created = await client.SendAsync(create);
|
||||
Assert.Equal(HttpStatusCode.Created, created.StatusCode);
|
||||
var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
|
||||
Assert.NotNull(proposal);
|
||||
|
||||
var sourceCsrf = await SetupHelper.GetCsrfTokenAsync(client);
|
||||
using var multipart = new MultipartFormDataContent();
|
||||
var sourceBody = new ByteArrayContent(System.Text.Encoding.UTF8.GetBytes(sensitiveSource));
|
||||
sourceBody.Headers.ContentType = new MediaTypeHeaderValue("application/pdf");
|
||||
multipart.Add(sourceBody, "file", sensitiveFileName);
|
||||
using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/sources")
|
||||
{
|
||||
Content = multipart
|
||||
};
|
||||
upload.Headers.Add(sourceCsrf.HeaderName, sourceCsrf.Value);
|
||||
upload.Headers.Add("Idempotency-Key", "safe-observability-source");
|
||||
upload.Headers.Add("If-Match", created.Headers.ETag!.Tag);
|
||||
upload.Headers.Add("X-Correlation-ID", "safe-observability-correlation");
|
||||
var staged = await client.SendAsync(upload);
|
||||
Assert.Equal(HttpStatusCode.OK, staged.StatusCode);
|
||||
|
||||
var logText = string.Join('\n', captured);
|
||||
Assert.Contains("safe-observability-correlation", logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveManifest, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveFileName, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveSource, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(sensitiveUrl, logText, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private sealed class CapturingLoggerProvider(ConcurrentQueue<string> sink) : ILoggerProvider
|
||||
{
|
||||
public ILogger CreateLogger(string categoryName) => new CapturingLogger(categoryName, sink);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class CapturingLogger(string categoryName, ConcurrentQueue<string> sink) : ILogger
|
||||
{
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull
|
||||
{
|
||||
sink.Enqueue($"[{categoryName}] scope: {Format(state)}");
|
||||
return NoopDisposable.Instance;
|
||||
}
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel,
|
||||
EventId eventId,
|
||||
TState state,
|
||||
Exception? exception,
|
||||
Func<TState, Exception?, string> formatter) =>
|
||||
sink.Enqueue($"[{categoryName}] {formatter(state, exception)} {Format(state)} {exception}");
|
||||
|
||||
private static string Format<TState>(TState state) => state is IEnumerable<KeyValuePair<string, object?>> fields
|
||||
? string.Join(' ', fields.Select(field => $"{field.Key}={field.Value}"))
|
||||
: state?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
private sealed class NoopDisposable : IDisposable
|
||||
{
|
||||
public static NoopDisposable Instance { get; } = new();
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user