itworx-knowledge-hub-Public/tests/backend/ItWorx.KnowledgeHub.IntegrationTests/Ingestion/IngestionObservabilityTests.cs

using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using ItWorx.KnowledgeHub.Application.Ingestion;
using ItWorx.KnowledgeHub.Contracts.Ingestion;
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Logging;

namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;

/// <summary>These metric contract tests are Docker-free. They exercise the production
/// metric surface directly, with synthetic dimensions only.</summary>
public sealed class IngestionMetricContractTests
{
    [Fact]
    public void Pilot_metrics_emit_only_bounded_labels_and_normalize_untrusted_dimensions()
    {
        using var meter = new Meter("ingestion-observability-test");
        var measurements = new List<ObservedMeasurement>();
        using var listener = new MeterListener
        {
            InstrumentPublished = (instrument, currentListener) =>
            {
                if (instrument.Meter == meter)
                    currentListener.EnableMeasurementEvents(instrument);
            }
        };
        listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
            measurements.Add(Observe(instrument.Name, value, tags)));
        listener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
            measurements.Add(Observe(instrument.Name, value, tags)));
        listener.Start();

        var metrics = new IngestionMetrics(meter);
        metrics.RecordProposal("success", "none");
        metrics.RecordStageDuration("source_staging", "success", 1.25);
        metrics.RecordReviewDuration("success", 2.5);
        metrics.RecordPublication("success", "none");

        const string sensitiveBody = "do not log me";
        const string sensitiveFileName = "secret-plan.pdf";
        const string sensitiveSourceUrl = "https://private.example/source?token=secret";
        metrics.RecordProposal(sensitiveBody, sensitiveSourceUrl);
        metrics.RecordStageDuration(sensitiveFileName, sensitiveBody, 3.75);
        metrics.RecordReviewDuration(sensitiveBody, 4.5);
        metrics.RecordPublication(sensitiveBody, sensitiveSourceUrl);

        var expected = new[]
        {
            ("ingestion_proposals_total", "outcome=other|problem_type=other|stage=proposal"),
            ("ingestion_proposals_total", "outcome=success|problem_type=none|stage=proposal"),
            ("ingestion_stage_duration_seconds", "outcome=other|problem_type=none|stage=other"),
            ("ingestion_stage_duration_seconds", "outcome=success|problem_type=none|stage=source_staging"),
            ("ingestion_review_duration_seconds", "outcome=other|problem_type=none|stage=review"),
            ("ingestion_review_duration_seconds", "outcome=success|problem_type=none|stage=review"),
            ("ingestion_publications_total", "outcome=other|problem_type=other|stage=publication"),
            ("ingestion_publications_total", "outcome=success|problem_type=none|stage=publication")
        }
            .OrderBy(measurement => measurement.Item1, StringComparer.Ordinal)
            .ThenBy(measurement => measurement.Item2, StringComparer.Ordinal)
            .ToArray();

        var actual = measurements
            .Select(measurement => (measurement.Name, Tags: SerializeTags(measurement.Tags)))
            .OrderBy(measurement => measurement.Name, StringComparer.Ordinal)
            .ThenBy(measurement => measurement.Tags, StringComparer.Ordinal)
            .ToArray();
        Assert.Equal(expected, actual);

        var allowedLabelKeys = new HashSet<string>(["stage", "outcome", "problem_type"], StringComparer.Ordinal);
        Assert.All(measurements, measurement => Assert.Subset(allowedLabelKeys, measurement.Tags.Keys.ToHashSet(StringComparer.Ordinal)));
        Assert.All(measurements, measurement => Assert.True(measurement.Value > 0));
        var serializedMeasurements = string.Join('\n', actual.Select(measurement => measurement.Tags));
        Assert.DoesNotContain(sensitiveBody, serializedMeasurements, StringComparison.Ordinal);
        Assert.DoesNotContain(sensitiveFileName, serializedMeasurements, StringComparison.Ordinal);
        Assert.DoesNotContain(sensitiveSourceUrl, serializedMeasurements, StringComparison.Ordinal);
    }

    private static ObservedMeasurement Observe(string name, double value, ReadOnlySpan<KeyValuePair<string, object?>> tags) =>
        new(name, value, tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value?.ToString() ?? string.Empty, StringComparer.Ordinal));

    private static string SerializeTags(IReadOnlyDictionary<string, string> tags) =>
        string.Join('|', tags.OrderBy(tag => tag.Key, StringComparer.Ordinal).Select(tag => $"{tag.Key}={tag.Value}"));

    private sealed record ObservedMeasurement(string Name, double Value, IReadOnlyDictionary<string, string> Tags);
}

[Collection(PostgresTestGroup.Name)]
public sealed class IngestionObservabilityTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(
    postgres, new Dictionary<string, string?> { ["IngestionProposal:Enabled"] = "true" })
{
    [Fact]
    public async Task Proposal_and_source_requests_never_write_source_content_or_file_metadata_to_logs()
    {
        const string sensitiveFileName = "secret-plan.pdf";
        const string sensitiveSource = "source bytes must stay out of logs";
        const string sensitiveUrl = "https://private.example/source?token=secret";
        var sensitiveManifest = $"do not log me {sensitiveUrl}";
        var captured = new ConcurrentQueue<string>();

        using var loggingFactory = Factory.WithWebHostBuilder(builder =>
            builder.ConfigureLogging(logging => logging.AddProvider(new CapturingLoggerProvider(captured))));
        using var client = loggingFactory.CreateClient();
        var login = await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
        login.EnsureSuccessStatusCode();

        var csrf = await SetupHelper.GetCsrfTokenAsync(client);
        using var create = new HttpRequestMessage(HttpMethod.Post, "/api/v1/ingestion-proposals")
        {
            Content = JsonContent.Create(new CreateIngestionProposalRequest(sensitiveManifest, "safe-observability-correlation"))
        };
        create.Headers.Add(csrf.HeaderName, csrf.Value);
        create.Headers.Add("Idempotency-Key", "safe-observability-create");
        create.Headers.Add("X-Correlation-ID", "safe-observability-correlation");
        var created = await client.SendAsync(create);
        Assert.Equal(HttpStatusCode.Created, created.StatusCode);
        var proposal = await created.Content.ReadFromJsonAsync<IngestionProposalDetail>();
        Assert.NotNull(proposal);

        var sourceCsrf = await SetupHelper.GetCsrfTokenAsync(client);
        using var multipart = new MultipartFormDataContent();
        var sourceBody = new ByteArrayContent(System.Text.Encoding.UTF8.GetBytes(sensitiveSource));
        sourceBody.Headers.ContentType = new MediaTypeHeaderValue("application/pdf");
        multipart.Add(sourceBody, "file", sensitiveFileName);
        using var upload = new HttpRequestMessage(HttpMethod.Post, $"/api/v1/ingestion-proposals/{proposal!.Id}/sources")
        {
            Content = multipart
        };
        upload.Headers.Add(sourceCsrf.HeaderName, sourceCsrf.Value);
        upload.Headers.Add("Idempotency-Key", "safe-observability-source");
        upload.Headers.Add("If-Match", created.Headers.ETag!.Tag);
        upload.Headers.Add("X-Correlation-ID", "safe-observability-correlation");
        var staged = await client.SendAsync(upload);
        Assert.Equal(HttpStatusCode.OK, staged.StatusCode);

        var logText = string.Join('\n', captured);
        Assert.Contains("safe-observability-correlation", logText, StringComparison.Ordinal);
        Assert.DoesNotContain(sensitiveManifest, logText, StringComparison.Ordinal);
        Assert.DoesNotContain(sensitiveFileName, logText, StringComparison.Ordinal);
        Assert.DoesNotContain(sensitiveSource, logText, StringComparison.Ordinal);
        Assert.DoesNotContain(sensitiveUrl, logText, StringComparison.Ordinal);
    }

    private sealed class CapturingLoggerProvider(ConcurrentQueue<string> sink) : ILoggerProvider
    {
        public ILogger CreateLogger(string categoryName) => new CapturingLogger(categoryName, sink);

        public void Dispose()
        {
        }

        private sealed class CapturingLogger(string categoryName, ConcurrentQueue<string> sink) : ILogger
        {
            public IDisposable? BeginScope<TState>(TState state) where TState : notnull
            {
                sink.Enqueue($"[{categoryName}] scope: {Format(state)}");
                return NoopDisposable.Instance;
            }

            public bool IsEnabled(LogLevel logLevel) => true;

            public void Log<TState>(
                LogLevel logLevel,
                EventId eventId,
                TState state,
                Exception? exception,
                Func<TState, Exception?, string> formatter) =>
                sink.Enqueue($"[{categoryName}] {formatter(state, exception)} {Format(state)} {exception}");

            private static string Format<TState>(TState state) => state is IEnumerable<KeyValuePair<string, object?>> fields
                ? string.Join(' ', fields.Select(field => $"{field.Key}={field.Value}"))
                : state?.ToString() ?? string.Empty;
        }

        private sealed class NoopDisposable : IDisposable
        {
            public static NoopDisposable Instance { get; } = new();
            public void Dispose() { }
        }
    }
}