itworx-knowledge-hub-Public/tests/backend/ItWorx.KnowledgeHub.IntegrationTests/Ingestion/IngestionSourceRetentionTests.cs

using System.Diagnostics.Metrics;
using ItWorx.KnowledgeHub.Application.Attachments;
using ItWorx.KnowledgeHub.Application.Audit;
using ItWorx.KnowledgeHub.Application.Ingestion;
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
using ItWorx.KnowledgeHub.Domain.Attachments;

namespace ItWorx.KnowledgeHub.IntegrationTests.Ingestion;

/// <summary>Docker-free retention tests. The in-memory ports model the same shared
/// scoped unit of work used by the EF repositories, while making the safety boundary
/// and audit/metric payloads independently observable.</summary>
public sealed class IngestionSourceRetentionTests
{
    private static readonly DateTimeOffset Now = new(2026, 9, 20, 12, 0, 0, TimeSpan.Zero);

    [Fact]
    public async Task Cleanup_removes_only_expired_proposal_source_usage_and_keeps_safe_audit_metadata()
    {
        var expired = Source(Now.AddDays(-31));
        var active = Source(Now);
        var published = Source(Now.AddDays(-31));
        var retained = Source(Now.AddDays(-31));
        var retention = new RetentionRepository([expired, active, published, retained]);
        var attachments = new Attachments([
            new(expired, isProposalOwned: true, hasOtherUsage: false),
            new(active, isProposalOwned: true, hasOtherUsage: false),
            new(published, isProposalOwned: false, hasOtherUsage: true),
            new(retained, isProposalOwned: true, hasOtherUsage: true)
        ]);
        var audits = new Audits();
        var metrics = new IngestionMetrics(new Meter("retention-test-metrics"));
        var cleanup = new CleanupExpiredIngestionSourcesUseCase(retention, attachments, audits, metrics, new FixedTimeProvider(Now));

        var result = await cleanup.ExecuteAsync(CancellationToken.None);

        Assert.Equal(2, result.SourceUsagesRemoved);
        Assert.Equal(1, result.AttachmentReferencesRemoved);
        Assert.False(attachments.Exists(expired.AttachmentReferenceId));
        Assert.True(attachments.Exists(active.AttachmentReferenceId));
        Assert.True(attachments.Exists(published.AttachmentReferenceId));
        Assert.True(attachments.Exists(retained.AttachmentReferenceId));

        Assert.Equal(2, audits.Events.Count);
        Assert.All(audits.Events, audit =>
        {
            Assert.Equal("ingestion-source.retained-reference-removed", audit.Action);
            Assert.Equal("Expired proposal source reference removed.", audit.Summary);
            var serializedAudit = string.Join(' ', audit.Action, audit.TargetType, audit.TargetId, audit.Summary, audit.Source, audit.CorrelationId);
            Assert.DoesNotContain("source-body", serializedAudit, StringComparison.OrdinalIgnoreCase);
            Assert.DoesNotContain("C:\\private\\intake", serializedAudit, StringComparison.OrdinalIgnoreCase);
            Assert.DoesNotContain("https://private.example", serializedAudit, StringComparison.OrdinalIgnoreCase);
            Assert.DoesNotContain("token=secret", serializedAudit, StringComparison.OrdinalIgnoreCase);
        });
    }

    [Fact]
    public async Task Cleanup_emits_only_bounded_ingestion_metric_labels()
    {
        var expired = Source(Now.AddDays(-31));
        var retention = new RetentionRepository([expired]);
        var attachments = new Attachments([new(expired, isProposalOwned: true, hasOtherUsage: false)]);
        var audits = new Audits();
        var meter = new Meter("retention-metrics-contract");
        var metrics = new IngestionMetrics(meter);
        var observedTags = new List<KeyValuePair<string, object?>>();
        using var listener = new MeterListener();
        listener.InstrumentPublished = (instrument, currentListener) =>
        {
            if (instrument.Meter == meter)
                currentListener.EnableMeasurementEvents(instrument);
        };
        listener.SetMeasurementEventCallback<long>((instrument, _, tags, _) =>
        {
            if (instrument.Meter == meter)
                observedTags.AddRange(tags.ToArray());
        });
        listener.Start();
        var cleanup = new CleanupExpiredIngestionSourcesUseCase(retention, attachments, audits, metrics, new FixedTimeProvider(Now));

        await cleanup.ExecuteAsync(CancellationToken.None);

        Assert.Equal(["outcome", "problem_type", "stage"], observedTags.Select(tag => tag.Key).OrderBy(key => key).ToArray());
        Assert.All(observedTags, tag => Assert.DoesNotContain("source-body-and-token=secret", tag.Value?.ToString() ?? string.Empty, StringComparison.Ordinal));
        Assert.Contains(observedTags, tag => tag.Key == "stage" && Equals(tag.Value, "retention"));
        Assert.Contains(observedTags, tag => tag.Key == "outcome" && Equals(tag.Value, "success"));
        Assert.Contains(observedTags, tag => tag.Key == "problem_type" && Equals(tag.Value, "none"));
    }

    private static IngestionSourceRetentionCandidate Source(DateTimeOffset createdAt) => new(
        Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), createdAt);

    private sealed class RetentionRepository(IReadOnlyList<IngestionSourceRetentionCandidate> sources) : IIngestionSourceRetentionRepository
    {
        public int SaveCalls { get; private set; }

        public Task<IReadOnlyList<IngestionSourceRetentionCandidate>> FindExpiredProposalSourcesAsync(
            DateTimeOffset createdOnOrBefore, int batchSize, CancellationToken cancellationToken) =>
            Task.FromResult<IReadOnlyList<IngestionSourceRetentionCandidate>>(
                sources.Where(source => source.CreatedAt <= createdOnOrBefore).Take(batchSize).ToArray());

        public Task<int> SaveChangesAsync(CancellationToken cancellationToken)
        {
            SaveCalls++;
            return Task.FromResult(1);
        }
    }

    private sealed class Attachments(IEnumerable<Row> rows) : IAttachmentRepository
    {
        private readonly List<Row> rows = rows.ToList();

        public bool Exists(Guid referenceId) => rows.Any(row => row.Source.AttachmentReferenceId == referenceId && row.ReferenceExists);

        public Task<ProposalSourceAttachmentCleanupResult> RemoveExpiredProposalSourceUsageAsync(
            IngestionSourceRetentionCandidate source, CancellationToken cancellationToken)
        {
            var row = rows.Single(row => row.Source.SourceId == source.SourceId);
            if (!row.IsProposalOwned || !row.UsageExists)
                return Task.FromResult(ProposalSourceAttachmentCleanupResult.None);

            row.UsageExists = false;
            if (!row.HasOtherUsage)
            {
                row.ReferenceExists = false;
                return Task.FromResult(new ProposalSourceAttachmentCleanupResult(true, true));
            }

            return Task.FromResult(new ProposalSourceAttachmentCleanupResult(true, false));
        }

        public Task<AttachmentBlob?> FindBlobAsync(string sha256, CancellationToken cancellationToken) => throw new NotSupportedException();
        public void AddBlob(AttachmentBlob blob) => throw new NotSupportedException();
        public Task<AttachmentReference?> FindReferenceAsync(Guid referenceId, CancellationToken cancellationToken) => throw new NotSupportedException();
        public void AddReference(AttachmentReference reference) => throw new NotSupportedException();
        public void AddUsage(AttachmentUsage usage) => throw new NotSupportedException();
        public Task<IReadOnlyList<AttachmentReference>> GetReferencesForPageAsync(Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task<IReadOnlyDictionary<string, AttachmentBlob>> FindBlobsAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task<AttachmentUsage?> FindUsageForReferenceAsync(Guid attachmentReferenceId, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task<IReadOnlyList<string>> FindUnreferencedBlobHashesAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task<int> RemoveBlobRecordsAsync(IReadOnlyCollection<string> hashes, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task<IReadOnlyList<AttachmentUsage>> GetUsagesForStagingSessionAsync(Guid stagingSessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task TransferProposalUsagesToPageAsync(Guid ingestionProposalId, Guid pageId, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task<StagingAttachmentCleanupResult> RemoveOrphanedStagingAttachmentsAsync(Guid stagingSessionId, CancellationToken cancellationToken) => throw new NotSupportedException();
        public Task SaveChangesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
    }

    private sealed class Audits : IAuditEventRecorder
    {
        public List<AuditEventRecord> Events { get; } = [];
        public void Add(AuditEventRecord auditEvent) => Events.Add(auditEvent);
        public Task RecordAsync(AuditEventRecord auditEvent, CancellationToken cancellationToken) => Task.CompletedTask;
    }

    private sealed class Row(IngestionSourceRetentionCandidate source, bool isProposalOwned, bool hasOtherUsage)
    {
        public IngestionSourceRetentionCandidate Source { get; } = source;
        public bool IsProposalOwned { get; } = isProposalOwned;
        public bool HasOtherUsage { get; } = hasOtherUsage;
        public bool UsageExists { get; set; } = true;
        public bool ReferenceExists { get; set; } = true;
    }

    private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider
    {
        public override DateTimeOffset GetUtcNow() => now;
    }
}