itworx-knowledge-hub-Public/tests/backend/ItWorx.KnowledgeHub.IntegrationTests/Knowledge/RagPublicationManifestTests.cs

using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using ItWorx.KnowledgeHub.Contracts.Automation;
using ItWorx.KnowledgeHub.Contracts.Knowledge;
using ItWorx.KnowledgeHub.Domain.Automation;
using ItWorx.KnowledgeHub.Domain.Knowledge;
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;

namespace ItWorx.KnowledgeHub.IntegrationTests.Knowledge;

/// <summary>
/// The RAG publication manifest (docs/32-rag-publication-convention.md): a single
/// narrow read-only endpoint for RAGcore's n8n sync, entirely driven by the
/// <c>rag:publish</c>/<c>rag:space:</c>/<c>rag:allow-deprecated</c>/<c>locale:</c> tag
/// convention rather than a new editorial model.
/// </summary>
[Collection(PostgresTestGroup.Name)]
public sealed class RagPublicationManifestTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
{
    private static async Task<PageDetail> CreatePageAsync(
        HttpClient client, string title, IReadOnlyList<string>? tags = null, string markdown = "Inhoud.")
    {
        var response = await SetupHelper.PostWithCsrfAsync(
            client, "/api/v1/pages", new CreatePageRequest(null, title, "WikiPage", markdown, null, tags));
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
        var page = await response.Content.ReadFromJsonAsync<PageDetail>();
        Assert.NotNull(page);
        return page;
    }

    /// <summary>Publishes a page to <c>Active</c> — every page starts as a <c>Draft</c>
    /// "concept" at creation (<c>Page.Create</c>), so live-content assertions must
    /// explicitly advance it, exactly like an editor publishing from the UI would.
    /// </summary>
    private static async Task<PageDetail> PublishAsync(HttpClient client, PageDetail page)
    {
        var response = await SetupHelper.PutWithCsrfAsync(
            client,
            $"/api/v1/pages/{page.Id}",
            new UpdatePageRequest(page.Title, page.Markdown, page.DocumentType, "Active", "Gepubliceerd.", page.Tags),
            page.ETag);
        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
        var updated = await response.Content.ReadFromJsonAsync<PageDetail>();
        Assert.NotNull(updated);
        return updated;
    }

    [Fact]
    public async Task Only_pages_tagged_rag_publish_ever_appear_in_the_manifest()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var tagged = await CreatePageAsync(client, $"Gepubliceerd {Guid.NewGuid():N}", [RagPublicationTags.Publish]);
        var untagged = await CreatePageAsync(client, $"Niet gepubliceerd {Guid.NewGuid():N}");

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?limit=200");

        Assert.NotNull(manifest);
        Assert.Contains(manifest.Items, item => item.ContentId == tagged.Id);
        Assert.DoesNotContain(manifest.Items, item => item.ContentId == untagged.Id);
    }

    [Fact]
    public async Task A_draft_concept_comes_back_retired_without_leaking_its_markdown_or_checksum()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        // Deliberately never published — stays Draft ("concept"), matching Page.Create's
        // default status.
        var draft = await CreatePageAsync(
            client, $"Concept {Guid.NewGuid():N}", [RagPublicationTags.Publish], "Geheime conceptinhoud.");

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?limit=200");

        Assert.NotNull(manifest);
        var item = Assert.Single(manifest.Items, entry => entry.ContentId == draft.Id);
        Assert.True(item.Retired);
        Assert.Equal("draft", item.RetiredReason);
        Assert.Null(item.Markdown);
        Assert.Null(item.Checksum);
    }

    [Fact]
    public async Task An_active_published_page_comes_back_live_with_markdown_and_a_matching_checksum()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var created = await CreatePageAsync(
            client, $"Live {Guid.NewGuid():N}", [RagPublicationTags.Publish], "# Live inhoud");
        var published = await PublishAsync(client, created);

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?limit=200");

        Assert.NotNull(manifest);
        var item = Assert.Single(manifest.Items, entry => entry.ContentId == published.Id);
        Assert.False(item.Retired);
        Assert.Null(item.RetiredReason);
        Assert.Equal("# Live inhoud", item.Markdown);
        Assert.NotNull(item.Checksum);
        Assert.Equal(published.CurrentRevisionNumber, item.Revision);
        Assert.Contains($"/knowledge/{published.Id}/{published.Slug}", item.CanonicalUrl, StringComparison.Ordinal);
    }

    [Fact]
    public async Task A_trashed_page_comes_back_retired_without_markdown_so_the_sync_can_withdraw_it()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var created = await CreatePageAsync(client, $"Te verwijderen {Guid.NewGuid():N}", [RagPublicationTags.Publish]);
        var published = await PublishAsync(client, created);

        await SetupHelper.PostWithCsrfAndIfMatchAsync(
            client, $"/api/v1/pages/{published.Id}/trash", (object?)null, published.ETag);

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?limit=200");

        Assert.NotNull(manifest);
        var item = Assert.Single(manifest.Items, entry => entry.ContentId == published.Id);
        Assert.True(item.Retired);
        Assert.Equal("trashed", item.RetiredReason);
        Assert.Null(item.Markdown);
        Assert.Null(item.Checksum);
    }

    [Fact]
    public async Task A_deprecated_page_without_the_allow_flag_is_retired_but_with_the_flag_it_stays_live()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var withoutFlag = await PublishAsync(
            client, await CreatePageAsync(client, $"Verouderd {Guid.NewGuid():N}", [RagPublicationTags.Publish]));
        var deprecatedWithoutFlag = await SetupHelper.PutWithCsrfAsync(
            client,
            $"/api/v1/pages/{withoutFlag.Id}",
            new UpdatePageRequest(withoutFlag.Title, withoutFlag.Markdown, withoutFlag.DocumentType, "Deprecated", "Verouderd.", withoutFlag.Tags),
            withoutFlag.ETag);
        Assert.Equal(HttpStatusCode.OK, deprecatedWithoutFlag.StatusCode);

        var withFlagTags = new[] { RagPublicationTags.Publish, RagPublicationTags.AllowDeprecated };
        var withFlag = await PublishAsync(
            client, await CreatePageAsync(client, $"Verouderd maar toegestaan {Guid.NewGuid():N}", withFlagTags));
        var deprecatedWithFlag = await SetupHelper.PutWithCsrfAsync(
            client,
            $"/api/v1/pages/{withFlag.Id}",
            new UpdatePageRequest(withFlag.Title, withFlag.Markdown, withFlag.DocumentType, "Deprecated", "Verouderd, maar nog toegestaan.", withFlagTags),
            withFlag.ETag);
        Assert.Equal(HttpStatusCode.OK, deprecatedWithFlag.StatusCode);

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?limit=200");

        Assert.NotNull(manifest);
        var retiredItem = Assert.Single(manifest.Items, entry => entry.ContentId == withoutFlag.Id);
        Assert.True(retiredItem.Retired);
        Assert.Equal("deprecated", retiredItem.RetiredReason);
        Assert.Null(retiredItem.Markdown);

        var liveItem = Assert.Single(manifest.Items, entry => entry.ContentId == withFlag.Id);
        Assert.False(liveItem.Retired);
        Assert.NotNull(liveItem.Markdown);
    }

    [Fact]
    public async Task Locale_falls_back_to_the_configured_primary_locale_and_never_defaults_to_english()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var undeclared = await PublishAsync(
            client, await CreatePageAsync(client, $"Geen locale {Guid.NewGuid():N}", [RagPublicationTags.Publish]));
        var declaredTags = new[] { RagPublicationTags.Publish, "locale:en-US" };
        var declared = await PublishAsync(
            client, await CreatePageAsync(client, $"Expliciete locale {Guid.NewGuid():N}", declaredTags));

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?limit=200");

        Assert.NotNull(manifest);
        var undeclaredItem = Assert.Single(manifest.Items, entry => entry.ContentId == undeclared.Id);
        // appsettings.json's KnowledgeHub:PrimaryLocale — never a silent "en".
        Assert.Equal("nl-BE", undeclaredItem.Locale);

        var declaredItem = Assert.Single(manifest.Items, entry => entry.ContentId == declared.Id);
        Assert.Equal("en-US", declaredItem.Locale);
    }

    [Fact]
    public async Task The_space_filter_only_returns_pages_carrying_that_exact_space_tag()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var fleetOps = await PublishAsync(
            client,
            await CreatePageAsync(
                client, $"Fleet-ops {Guid.NewGuid():N}", [RagPublicationTags.Publish, "rag:space:fleet-ops"]));
        var otherSpace = await PublishAsync(
            client,
            await CreatePageAsync(
                client, $"Andere space {Guid.NewGuid():N}", [RagPublicationTags.Publish, "rag:space:other"]));

        var manifest = await client.GetFromJsonAsync<RagPublicationManifestResponse>(
            "/api/v1/rag/publication-manifest?space=fleet-ops&limit=200");

        Assert.NotNull(manifest);
        Assert.Contains(manifest.Items, item => item.ContentId == fleetOps.Id);
        Assert.DoesNotContain(manifest.Items, item => item.ContentId == otherSpace.Id);
    }

    [Fact]
    public async Task An_invalid_space_parameter_is_rejected_rather_than_smuggled_into_the_array_filter()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var response = await client.GetAsync("/api/v1/rag/publication-manifest?space=Not_Valid!");

        Assert.Equal(HttpStatusCode.UnprocessableEntity, response.StatusCode);
    }

    [Fact]
    public async Task The_manifest_requires_authentication()
    {
        using var client = Factory.CreateClient();

        var response = await client.GetAsync("/api/v1/rag/publication-manifest");

        Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
    }

    [Fact]
    public async Task A_bearer_token_with_only_pages_read_can_call_the_manifest()
    {
        using var adminClient = Factory.CreateClient();
        await SetupHelper.LoginAsync(adminClient, OwnerEmail, OwnerPassword);

        var createServiceAccount = await SetupHelper.PostWithCsrfAsync(
            adminClient,
            "/api/v1/service-accounts",
            new CreateServiceAccountRequest("RAGcore Knowledge Sync (test)", "Alleen-lezen synctoken.", [Scopes.PagesRead]));
        Assert.Equal(HttpStatusCode.OK, createServiceAccount.StatusCode);
        var created = await createServiceAccount.Content.ReadFromJsonAsync<CreatedServiceAccountToken>();
        Assert.NotNull(created);

        using var bearerClient = Factory.CreateClient();
        bearerClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", created.RawToken);

        var response = await bearerClient.GetAsync("/api/v1/rag/publication-manifest");

        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
    }

    [Fact]
    public async Task Cursor_pagination_is_deterministic_and_covers_every_matching_page_without_duplicates()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var created = new List<Guid>();
        for (var i = 0; i < 7; i++)
        {
            var page = await CreatePageAsync(client, $"Manifestreeks {i:D2} {Guid.NewGuid():N}", [RagPublicationTags.Publish]);
            created.Add(page.Id);
        }

        var seen = new HashSet<Guid>();
        string? cursor = null;
        var pages = 0;

        do
        {
            var url = cursor is null
                ? "/api/v1/rag/publication-manifest?limit=3"
                : $"/api/v1/rag/publication-manifest?limit=3&cursor={Uri.EscapeDataString(cursor)}";

            var response = await client.GetFromJsonAsync<RagPublicationManifestResponse>(url);
            Assert.NotNull(response);

            foreach (var item in response.Items)
            {
                Assert.True(seen.Add(item.ContentId), $"Item {item.ContentId} was returned on more than one page.");
            }

            cursor = response.NextCursor;
            pages++;
            Assert.True(pages < 50, "Pagination did not terminate within a reasonable number of pages.");
        } while (cursor is not null);

        foreach (var id in created)
        {
            Assert.Contains(id, seen);
        }
    }
}