itworx-knowledge-hub-Public/tests/backend/ItWorx.KnowledgeHub.IntegrationTests/InfrastructureKnowledge/InfrastructureCatalogScaleTests.cs

using System.Diagnostics;
using System.Net.Http.Json;
using ItWorx.KnowledgeHub.Contracts.Infrastructure;
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Npgsql;

namespace ItWorx.KnowledgeHub.IntegrationTests.InfrastructureKnowledge;

/// <summary>A large workspace's service catalog stays bounded and fast to page through
/// (V2-R2, docs/adr/ADR-026-v2-r2-infrastructure-scale-and-sync-durability.md, Finding
/// A): before this milestone, listing services always loaded all nine related fact
/// tables for every service in the workspace, unpaginated. This proves the catalog
/// endpoint now serves one bounded, correctly-cursored page regardless of how many
/// services (and related facts) exist in total — mirroring
/// <c>PageTreeScaleTests</c>'s established "bounded and fast, not a precise
/// benchmark" pattern for the knowledge tree.</summary>
[Collection(PostgresTestGroup.Name)]
public sealed class InfrastructureCatalogScaleTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
{
    private const int ServiceCount = 3_000;

    [Fact]
    public async Task Fetching_one_page_of_a_3000_service_catalog_stays_fast_and_bounded()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);

        var workspaceId = await SeedServicesDirectlyAsync();

        var stopwatch = Stopwatch.StartNew();
        var firstPage = await client.GetFromJsonAsync<InfrastructureServiceListResponse>(
            "/api/v1/infrastructure/services?limit=50");
        stopwatch.Stop();

        Assert.NotNull(firstPage);
        Assert.Equal(50, firstPage.Items.Count);
        Assert.False(string.IsNullOrEmpty(firstPage.NextCursor));
        // Every summary row must still carry the fact counts the catalog card renders,
        // even though the query behind it now fetches only counts, not full related rows.
        Assert.All(firstPage.Items, item =>
        {
            Assert.Equal(2, item.ComponentCount);
            Assert.Equal(1, item.EndpointCount);
            Assert.Equal(1, item.VolumeCount);
            Assert.Equal(1, item.DatabaseCount);
        });

        // Generous bound for a CI/dev machine: the point is "bounded and fast",
        // regardless of the {ServiceCount} total services and their ~5x related facts.
        Assert.True(
            stopwatch.Elapsed < TimeSpan.FromSeconds(5),
            $"Fetching one bounded page of services took {stopwatch.Elapsed}, expected well under 5s at {ServiceCount} total services.");

        var secondPage = await client.GetFromJsonAsync<InfrastructureServiceListResponse>(
            $"/api/v1/infrastructure/services?limit=50&cursor={Uri.EscapeDataString(firstPage.NextCursor!)}");
        Assert.NotNull(secondPage);
        Assert.Equal(50, secondPage.Items.Count);
        Assert.Empty(firstPage.Items.Select(item => item.Id).Intersect(secondPage.Items.Select(item => item.Id)));

        _ = workspaceId;
    }

    private async Task<Guid> SeedServicesDirectlyAsync()
    {
        // Seeded through a handful of server-side INSERT...SELECT statements (one round
        // trip per table, not one per row or per fact) — the same generate_series
        // pattern SearchFullScaleTests already established for bulk-seeding at scale.
        using var scope = Factory.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
        var workspace = await db.Workspaces.AsNoTracking().FirstAsync();
        var workspaceId = workspace.Id;

        var connectionOptions = scope.ServiceProvider.GetRequiredService<DatabaseConnectionOptions>();
        await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
        await connection.OpenAsync();

        var snapshotId = Guid.NewGuid();

        const string servicesSql = """
            INSERT INTO infrastructure_services (
                id, workspace_id, host_id, source_key, detected_name, runtime_kind, runtime_status,
                health, image_reference, image_digest, runtime_user, read_only_root_filesystem,
                no_new_privileges, privileged, presence, confidence, last_snapshot_id, first_seen_at, last_seen_at)
            SELECT
                gen_random_uuid(), @workspaceId, NULL,
                'docker:schaal-service-' || n, 'schaal-service-' || lpad(n::text, 5, '0'),
                'container', 'running', 1, 'itworx/schaal:' || n, '', '', false, false, false,
                1, 2, @snapshotId, now(), now()
            FROM generate_series(1, @serviceCount) AS n
            """;
        await using (var command = new NpgsqlCommand(servicesSql, connection) { CommandTimeout = 120 })
        {
            command.Parameters.AddWithValue("workspaceId", workspaceId);
            command.Parameters.AddWithValue("snapshotId", snapshotId);
            command.Parameters.AddWithValue("serviceCount", ServiceCount);
            await command.ExecuteNonQueryAsync();
        }

        const string componentsSql = """
            INSERT INTO service_components (
                id, workspace_id, service_id, source_key, confidence, last_snapshot_id, observed_at,
                name, kind, runtime_id, status, image_reference)
            SELECT
                gen_random_uuid(), @workspaceId, s.id, 'container:' || c.n, 2, @snapshotId, now(),
                'component-' || c.n, 'container', 'rt-' || c.n, 'running', ''
            FROM infrastructure_services s
            CROSS JOIN generate_series(1, 2) AS c(n)
            WHERE s.workspace_id = @workspaceId
            """;
        await using (var command = new NpgsqlCommand(componentsSql, connection) { CommandTimeout = 120 })
        {
            command.Parameters.AddWithValue("workspaceId", workspaceId);
            command.Parameters.AddWithValue("snapshotId", snapshotId);
            await command.ExecuteNonQueryAsync();
        }

        const string endpointsSql = """
            INSERT INTO infrastructure_endpoints (
                id, workspace_id, service_id, source_key, confidence, last_snapshot_id, observed_at,
                scheme, host, published_port, target_port, url, is_public)
            SELECT
                gen_random_uuid(), @workspaceId, s.id, 'port:main', 2, @snapshotId, now(),
                'http', '', NULL, NULL, '', false
            FROM infrastructure_services s
            WHERE s.workspace_id = @workspaceId
            """;
        await using (var command = new NpgsqlCommand(endpointsSql, connection) { CommandTimeout = 120 })
        {
            command.Parameters.AddWithValue("workspaceId", workspaceId);
            command.Parameters.AddWithValue("snapshotId", snapshotId);
            await command.ExecuteNonQueryAsync();
        }

        const string volumesSql = """
            INSERT INTO infrastructure_volumes (
                id, workspace_id, service_id, source_key, confidence, last_snapshot_id, observed_at,
                name, mount_path, storage_class, read_only, size_bytes)
            SELECT
                gen_random_uuid(), @workspaceId, s.id, 'volume:main', 2, @snapshotId, now(),
                'data', '/data', '', false, NULL
            FROM infrastructure_services s
            WHERE s.workspace_id = @workspaceId
            """;
        await using (var command = new NpgsqlCommand(volumesSql, connection) { CommandTimeout = 120 })
        {
            command.Parameters.AddWithValue("workspaceId", workspaceId);
            command.Parameters.AddWithValue("snapshotId", snapshotId);
            await command.ExecuteNonQueryAsync();
        }

        const string databasesSql = """
            INSERT INTO infrastructure_databases (
                id, workspace_id, service_id, source_key, confidence, last_snapshot_id, observed_at,
                name, engine, version, host, port, host_port_published)
            SELECT
                gen_random_uuid(), @workspaceId, s.id, 'database:main', 2, @snapshotId, now(),
                'main', 'postgresql', '17', '', NULL, false
            FROM infrastructure_services s
            WHERE s.workspace_id = @workspaceId
            """;
        await using (var command = new NpgsqlCommand(databasesSql, connection) { CommandTimeout = 120 })
        {
            command.Parameters.AddWithValue("workspaceId", workspaceId);
            command.Parameters.AddWithValue("snapshotId", snapshotId);
            await command.ExecuteNonQueryAsync();
        }

        return workspaceId;
    }
}