itworx-knowledge-hub-Public/tests/backend/ItWorx.KnowledgeHub.IntegrationTests/UnifiedSearch/UnifiedSearchScaleTests.cs

using System.Diagnostics;
using System.Globalization;
using System.Net.Http.Json;
using System.Text;
using ItWorx.KnowledgeHub.Contracts.UnifiedSearch;
using ItWorx.KnowledgeHub.Domain.Knowledge;
using ItWorx.KnowledgeHub.Domain.Projects;
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
using ItWorx.KnowledgeHub.Infrastructure.UnifiedSearch;
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using SortKey = ItWorx.KnowledgeHub.Domain.Knowledge.SortKey;

namespace ItWorx.KnowledgeHub.IntegrationTests.UnifiedSearch;

[Collection(PostgresTestGroup.Name)]
[Trait("Category", "ReleaseScale")]
public sealed class UnifiedSearchScaleTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
{
    [Fact]
    public async Task Unified_search_counts_and_cursor_pages_stay_bounded_with_10000_pages_and_2000_projects()
    {
        using var client = Factory.CreateClient();
        await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
        using var scope = Factory.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
        var workspaceId = await db.Workspaces.Select(item => item.Id).SingleAsync();
        var actorId = await db.Users.Select(item => item.Id).SingleAsync();
        var now = DateTimeOffset.UtcNow;
        var sortKey = SortKey.First();

        for (var offset = 0; offset < 10_000; offset += 500)
        {
            for (var index = offset; index < offset + 500; index++)
            {
                var title = index == 0 ? "Measure uniqueneedle page" : $"Measure page {index:D5}";
                var page = Page.Create(workspaceId, null, sortKey, PageTitle.Create(title), DocumentType.WikiPage,
                    "Measure synthetic documentation with a bounded result snippet.", actorId, now);
                page.ReplaceTags(["platform-scale"]);
                db.Pages.Add(page);
                sortKey = SortKey.Next(sortKey);
            }
            await db.SaveChangesAsync();
            db.ChangeTracker.Clear();
        }

        for (var offset = 0; offset < 2_000; offset += 500)
        {
            for (var index = offset; index < offset + 500; index++)
            {
                var title = index == 0 ? "Measure uniqueneedle project" : $"Measure project {index:D5}";
                db.Projects.Add(Project.Create(workspaceId, title, "Measure synthetic project summary.",
                    ProjectPriority.Medium, "Measure project state.", actorId, actorId, now));
            }
            await db.SaveChangesAsync();
            db.ChangeTracker.Clear();
        }
        await db.Database.ExecuteSqlRawAsync("ANALYZE pages; ANALYZE projects;");

        var repository = new UnifiedSearchRepository(db);
        var first = await repository.SearchAsync(workspaceId, "measure", null, null, null, 50, CancellationToken.None);
        Assert.Equal(10_000, first.Counts.Knowledge);
        Assert.Equal(2_000, first.Counts.Projects);
        Assert.Equal(50, first.Items.Count);
        Assert.NotNull(first.NextCursor);
        var next = await repository.SearchAsync(workspaceId, "measure", null, null, first.NextCursor, 50, CancellationToken.None);
        Assert.Equal(first.Counts, next.Counts);
        Assert.Equal(50, next.Items.Count);
        Assert.Empty(first.Items.Select(item => item.Id).Intersect(next.Items.Select(item => item.Id)));

        var report = new StringBuilder();
        report.AppendLine(CultureInfo.InvariantCulture, $"Unified search scale — {DateTimeOffset.UtcNow:O}; 10000 pages, 2000 projects; 30 warm samples per query.");
        foreach (var query in new[] { "measure", "uniqueneedle" })
        {
            var samples = new List<double>();
            for (var sample = 0; sample < 30; sample++)
            {
                var watch = Stopwatch.StartNew();
                var result = await repository.SearchAsync(workspaceId, query, null, null, null, 50, CancellationToken.None);
                watch.Stop();
                Assert.Equal(query == "measure" ? 10_000 : 1, result.Counts.Knowledge);
                Assert.Equal(query == "measure" ? 2_000 : 1, result.Counts.Projects);
                Assert.InRange(result.Items.Count, 1, 50);
                samples.Add(watch.Elapsed.TotalMilliseconds);
            }
            samples.Sort();
            var p95 = samples[28];
            report.AppendLine(CultureInfo.InvariantCulture, $"{query}: p50={samples[14]:F2}ms p95={p95:F2}ms max={samples[^1]:F2}ms");
            // Same broad correctness/latency budget as existing scale gates; this
            // exercises the new grouped-count query rather than the legacy search.
            Assert.True(p95 < 3_000, $"Unified search p95 for '{query}' was {p95:F2}ms (budget 3000ms).");
        }

        using var response = await client.GetAsync("/api/v1/unified-search?q=measure&limit=50");
        response.EnsureSuccessStatusCode();
        var bytes = await response.Content.ReadAsByteArrayAsync();
        var body = await response.Content.ReadFromJsonAsync<UnifiedSearchResponse>();
        Assert.NotNull(body);
        Assert.Equal(50, body.Items.Count);
        Assert.Equal(12_000, body.Counts.Total);
        Assert.True(bytes.Length < 100_000, $"One 50-item search page contained {bytes.Length} bytes.");
        report.AppendLine(CultureInfo.InvariantCulture, $"HTTP 50-item response: {bytes.Length} bytes; total=12000; no duplicate IDs across cursor pages.");
        Directory.CreateDirectory("artifacts/performance");
        await File.WriteAllTextAsync("artifacts/performance/platform-unified-search-scale.txt", report.ToString());
    }
}