436 lines
21 KiB
C#
436 lines
21 KiB
C#
using System.Diagnostics;
|
|
using System.Globalization;
|
|
using System.Net.Http.Json;
|
|
using System.Text;
|
|
using ItWorx.KnowledgeHub.Contracts.Knowledge;
|
|
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
|
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Npgsql;
|
|
|
|
namespace ItWorx.KnowledgeHub.IntegrationTests.Knowledge;
|
|
|
|
/// <summary>
|
|
/// The full-scale proof docs/10-search-and-indexing.md's "Scale verification" section
|
|
/// describes and ADR-012 decision 7 deferred to M8: 100,000 current pages and
|
|
/// 1,000,000 revisions, with captured p50/p95/p99 latency, query plans, index sizes,
|
|
/// result correctness and a no-leak assertion. Tagged "ReleaseScale" and excluded from
|
|
/// the routine test run (scripts/verify.mjs filters it out by default) — seeding and
|
|
/// measuring at this scale is a deliberate, occasional release-gate exercise, not a
|
|
/// per-commit check. Run explicitly via:
|
|
/// <c>dotnet test --filter "Category=ReleaseScale"</c>.
|
|
/// Data is generated directly in PostgreSQL via <c>generate_series</c> (a single
|
|
/// server-side INSERT...SELECT per table), not one round trip per row — the only way
|
|
/// 1.1 million rows seed in a runtime proportionate to a release-gate step rather than
|
|
/// a dedicated benchmarking rig.
|
|
/// </summary>
|
|
[Collection(PostgresTestGroup.Name)]
|
|
[Trait("Category", "ReleaseScale")]
|
|
public sealed class SearchFullScaleTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
|
|
{
|
|
private const int PageCount = 100_000;
|
|
private const int RevisionsPerPage = 10; // 100,000 * 10 = 1,000,000 revisions.
|
|
private const int SampleCount = 30;
|
|
|
|
[Fact]
|
|
public async Task Search_and_tree_reads_stay_correct_and_bounded_at_the_documented_full_scale()
|
|
{
|
|
using var client = Factory.CreateClient();
|
|
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
|
|
|
var report = new StringBuilder();
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"M8 full-scale search/tree report — {DateTimeOffset.UtcNow:O}");
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"Target: {PageCount:N0} current pages, {PageCount * RevisionsPerPage:N0} revisions (docs/10-search-and-indexing.md).");
|
|
|
|
Guid workspaceId;
|
|
Guid needleId;
|
|
var seedStopwatch = Stopwatch.StartNew();
|
|
using (var scope = Factory.Services.CreateScope())
|
|
{
|
|
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
|
var workspace = await db.Workspaces.AsNoTracking().FirstAsync();
|
|
workspaceId = workspace.Id;
|
|
|
|
var connectionOptions = scope.ServiceProvider.GetRequiredService<ItWorx.KnowledgeHub.Infrastructure.Persistence.DatabaseConnectionOptions>();
|
|
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
|
await connection.OpenAsync();
|
|
|
|
needleId = await SeedPagesAsync(connection, workspaceId);
|
|
await SeedRevisionsAsync(connection, workspaceId);
|
|
|
|
// A deliberately separate, inaccessible-from-workspace-1 workspace with a
|
|
// rare term in its title — proves no cross-workspace leak at scale
|
|
// (docs/10-search-and-indexing.md "Tests must include private pages whose
|
|
// titles and rare terms would reveal their existence").
|
|
await SeedOtherWorkspaceLeakCheckAsync(connection);
|
|
|
|
// The release measurement represents steady-state reads after an import,
|
|
// not PostgreSQL's concurrent statistics, autovacuum or checkpoint work
|
|
// immediately after 1.1 million bulk inserts. Make that boundary explicit
|
|
// so host IO timing cannot decide which of the 30 samples cross the
|
|
// unchanged 1s p95 gate. This does not warm the measured HTTP queries or
|
|
// weaken their dataset, sample count or latency bound.
|
|
await PrepareReadMeasurementAsync(connection);
|
|
}
|
|
|
|
seedStopwatch.Stop();
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"Seed time: {seedStopwatch.Elapsed}.");
|
|
|
|
using (var scope = Factory.Services.CreateScope())
|
|
{
|
|
// These counts validate the completed 1.1-million-row seed; they are not
|
|
// part of the measured HTTP read latency below. Use the same explicit
|
|
// release-scale command timeout as the bulk seed/ANALYZE operations.
|
|
// EF's normal 30-second application timeout can expire while Docker
|
|
// Desktop is still flushing the just-seeded relation, which makes host IO
|
|
// timing fail setup before any actual search performance is measured.
|
|
var connectionOptions = scope.ServiceProvider.GetRequiredService<ItWorx.KnowledgeHub.Infrastructure.Persistence.DatabaseConnectionOptions>();
|
|
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
|
await connection.OpenAsync();
|
|
await using var countCommand = new NpgsqlCommand(
|
|
"SELECT (SELECT count(*) FROM pages), (SELECT count(*) FROM page_revisions)",
|
|
connection)
|
|
{
|
|
CommandTimeout = 300
|
|
};
|
|
await using var reader = await countCommand.ExecuteReaderAsync();
|
|
Assert.True(await reader.ReadAsync(), "Release-scale row-count query returned no row.");
|
|
var actualPages = reader.GetInt64(0);
|
|
var actualRevisions = reader.GetInt64(1);
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"Actual seeded rows: pages={actualPages:N0}, page_revisions={actualRevisions:N0}.");
|
|
Assert.True(actualPages >= PageCount, $"Expected at least {PageCount:N0} pages, found {actualPages:N0}.");
|
|
Assert.True(actualRevisions >= PageCount * (long)RevisionsPerPage, $"Expected at least {PageCount * RevisionsPerPage:N0} revisions, found {actualRevisions:N0}.");
|
|
}
|
|
|
|
// --- correctness: the needle is found, and nothing from the other workspace leaks ---
|
|
var needleResponse = await client.GetFromJsonAsync<SearchResponse>("/api/v1/search?q=xyzzyplugh");
|
|
Assert.NotNull(needleResponse);
|
|
Assert.Contains(needleResponse.Items, item => item.Id == needleId);
|
|
|
|
var leakResponse = await client.GetFromJsonAsync<SearchResponse>("/api/v1/search?q=quibblefrastication");
|
|
Assert.NotNull(leakResponse);
|
|
Assert.Empty(leakResponse.Items);
|
|
report.AppendLine("Correctness: needle term found; other-workspace rare term does not leak. PASS.");
|
|
|
|
var selectiveTagResponse = await client.GetFromJsonAsync<SearchResponse>(
|
|
"/api/v1/search?q=documentatie&tag=selective-scale-test");
|
|
Assert.NotNull(selectiveTagResponse);
|
|
Assert.NotEmpty(selectiveTagResponse.Items);
|
|
Assert.All(selectiveTagResponse.Items, item => Assert.Contains("selective-scale-test", item.Tags));
|
|
report.AppendLine("Correctness: selective tag returns only tagged pages. PASS.");
|
|
|
|
// --- latency: p50/p95/p99 across representative query shapes ---
|
|
report.AppendLine();
|
|
report.AppendLine("Latency (ms), 30 samples per query shape:");
|
|
var measurements = new[]
|
|
{
|
|
await MeasureAsync(client, report, "common term (\"documentatie\")", "/api/v1/search?q=documentatie"),
|
|
await MeasureAsync(client, report, "rare needle term", "/api/v1/search?q=xyzzyplugh"),
|
|
await MeasureAsync(client, report, "typo-tolerant title (trigram)", "/api/v1/search?q=documentaite"),
|
|
await MeasureAsync(client, report, "tag filter (all pages carry the tag)", "/api/v1/search?q=documentatie&tag=scale-test"),
|
|
await MeasureAsync(client, report, "selective tag filter", "/api/v1/search?q=documentatie&tag=selective-scale-test")
|
|
};
|
|
|
|
// --- query plan + index sizes ---
|
|
using (var scope = Factory.Services.CreateScope())
|
|
{
|
|
var connectionOptions = scope.ServiceProvider.GetRequiredService<ItWorx.KnowledgeHub.Infrastructure.Persistence.DatabaseConnectionOptions>();
|
|
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
|
await connection.OpenAsync();
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the common-term search:");
|
|
report.AppendLine(await ExplainAsync(connection, workspaceId, "documentatie"));
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the typo-only trigram search:");
|
|
report.AppendLine(await ExplainFuzzyAsync(connection, workspaceId, "documentaite"));
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the broad tag-filter candidate scan:");
|
|
report.AppendLine(await ExplainTagFilterAsync(connection, workspaceId, "documentatie", "scale-test"));
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the selective tag-filter candidate scan:");
|
|
var selectiveTagPlan = await ExplainTagFilterAsync(connection, workspaceId, "documentatie", "selective-scale-test");
|
|
report.AppendLine(selectiveTagPlan);
|
|
Assert.Contains("ix_pages_tags", selectiveTagPlan, StringComparison.Ordinal);
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Index sizes:");
|
|
report.AppendLine(await IndexSizesAsync(connection, "pages"));
|
|
}
|
|
|
|
var reportPath = System.IO.Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", "artifacts", "m8-scale-report.txt");
|
|
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(reportPath)!);
|
|
await System.IO.File.WriteAllTextAsync(reportPath, report.ToString());
|
|
|
|
foreach (var measurement in measurements)
|
|
{
|
|
// Generous release-gate bound — proving the indexed query stays
|
|
// sub-second at true documented scale, not a tight micro-benchmark.
|
|
Assert.True(
|
|
measurement.P95 < 1000,
|
|
$"{measurement.Label}: p95 was {measurement.P95:F1}ms, expected well under 1000ms at {PageCount:N0}-page scale.");
|
|
}
|
|
}
|
|
|
|
private static async Task<Guid> SeedPagesAsync(NpgsqlConnection connection, Guid workspaceId)
|
|
{
|
|
var needleId = Guid.NewGuid();
|
|
|
|
const string sql = """
|
|
INSERT INTO pages (
|
|
id, workspace_id, parent_page_id, sort_key, title, slug, markdown,
|
|
document_type, document_status, current_revision_number, content_hash,
|
|
created_by, updated_by, created_at, updated_at, tags)
|
|
SELECT
|
|
CASE WHEN n = 1 THEN @needleId::uuid ELSE gen_random_uuid() END,
|
|
@workspaceId,
|
|
NULL,
|
|
lpad(n::text, 12, '0'),
|
|
CASE WHEN n = 1 THEN 'Uniek vindbaar trefwoord xyzzyplugh'
|
|
ELSE 'Testpagina ' || n || ' ' ||
|
|
(CASE WHEN n % 2 = 0 THEN 'documentatie en projecten' ELSE 'technical procedures and code' END)
|
|
END,
|
|
'schaal-pagina-' || n,
|
|
CASE WHEN n = 1
|
|
THEN 'Deze pagina bevat het unieke trefwoord xyzzyplugh nergens anders vermeld.'
|
|
ELSE repeat(
|
|
'Gewone opvultekst met Nederlandse en Engelse woorden door elkaar over documentatie, projecten, code en procedures. ',
|
|
5)
|
|
END,
|
|
0,
|
|
1,
|
|
1,
|
|
md5(n::text),
|
|
NULL,
|
|
NULL,
|
|
now() - (n || ' seconds')::interval,
|
|
now(),
|
|
CASE WHEN n % 1000 = 0
|
|
THEN ARRAY['scale-test', 'selective-scale-test']::text[]
|
|
ELSE ARRAY['scale-test']::text[]
|
|
END
|
|
FROM generate_series(1, @pageCount) AS n
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
|
|
command.Parameters.AddWithValue("needleId", needleId);
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
command.Parameters.AddWithValue("pageCount", PageCount);
|
|
await command.ExecuteNonQueryAsync();
|
|
|
|
return needleId;
|
|
}
|
|
|
|
private static async Task SeedRevisionsAsync(NpgsqlConnection connection, Guid workspaceId)
|
|
{
|
|
const string sql = """
|
|
INSERT INTO page_revisions (
|
|
id, page_id, revision_number, title, slug, markdown, parent_page_id,
|
|
sort_key, document_type, document_status, content_hash, authored_by,
|
|
change_summary, created_at)
|
|
SELECT
|
|
gen_random_uuid(), p.id, r.rn, p.title, p.slug, p.markdown, p.parent_page_id,
|
|
p.sort_key, p.document_type, p.document_status, p.content_hash, NULL, NULL,
|
|
p.created_at + (r.rn || ' minutes')::interval
|
|
FROM pages p
|
|
CROSS JOIN generate_series(1, @revisionsPerPage) AS r(rn)
|
|
WHERE p.workspace_id = @workspaceId
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
command.Parameters.AddWithValue("revisionsPerPage", RevisionsPerPage);
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
private static async Task SeedOtherWorkspaceLeakCheckAsync(NpgsqlConnection connection)
|
|
{
|
|
const string sql = """
|
|
INSERT INTO workspaces (id, name, slug, settings_version, created_at)
|
|
VALUES (@workspaceId, 'Andere werkruimte (lekcontrole)', 'andere-werkruimte-lekcontrole', 1, now());
|
|
|
|
INSERT INTO pages (
|
|
id, workspace_id, parent_page_id, sort_key, title, slug, markdown,
|
|
document_type, document_status, current_revision_number, content_hash,
|
|
created_by, updated_by, created_at, updated_at, tags)
|
|
VALUES (
|
|
gen_random_uuid(), @workspaceId, NULL, '000000000001',
|
|
'Ontoegankelijke pagina quibblefrastication',
|
|
'ontoegankelijke-pagina', 'Bevat de zeldzame term quibblefrastication.',
|
|
0, 1, 1, md5('other-workspace'), NULL, NULL, now(), now(), ARRAY[]::text[])
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection);
|
|
command.Parameters.AddWithValue("workspaceId", Guid.NewGuid());
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
private static async Task PrepareReadMeasurementAsync(NpgsqlConnection connection)
|
|
{
|
|
// Run each VACUUM outside an explicit/multi-statement transaction. Both tables
|
|
// are bulk-loaded by this fixture; leaving page_revisions for autovacuum can
|
|
// create the same disk contention even though the search query reads pages.
|
|
foreach (var statement in new[]
|
|
{
|
|
"VACUUM (ANALYZE) pages;",
|
|
"VACUUM (ANALYZE) page_revisions;",
|
|
"CHECKPOINT;"
|
|
})
|
|
{
|
|
await using var command = new NpgsqlCommand(statement, connection)
|
|
{
|
|
CommandTimeout = 300
|
|
};
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
}
|
|
|
|
private static async Task<LatencyMeasurement> MeasureAsync(HttpClient client, StringBuilder report, string label, string path)
|
|
{
|
|
var samples = new List<double>(SampleCount);
|
|
for (var i = 0; i < SampleCount; i++)
|
|
{
|
|
var stopwatch = Stopwatch.StartNew();
|
|
using var response = await client.GetAsync(path);
|
|
stopwatch.Stop();
|
|
response.EnsureSuccessStatusCode();
|
|
samples.Add(stopwatch.Elapsed.TotalMilliseconds);
|
|
}
|
|
|
|
samples.Sort();
|
|
var p50 = Percentile(samples, 0.50);
|
|
var p95 = Percentile(samples, 0.95);
|
|
var p99 = Percentile(samples, 0.99);
|
|
report.AppendLine(CultureInfo.InvariantCulture, $" {label}: p50={p50:F1}ms p95={p95:F1}ms p99={p99:F1}ms (min={samples[0]:F1}ms max={samples[^1]:F1}ms)");
|
|
|
|
return new LatencyMeasurement(label, p95);
|
|
}
|
|
|
|
private static double Percentile(IReadOnlyList<double> sortedSamples, double percentile)
|
|
{
|
|
var index = (int)Math.Ceiling(percentile * sortedSamples.Count) - 1;
|
|
return sortedSamples[Math.Clamp(index, 0, sortedSamples.Count - 1)];
|
|
}
|
|
|
|
private static async Task<string> ExplainAsync(NpgsqlConnection connection, Guid workspaceId, string term)
|
|
{
|
|
const string sql = """
|
|
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
|
|
SELECT id, title
|
|
FROM pages
|
|
WHERE workspace_id = @workspaceId
|
|
AND trashed_at IS NULL
|
|
AND search_vector @@ plainto_tsquery('simple', @term)
|
|
ORDER BY ts_rank_cd(search_vector, plainto_tsquery('simple', @term)) DESC
|
|
LIMIT 20
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection);
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
command.Parameters.AddWithValue("term", term);
|
|
await using var reader = await command.ExecuteReaderAsync();
|
|
var lines = new StringBuilder();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
lines.AppendLine(" " + reader.GetString(0));
|
|
}
|
|
|
|
return lines.ToString();
|
|
}
|
|
|
|
private static async Task<string> ExplainFuzzyAsync(NpgsqlConnection connection, Guid workspaceId, string term)
|
|
{
|
|
await using var transaction = await connection.BeginTransactionAsync();
|
|
await using (var threshold = new NpgsqlCommand("SET LOCAL pg_trgm.similarity_threshold = '0.2';", connection, transaction))
|
|
{
|
|
await threshold.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
const string sql = """
|
|
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
|
|
SELECT id, title
|
|
FROM pages
|
|
WHERE workspace_id = @workspaceId
|
|
AND trashed_at IS NULL
|
|
AND title % @term
|
|
ORDER BY similarity(title, @term) DESC, updated_at DESC, id DESC
|
|
LIMIT 20
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection, transaction);
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
command.Parameters.AddWithValue("term", term);
|
|
var lines = new StringBuilder();
|
|
{
|
|
await using var reader = await command.ExecuteReaderAsync();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
lines.AppendLine(" " + reader.GetString(0));
|
|
}
|
|
}
|
|
|
|
await transaction.CommitAsync();
|
|
return lines.ToString();
|
|
}
|
|
|
|
private static async Task<string> ExplainTagFilterAsync(
|
|
NpgsqlConnection connection,
|
|
Guid workspaceId,
|
|
string term,
|
|
string tag)
|
|
{
|
|
const string sql = """
|
|
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
|
|
SELECT p.id, p.updated_at,
|
|
ts_rank_cd(p.search_vector, plainto_tsquery('simple', @term))::double precision AS rank
|
|
FROM pages AS p
|
|
WHERE p.workspace_id = @workspaceId
|
|
AND p.trashed_at IS NULL
|
|
AND p.search_vector @@ plainto_tsquery('simple', @term)
|
|
AND p.tags @> ARRAY[@tag]::text[]
|
|
ORDER BY rank DESC, p.updated_at DESC, p.id DESC
|
|
LIMIT 20
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection);
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
command.Parameters.AddWithValue("term", term);
|
|
command.Parameters.AddWithValue("tag", tag);
|
|
await using var reader = await command.ExecuteReaderAsync();
|
|
var lines = new StringBuilder();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
lines.AppendLine(" " + reader.GetString(0));
|
|
}
|
|
|
|
return lines.ToString();
|
|
}
|
|
|
|
private static async Task<string> IndexSizesAsync(NpgsqlConnection connection, string tableName)
|
|
{
|
|
const string sql = """
|
|
SELECT indexname, pg_size_pretty(pg_relation_size(indexname::regclass))
|
|
FROM pg_indexes
|
|
WHERE tablename = @tableName
|
|
ORDER BY indexname
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection);
|
|
command.Parameters.AddWithValue("tableName", tableName);
|
|
await using var reader = await command.ExecuteReaderAsync();
|
|
var lines = new StringBuilder();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
lines.AppendLine(CultureInfo.InvariantCulture, $" {reader.GetString(0)}: {reader.GetString(1)}");
|
|
}
|
|
|
|
return lines.ToString();
|
|
}
|
|
|
|
private sealed record LatencyMeasurement(string Label, double P95);
|
|
}
|