Jens/itworx-knowledge-hub-Public
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial curated itworx-knowledge-hub source from 3b1751d0d9dee551aa6994eee0093704dc334817

Browse Source
This commit is contained in:
Public Source Publisher committed 2026-10-01 01:37:37 +02:00
commit ad8ad252a9
825 files changed
+124216
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/backend/ItWorx.KnowledgeHub.IntegrationTests/Knowledge/SearchFullScaleTests.cs
+435
View File
@@ -0,0 +1,435 @@
using System.Diagnostics;
using System.Globalization;
using System.Net.Http.Json;
using System.Text;
using ItWorx.KnowledgeHub.Contracts.Knowledge;
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Npgsql;
namespace ItWorx.KnowledgeHub.IntegrationTests.Knowledge;
/// <summary>
/// The full-scale proof docs/10-search-and-indexing.md's "Scale verification" section
/// describes and ADR-012 decision 7 deferred to M8: 100,000 current pages and
/// 1,000,000 revisions, with captured p50/p95/p99 latency, query plans, index sizes,
/// result correctness and a no-leak assertion. Tagged "ReleaseScale" and excluded from
/// the routine test run (scripts/verify.mjs filters it out by default) — seeding and
/// measuring at this scale is a deliberate, occasional release-gate exercise, not a
/// per-commit check. Run explicitly via:
/// <c>dotnet test --filter "Category=ReleaseScale"</c>.
/// Data is generated directly in PostgreSQL via <c>generate_series</c> (a single
/// server-side INSERT...SELECT per table), not one round trip per row — the only way
/// 1.1 million rows seed in a runtime proportionate to a release-gate step rather than
/// a dedicated benchmarking rig.
/// </summary>
[Collection(PostgresTestGroup.Name)]
[Trait("Category", "ReleaseScale")]
public sealed class SearchFullScaleTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
{
private const int PageCount = 100_000;
private const int RevisionsPerPage = 10; // 100,000 * 10 = 1,000,000 revisions.
private const int SampleCount = 30;
[Fact]
public async Task Search_and_tree_reads_stay_correct_and_bounded_at_the_documented_full_scale()
{
using var client = Factory.CreateClient();
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
var report = new StringBuilder();
report.AppendLine(CultureInfo.InvariantCulture, $"M8 full-scale search/tree report — {DateTimeOffset.UtcNow:O}");
report.AppendLine(CultureInfo.InvariantCulture, $"Target: {PageCount:N0} current pages, {PageCount * RevisionsPerPage:N0} revisions (docs/10-search-and-indexing.md).");
Guid workspaceId;
Guid needleId;
var seedStopwatch = Stopwatch.StartNew();
using (var scope = Factory.Services.CreateScope())
{
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
var workspace = await db.Workspaces.AsNoTracking().FirstAsync();
workspaceId = workspace.Id;
var connectionOptions = scope.ServiceProvider.GetRequiredService<ItWorx.KnowledgeHub.Infrastructure.Persistence.DatabaseConnectionOptions>();
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
await connection.OpenAsync();
needleId = await SeedPagesAsync(connection, workspaceId);
await SeedRevisionsAsync(connection, workspaceId);
// A deliberately separate, inaccessible-from-workspace-1 workspace with a
// rare term in its title — proves no cross-workspace leak at scale
// (docs/10-search-and-indexing.md "Tests must include private pages whose
// titles and rare terms would reveal their existence").
await SeedOtherWorkspaceLeakCheckAsync(connection);
// The release measurement represents steady-state reads after an import,
// not PostgreSQL's concurrent statistics, autovacuum or checkpoint work
// immediately after 1.1 million bulk inserts. Make that boundary explicit
// so host IO timing cannot decide which of the 30 samples cross the
// unchanged 1s p95 gate. This does not warm the measured HTTP queries or
// weaken their dataset, sample count or latency bound.
await PrepareReadMeasurementAsync(connection);
}
seedStopwatch.Stop();
report.AppendLine(CultureInfo.InvariantCulture, $"Seed time: {seedStopwatch.Elapsed}.");
using (var scope = Factory.Services.CreateScope())
{
// These counts validate the completed 1.1-million-row seed; they are not
// part of the measured HTTP read latency below. Use the same explicit
// release-scale command timeout as the bulk seed/ANALYZE operations.
// EF's normal 30-second application timeout can expire while Docker
// Desktop is still flushing the just-seeded relation, which makes host IO
// timing fail setup before any actual search performance is measured.
var connectionOptions = scope.ServiceProvider.GetRequiredService<ItWorx.KnowledgeHub.Infrastructure.Persistence.DatabaseConnectionOptions>();
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
await connection.OpenAsync();
await using var countCommand = new NpgsqlCommand(
"SELECT (SELECT count(*) FROM pages), (SELECT count(*) FROM page_revisions)",
connection)
{
CommandTimeout = 300
};
await using var reader = await countCommand.ExecuteReaderAsync();
Assert.True(await reader.ReadAsync(), "Release-scale row-count query returned no row.");
var actualPages = reader.GetInt64(0);
var actualRevisions = reader.GetInt64(1);
report.AppendLine(CultureInfo.InvariantCulture, $"Actual seeded rows: pages={actualPages:N0}, page_revisions={actualRevisions:N0}.");
Assert.True(actualPages >= PageCount, $"Expected at least {PageCount:N0} pages, found {actualPages:N0}.");
Assert.True(actualRevisions >= PageCount * (long)RevisionsPerPage, $"Expected at least {PageCount * RevisionsPerPage:N0} revisions, found {actualRevisions:N0}.");
}
// --- correctness: the needle is found, and nothing from the other workspace leaks ---
var needleResponse = await client.GetFromJsonAsync<SearchResponse>("/api/v1/search?q=xyzzyplugh");
Assert.NotNull(needleResponse);
Assert.Contains(needleResponse.Items, item => item.Id == needleId);
var leakResponse = await client.GetFromJsonAsync<SearchResponse>("/api/v1/search?q=quibblefrastication");
Assert.NotNull(leakResponse);
Assert.Empty(leakResponse.Items);
report.AppendLine("Correctness: needle term found; other-workspace rare term does not leak. PASS.");
var selectiveTagResponse = await client.GetFromJsonAsync<SearchResponse>(
"/api/v1/search?q=documentatie&tag=selective-scale-test");
Assert.NotNull(selectiveTagResponse);
Assert.NotEmpty(selectiveTagResponse.Items);
Assert.All(selectiveTagResponse.Items, item => Assert.Contains("selective-scale-test", item.Tags));
report.AppendLine("Correctness: selective tag returns only tagged pages. PASS.");
// --- latency: p50/p95/p99 across representative query shapes ---
report.AppendLine();
report.AppendLine("Latency (ms), 30 samples per query shape:");
var measurements = new[]
{
await MeasureAsync(client, report, "common term (\"documentatie\")", "/api/v1/search?q=documentatie"),
await MeasureAsync(client, report, "rare needle term", "/api/v1/search?q=xyzzyplugh"),
await MeasureAsync(client, report, "typo-tolerant title (trigram)", "/api/v1/search?q=documentaite"),
await MeasureAsync(client, report, "tag filter (all pages carry the tag)", "/api/v1/search?q=documentatie&tag=scale-test"),
await MeasureAsync(client, report, "selective tag filter", "/api/v1/search?q=documentatie&tag=selective-scale-test")
};
// --- query plan + index sizes ---
using (var scope = Factory.Services.CreateScope())
{
var connectionOptions = scope.ServiceProvider.GetRequiredService<ItWorx.KnowledgeHub.Infrastructure.Persistence.DatabaseConnectionOptions>();
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
await connection.OpenAsync();
report.AppendLine();
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the common-term search:");
report.AppendLine(await ExplainAsync(connection, workspaceId, "documentatie"));
report.AppendLine();
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the typo-only trigram search:");
report.AppendLine(await ExplainFuzzyAsync(connection, workspaceId, "documentaite"));
report.AppendLine();
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the broad tag-filter candidate scan:");
report.AppendLine(await ExplainTagFilterAsync(connection, workspaceId, "documentatie", "scale-test"));
report.AppendLine();
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the selective tag-filter candidate scan:");
var selectiveTagPlan = await ExplainTagFilterAsync(connection, workspaceId, "documentatie", "selective-scale-test");
report.AppendLine(selectiveTagPlan);
Assert.Contains("ix_pages_tags", selectiveTagPlan, StringComparison.Ordinal);
report.AppendLine();
report.AppendLine("Index sizes:");
report.AppendLine(await IndexSizesAsync(connection, "pages"));
}
var reportPath = System.IO.Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", "artifacts", "m8-scale-report.txt");
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(reportPath)!);
await System.IO.File.WriteAllTextAsync(reportPath, report.ToString());
foreach (var measurement in measurements)
{
// Generous release-gate bound — proving the indexed query stays
// sub-second at true documented scale, not a tight micro-benchmark.
Assert.True(
measurement.P95 < 1000,
$"{measurement.Label}: p95 was {measurement.P95:F1}ms, expected well under 1000ms at {PageCount:N0}-page scale.");
}
}
private static async Task<Guid> SeedPagesAsync(NpgsqlConnection connection, Guid workspaceId)
{
var needleId = Guid.NewGuid();
const string sql = """
INSERT INTO pages (
id, workspace_id, parent_page_id, sort_key, title, slug, markdown,
document_type, document_status, current_revision_number, content_hash,
created_by, updated_by, created_at, updated_at, tags)
SELECT
CASE WHEN n = 1 THEN @needleId::uuid ELSE gen_random_uuid() END,
@workspaceId,
NULL,
lpad(n::text, 12, '0'),
CASE WHEN n = 1 THEN 'Uniek vindbaar trefwoord xyzzyplugh'
ELSE 'Testpagina ' || n || ' ' ||
(CASE WHEN n % 2 = 0 THEN 'documentatie en projecten' ELSE 'technical procedures and code' END)
END,
'schaal-pagina-' || n,
CASE WHEN n = 1
THEN 'Deze pagina bevat het unieke trefwoord xyzzyplugh nergens anders vermeld.'
ELSE repeat(
'Gewone opvultekst met Nederlandse en Engelse woorden door elkaar over documentatie, projecten, code en procedures. ',
5)
END,
0,
1,
1,
md5(n::text),
NULL,
NULL,
now() - (n || ' seconds')::interval,
now(),
CASE WHEN n % 1000 = 0
THEN ARRAY['scale-test', 'selective-scale-test']::text[]
ELSE ARRAY['scale-test']::text[]
END
FROM generate_series(1, @pageCount) AS n
""";
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
command.Parameters.AddWithValue("needleId", needleId);
command.Parameters.AddWithValue("workspaceId", workspaceId);
command.Parameters.AddWithValue("pageCount", PageCount);
await command.ExecuteNonQueryAsync();
return needleId;
}
private static async Task SeedRevisionsAsync(NpgsqlConnection connection, Guid workspaceId)
{
const string sql = """
INSERT INTO page_revisions (
id, page_id, revision_number, title, slug, markdown, parent_page_id,
sort_key, document_type, document_status, content_hash, authored_by,
change_summary, created_at)
SELECT
gen_random_uuid(), p.id, r.rn, p.title, p.slug, p.markdown, p.parent_page_id,
p.sort_key, p.document_type, p.document_status, p.content_hash, NULL, NULL,
p.created_at + (r.rn || ' minutes')::interval
FROM pages p
CROSS JOIN generate_series(1, @revisionsPerPage) AS r(rn)
WHERE p.workspace_id = @workspaceId
""";
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
command.Parameters.AddWithValue("workspaceId", workspaceId);
command.Parameters.AddWithValue("revisionsPerPage", RevisionsPerPage);
await command.ExecuteNonQueryAsync();
}
private static async Task SeedOtherWorkspaceLeakCheckAsync(NpgsqlConnection connection)
{
const string sql = """
INSERT INTO workspaces (id, name, slug, settings_version, created_at)
VALUES (@workspaceId, 'Andere werkruimte (lekcontrole)', 'andere-werkruimte-lekcontrole', 1, now());
INSERT INTO pages (
id, workspace_id, parent_page_id, sort_key, title, slug, markdown,
document_type, document_status, current_revision_number, content_hash,
created_by, updated_by, created_at, updated_at, tags)
VALUES (
gen_random_uuid(), @workspaceId, NULL, '000000000001',
'Ontoegankelijke pagina quibblefrastication',
'ontoegankelijke-pagina', 'Bevat de zeldzame term quibblefrastication.',
0, 1, 1, md5('other-workspace'), NULL, NULL, now(), now(), ARRAY[]::text[])
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("workspaceId", Guid.NewGuid());
await command.ExecuteNonQueryAsync();
}
private static async Task PrepareReadMeasurementAsync(NpgsqlConnection connection)
{
// Run each VACUUM outside an explicit/multi-statement transaction. Both tables
// are bulk-loaded by this fixture; leaving page_revisions for autovacuum can
// create the same disk contention even though the search query reads pages.
foreach (var statement in new[]
{
"VACUUM (ANALYZE) pages;",
"VACUUM (ANALYZE) page_revisions;",
"CHECKPOINT;"
})
{
await using var command = new NpgsqlCommand(statement, connection)
{
CommandTimeout = 300
};
await command.ExecuteNonQueryAsync();
}
}
private static async Task<LatencyMeasurement> MeasureAsync(HttpClient client, StringBuilder report, string label, string path)
{
var samples = new List<double>(SampleCount);
for (var i = 0; i < SampleCount; i++)
{
var stopwatch = Stopwatch.StartNew();
using var response = await client.GetAsync(path);
stopwatch.Stop();
response.EnsureSuccessStatusCode();
samples.Add(stopwatch.Elapsed.TotalMilliseconds);
}
samples.Sort();
var p50 = Percentile(samples, 0.50);
var p95 = Percentile(samples, 0.95);
var p99 = Percentile(samples, 0.99);
report.AppendLine(CultureInfo.InvariantCulture, $" {label}: p50={p50:F1}ms p95={p95:F1}ms p99={p99:F1}ms (min={samples[0]:F1}ms max={samples[^1]:F1}ms)");
return new LatencyMeasurement(label, p95);
}
private static double Percentile(IReadOnlyList<double> sortedSamples, double percentile)
{
var index = (int)Math.Ceiling(percentile * sortedSamples.Count) - 1;
return sortedSamples[Math.Clamp(index, 0, sortedSamples.Count - 1)];
}
private static async Task<string> ExplainAsync(NpgsqlConnection connection, Guid workspaceId, string term)
{
const string sql = """
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
SELECT id, title
FROM pages
WHERE workspace_id = @workspaceId
AND trashed_at IS NULL
AND search_vector @@ plainto_tsquery('simple', @term)
ORDER BY ts_rank_cd(search_vector, plainto_tsquery('simple', @term)) DESC
LIMIT 20
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("workspaceId", workspaceId);
command.Parameters.AddWithValue("term", term);
await using var reader = await command.ExecuteReaderAsync();
var lines = new StringBuilder();
while (await reader.ReadAsync())
{
lines.AppendLine(" " + reader.GetString(0));
}
return lines.ToString();
}
private static async Task<string> ExplainFuzzyAsync(NpgsqlConnection connection, Guid workspaceId, string term)
{
await using var transaction = await connection.BeginTransactionAsync();
await using (var threshold = new NpgsqlCommand("SET LOCAL pg_trgm.similarity_threshold = '0.2';", connection, transaction))
{
await threshold.ExecuteNonQueryAsync();
}
const string sql = """
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
SELECT id, title
FROM pages
WHERE workspace_id = @workspaceId
AND trashed_at IS NULL
AND title % @term
ORDER BY similarity(title, @term) DESC, updated_at DESC, id DESC
LIMIT 20
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("workspaceId", workspaceId);
command.Parameters.AddWithValue("term", term);
var lines = new StringBuilder();
{
await using var reader = await command.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
lines.AppendLine(" " + reader.GetString(0));
}
}
await transaction.CommitAsync();
return lines.ToString();
}
private static async Task<string> ExplainTagFilterAsync(
NpgsqlConnection connection,
Guid workspaceId,
string term,
string tag)
{
const string sql = """
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
SELECT p.id, p.updated_at,
ts_rank_cd(p.search_vector, plainto_tsquery('simple', @term))::double precision AS rank
FROM pages AS p
WHERE p.workspace_id = @workspaceId
AND p.trashed_at IS NULL
AND p.search_vector @@ plainto_tsquery('simple', @term)
AND p.tags @> ARRAY[@tag]::text[]
ORDER BY rank DESC, p.updated_at DESC, p.id DESC
LIMIT 20
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("workspaceId", workspaceId);
command.Parameters.AddWithValue("term", term);
command.Parameters.AddWithValue("tag", tag);
await using var reader = await command.ExecuteReaderAsync();
var lines = new StringBuilder();
while (await reader.ReadAsync())
{
lines.AppendLine(" " + reader.GetString(0));
}
return lines.ToString();
}
private static async Task<string> IndexSizesAsync(NpgsqlConnection connection, string tableName)
{
const string sql = """
SELECT indexname, pg_size_pretty(pg_relation_size(indexname::regclass))
FROM pg_indexes
WHERE tablename = @tableName
ORDER BY indexname
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("tableName", tableName);
await using var reader = await command.ExecuteReaderAsync();
var lines = new StringBuilder();
while (await reader.ReadAsync())
{
lines.AppendLine(CultureInfo.InvariantCulture, $" {reader.GetString(0)}: {reader.GetString(1)}");
}
return lines.ToString();
}
private sealed record LatencyMeasurement(string Label, double P95);
}