259 lines
12 KiB
C#
259 lines
12 KiB
C#
using System.Diagnostics;
|
|
using System.Globalization;
|
|
using System.Net.Http.Json;
|
|
using System.Text;
|
|
using ItWorx.KnowledgeHub.Contracts.Infrastructure;
|
|
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
|
using ItWorx.KnowledgeHub.IntegrationTests.Infrastructure;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Npgsql;
|
|
|
|
namespace ItWorx.KnowledgeHub.IntegrationTests.InfrastructureKnowledge;
|
|
|
|
/// <summary>
|
|
/// The full-scale proof for infrastructure tables that
|
|
/// <c>docs/v2/V2_ACCEPTANCE_MATRIX.md</c>'s "Search/query-plan proof at scale extended
|
|
/// to infrastructure tables" row (previously `Planned`) asked for — the infrastructure
|
|
/// equivalent of <c>SearchFullScaleTests</c> (V1 M8), closing V2-R4's open release
|
|
/// risk 4 with real evidence rather than only the lighter-weight, routine-suite
|
|
/// <c>InfrastructureCatalogScaleTests</c> (V2-R2, 3,000 services). Tagged
|
|
/// "ReleaseScale" and excluded from the routine test run (matching
|
|
/// <c>SearchFullScaleTests</c>'s own precedent) — seeding and measuring at this scale
|
|
/// is a deliberate, occasional release-gate exercise. Run explicitly via:
|
|
/// <c>dotnet test --filter "Category=ReleaseScale"</c>.
|
|
/// </summary>
|
|
[Collection(PostgresTestGroup.Name)]
|
|
[Trait("Category", "ReleaseScale")]
|
|
public sealed class InfrastructureCatalogReleaseScaleTests(PostgresContainerFixture postgres) : OwnerSetupTestBase(postgres)
|
|
{
|
|
private const int ServiceCount = 50_000;
|
|
private const int SampleCount = 30;
|
|
|
|
[Fact]
|
|
public async Task Infrastructure_catalog_reads_stay_correct_and_bounded_at_50000_services()
|
|
{
|
|
using var client = Factory.CreateClient();
|
|
await SetupHelper.LoginAsync(client, OwnerEmail, OwnerPassword);
|
|
|
|
var report = new StringBuilder();
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"V2-R4 infrastructure-catalog release-scale report — {DateTimeOffset.UtcNow:O}");
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"Target: {ServiceCount:N0} services (docs/v2/V2_ACCEPTANCE_MATRIX.md, previously 'Planned').");
|
|
|
|
Guid workspaceId;
|
|
var seedStopwatch = Stopwatch.StartNew();
|
|
using (var scope = Factory.Services.CreateScope())
|
|
{
|
|
var db = scope.ServiceProvider.GetRequiredService<KnowledgeHubDbContext>();
|
|
var workspace = await db.Workspaces.AsNoTracking().FirstAsync();
|
|
workspaceId = workspace.Id;
|
|
|
|
var connectionOptions = scope.ServiceProvider.GetRequiredService<DatabaseConnectionOptions>();
|
|
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
|
await connection.OpenAsync();
|
|
|
|
await SeedServicesAsync(connection, workspaceId);
|
|
await SeedFactsAsync(connection, workspaceId);
|
|
await PrepareReadMeasurementAsync(connection);
|
|
}
|
|
|
|
seedStopwatch.Stop();
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"Seed time: {seedStopwatch.Elapsed}.");
|
|
|
|
using (var scope = Factory.Services.CreateScope())
|
|
{
|
|
var connectionOptions = scope.ServiceProvider.GetRequiredService<DatabaseConnectionOptions>();
|
|
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
|
await connection.OpenAsync();
|
|
await using var countCommand = new NpgsqlCommand(
|
|
"SELECT (SELECT count(*) FROM infrastructure_services), (SELECT count(*) FROM service_components)",
|
|
connection)
|
|
{
|
|
CommandTimeout = 300
|
|
};
|
|
await using var reader = await countCommand.ExecuteReaderAsync();
|
|
Assert.True(await reader.ReadAsync(), "Release-scale row-count query returned no row.");
|
|
var actualServices = reader.GetInt64(0);
|
|
var actualComponents = reader.GetInt64(1);
|
|
report.AppendLine(CultureInfo.InvariantCulture, $"Actual seeded rows: infrastructure_services={actualServices:N0}, service_components={actualComponents:N0}.");
|
|
Assert.True(actualServices >= ServiceCount, $"Expected at least {ServiceCount:N0} services, found {actualServices:N0}.");
|
|
}
|
|
|
|
// --- correctness: bounded, correctly-cursored pagination still holds at this scale ---
|
|
var firstPage = await client.GetFromJsonAsync<InfrastructureServiceListResponse>("/api/v1/infrastructure/services?limit=50");
|
|
Assert.NotNull(firstPage);
|
|
Assert.Equal(50, firstPage.Items.Count);
|
|
Assert.False(string.IsNullOrEmpty(firstPage.NextCursor));
|
|
var secondPage = await client.GetFromJsonAsync<InfrastructureServiceListResponse>(
|
|
$"/api/v1/infrastructure/services?limit=50&cursor={Uri.EscapeDataString(firstPage.NextCursor!)}");
|
|
Assert.NotNull(secondPage);
|
|
Assert.Equal(50, secondPage.Items.Count);
|
|
Assert.Empty(firstPage.Items.Select(item => item.Id).Intersect(secondPage.Items.Select(item => item.Id)));
|
|
report.AppendLine("Correctness: bounded first/second page, disjoint, correct size. PASS.");
|
|
|
|
// --- latency: p50/p95/p99 for the catalog's first page and a deep-cursor page ---
|
|
report.AppendLine();
|
|
report.AppendLine("Latency (ms), 30 samples per query shape:");
|
|
var measurements = new[]
|
|
{
|
|
await MeasureAsync(client, report, "first page (limit=50)", "/api/v1/infrastructure/services?limit=50"),
|
|
await MeasureAsync(client, report, "deep cursor page (after page 1)", $"/api/v1/infrastructure/services?limit=50&cursor={Uri.EscapeDataString(firstPage.NextCursor!)}")
|
|
};
|
|
|
|
// --- query plan + index sizes ---
|
|
using (var scope = Factory.Services.CreateScope())
|
|
{
|
|
var connectionOptions = scope.ServiceProvider.GetRequiredService<DatabaseConnectionOptions>();
|
|
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
|
await connection.OpenAsync();
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Query plan (EXPLAIN ANALYZE) for the catalog's first-page query:");
|
|
report.AppendLine(await ExplainCatalogPageAsync(connection, workspaceId));
|
|
|
|
report.AppendLine();
|
|
report.AppendLine("Index sizes:");
|
|
report.AppendLine(await IndexSizesAsync(connection, "infrastructure_services"));
|
|
}
|
|
|
|
var reportPath = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", "artifacts", "v2-r4-infrastructure-scale-report.txt");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(reportPath)!);
|
|
await File.WriteAllTextAsync(reportPath, report.ToString());
|
|
|
|
foreach (var measurement in measurements)
|
|
{
|
|
// Generous release-gate bound, matching SearchFullScaleTests' own — proving
|
|
// the paginated, counts-only query stays sub-second at true release scale,
|
|
// not a tight micro-benchmark.
|
|
Assert.True(
|
|
measurement.P95 < 1000,
|
|
$"{measurement.Label}: p95 was {measurement.P95:F1}ms, expected well under 1000ms at {ServiceCount:N0}-service scale.");
|
|
}
|
|
}
|
|
|
|
private static async Task SeedServicesAsync(NpgsqlConnection connection, Guid workspaceId)
|
|
{
|
|
const string sql = """
|
|
INSERT INTO infrastructure_services (
|
|
id, workspace_id, host_id, source_key, detected_name, runtime_kind, runtime_status,
|
|
health, image_reference, image_digest, runtime_user, read_only_root_filesystem,
|
|
no_new_privileges, privileged, presence, confidence, last_snapshot_id, first_seen_at, last_seen_at)
|
|
SELECT
|
|
gen_random_uuid(), @workspaceId, NULL,
|
|
'docker:release-scale-service-' || n, 'release-scale-service-' || lpad(n::text, 6, '0'),
|
|
'container', 'running', 1, 'itworx/release-scale:' || n, '', '', false, false, false,
|
|
1, 2, gen_random_uuid(), now(), now()
|
|
FROM generate_series(1, @serviceCount) AS n
|
|
""";
|
|
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
command.Parameters.AddWithValue("serviceCount", ServiceCount);
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
private static async Task SeedFactsAsync(NpgsqlConnection connection, Guid workspaceId)
|
|
{
|
|
const string sql = """
|
|
INSERT INTO service_components (
|
|
id, workspace_id, service_id, source_key, confidence, last_snapshot_id, observed_at,
|
|
name, kind, runtime_id, status, image_reference)
|
|
SELECT
|
|
gen_random_uuid(), @workspaceId, s.id, 'container:' || c.n, 2, gen_random_uuid(), now(),
|
|
'component-' || c.n, 'container', 'rt-' || c.n, 'running', ''
|
|
FROM infrastructure_services s
|
|
CROSS JOIN generate_series(1, 2) AS c(n)
|
|
WHERE s.workspace_id = @workspaceId
|
|
""";
|
|
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
private static async Task PrepareReadMeasurementAsync(NpgsqlConnection connection)
|
|
{
|
|
foreach (var statement in new[]
|
|
{
|
|
"VACUUM (ANALYZE) infrastructure_services;",
|
|
"VACUUM (ANALYZE) service_components;",
|
|
"CHECKPOINT;"
|
|
})
|
|
{
|
|
await using var command = new NpgsqlCommand(statement, connection) { CommandTimeout = 300 };
|
|
await command.ExecuteNonQueryAsync();
|
|
}
|
|
}
|
|
|
|
private static async Task<LatencyMeasurement> MeasureAsync(HttpClient client, StringBuilder report, string label, string path)
|
|
{
|
|
var samples = new List<double>(SampleCount);
|
|
for (var i = 0; i < SampleCount; i++)
|
|
{
|
|
var stopwatch = Stopwatch.StartNew();
|
|
using var response = await client.GetAsync(path);
|
|
stopwatch.Stop();
|
|
response.EnsureSuccessStatusCode();
|
|
samples.Add(stopwatch.Elapsed.TotalMilliseconds);
|
|
}
|
|
|
|
samples.Sort();
|
|
var p50 = Percentile(samples, 0.50);
|
|
var p95 = Percentile(samples, 0.95);
|
|
var p99 = Percentile(samples, 0.99);
|
|
report.AppendLine(CultureInfo.InvariantCulture, $" {label}: p50={p50:F1}ms p95={p95:F1}ms p99={p99:F1}ms (min={samples[0]:F1}ms max={samples[^1]:F1}ms)");
|
|
|
|
return new LatencyMeasurement(label, p95);
|
|
}
|
|
|
|
private static double Percentile(IReadOnlyList<double> sortedSamples, double percentile)
|
|
{
|
|
var index = (int)Math.Ceiling(percentile * sortedSamples.Count) - 1;
|
|
return sortedSamples[Math.Clamp(index, 0, sortedSamples.Count - 1)];
|
|
}
|
|
|
|
private static async Task<string> ExplainCatalogPageAsync(NpgsqlConnection connection, Guid workspaceId)
|
|
{
|
|
const string sql = """
|
|
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
|
|
SELECT id, detected_name
|
|
FROM infrastructure_services
|
|
WHERE workspace_id = @workspaceId
|
|
ORDER BY detected_name, id
|
|
LIMIT 51
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection);
|
|
command.Parameters.AddWithValue("workspaceId", workspaceId);
|
|
await using var reader = await command.ExecuteReaderAsync();
|
|
var lines = new StringBuilder();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
lines.AppendLine(" " + reader.GetString(0));
|
|
}
|
|
|
|
return lines.ToString();
|
|
}
|
|
|
|
private static async Task<string> IndexSizesAsync(NpgsqlConnection connection, string tableName)
|
|
{
|
|
const string sql = """
|
|
SELECT indexname, pg_size_pretty(pg_relation_size(indexname::regclass))
|
|
FROM pg_indexes
|
|
WHERE tablename = @tableName
|
|
ORDER BY indexname
|
|
""";
|
|
|
|
await using var command = new NpgsqlCommand(sql, connection);
|
|
command.Parameters.AddWithValue("tableName", tableName);
|
|
await using var reader = await command.ExecuteReaderAsync();
|
|
var lines = new StringBuilder();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
lines.AppendLine(CultureInfo.InvariantCulture, $" {reader.GetString(0)}: {reader.GetString(1)}");
|
|
}
|
|
|
|
return lines.ToString();
|
|
}
|
|
|
|
private sealed record LatencyMeasurement(string Label, double P95);
|
|
}
|