from __future__ import annotations import io import json import tempfile import unittest from pathlib import Path from packages.hub_observability import ( MetricsRegistry, StructuredLogger, TraceContext, new_trace_id, sanitize_fields, trace_from_headers, ) from packages.hub_operations.backup import BackupError, BackupManager, build_pg_dump_command from packages.hub_operations.recovery import RecoveryProbe, connector_isolation_report from scripts.compose_release_check import check_compose from scripts.discover_unraid import discover from services.observability.health import DependencyHealth, aggregate_health ROOT = Path(__file__).resolve().parents[1] class ObservabilityTests(unittest.TestCase): def test_trace_propagation_and_secret_safe_json_log(self) -> None: trace = new_trace_id() context = TraceContext(trace) self.assertEqual(context.response_headers()["X-Trace-Id"], trace) self.assertEqual( trace_from_headers({"traceparent": "00-" + trace + "-0123456789abcdef-01"}).trace_id, trace, ) stream = io.StringIO() record = StructuredLogger("test", stream=stream).emit( "TEST_EVENT", trace, fields={"authorization": "never-log-me", "project": "Ludarium"} ) self.assertEqual(record["fields"]["authorization"], "[REDACTED]") self.assertNotIn("never-log-me", stream.getvalue()) self.assertEqual( sanitize_fields({"nested": {"password": "secret"}})["nested"]["password"], "[REDACTED]" ) def test_metrics_are_bounded_and_sensitive_labels_are_omitted(self) -> None: metrics = MetricsRegistry() metrics.inc("itworx.requests", labels={"route": "/health", "authorization": "secret"}) metrics.observe("duration", 0.25, labels={"route": "/health"}) rendered = metrics.render_prometheus() self.assertIn('itworx_requests{route="/health"} 1', rendered) self.assertIn('duration_count{route="/health"} 1', rendered) self.assertNotIn("authorization", rendered) def test_aggregate_health_is_deny_safe_and_isolates_optional_failure(self) -> None: healthy = aggregate_health( [ DependencyHealth("postgres", "healthy"), DependencyHealth("redis", "unhealthy", required=False, authoritative=False), DependencyHealth("gitea", "unhealthy", required=False), ] ) self.assertEqual(healthy.status, "degraded") recovered = aggregate_health( [ DependencyHealth("postgres", "healthy"), DependencyHealth("redis", "healthy", required=False, authoritative=False), DependencyHealth("gitea", "healthy", required=False), ] ) self.assertEqual(recovered.status, "healthy") unavailable = aggregate_health([DependencyHealth("postgres", "unhealthy")]) self.assertEqual(unavailable.status, "unhealthy") isolation = connector_isolation_report({"knowledge": "healthy", "gitea": "unhealthy"}) self.assertEqual(isolation["failedConnectors"], ["gitea"]) self.assertTrue(isolation["isolated"]) class BackupAndRecoveryTests(unittest.TestCase): def test_hash_verified_clean_restore_and_secret_rejection(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) / "source" backup_root = Path(directory) / "backups" restore_root = Path(directory) / "restore" root.mkdir() (root / "catalog.json").write_text('{"revision":"abc"}\n', encoding="utf-8") manager = BackupManager(backup_root) artifact = manager.create_archive( root, name="release-001", include=[Path("catalog.json")] ) self.assertEqual(manager.verify(artifact).sha256, artifact.sha256) manager.restore_clean(artifact, restore_root) self.assertEqual( (restore_root / "catalog.json").read_text(encoding="utf-8"), '{"revision":"abc"}\n' ) (root / "secrets").mkdir() (root / "secrets" / "token").write_text("secret", encoding="utf-8") with self.assertRaises(BackupError): manager.create_archive(root, name="unsafe", include=[Path("secrets")]) def test_database_command_has_no_dsn_or_secret_argument(self) -> None: command = build_pg_dump_command("itworx_mcp_hub", Path("hub.dump")) self.assertEqual(command[0], "pg_dump") self.assertNotIn("password", " ".join(command).lower()) self.assertNotIn("@", " ".join(command)) def test_recovery_probe_proves_required_restore_contract(self) -> None: probe = RecoveryProbe( login=lambda: True, discovery=lambda: True, tool_call=lambda: True, negative_auth=lambda: True, audit_continuity=lambda: True, ) report = probe.run() self.assertTrue(report.passed) self.assertEqual( set(report.checks), {"login", "discovery", "toolCall", "negativeAuth", "auditContinuity"}, ) class DeploymentBoundaryTests(unittest.TestCase): def test_discovery_is_sanitized_and_marks_missing_live_inputs(self) -> None: inventory = discover() self.assertTrue(inventory["safety"]["readOnlyDiscovery"]) self.assertFalse(inventory["safety"]["credentialsRead"]) self.assertEqual(inventory["safety"]["mutatedSystems"], []) self.assertIn( inventory["discoveryStatus"], {"external-input-required", "verified-local-only"} ) self.assertEqual(inventory["endpoints"]["inspector"], "LAN-only; no public DNS") def test_compose_observer_profile_has_no_new_public_surface(self) -> None: path = ROOT / "infrastructure" / "compose" / "docker-compose.yml" result = check_compose(path) self.assertTrue(result["pass"]) self.assertNotIn("observability-collector", result["publishedServices"]) document = json.loads( json.dumps(__import__("yaml").safe_load(path.read_text(encoding="utf-8"))) ) self.assertEqual( document["services"]["observability-collector"]["profiles"], ["observability"] ) self.assertTrue(document["networks"]["hub-observe"]["internal"]) self.assertEqual( path.read_text(encoding="utf-8").count("docker_broker_service_token: {file:"), 1, ) if __name__ == "__main__": unittest.main()