153 lines
6.5 KiB
Python
153 lines
6.5 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from packages.hub_observability import (
|
|
MetricsRegistry,
|
|
StructuredLogger,
|
|
TraceContext,
|
|
new_trace_id,
|
|
sanitize_fields,
|
|
trace_from_headers,
|
|
)
|
|
from packages.hub_operations.backup import BackupError, BackupManager, build_pg_dump_command
|
|
from packages.hub_operations.recovery import RecoveryProbe, connector_isolation_report
|
|
from scripts.compose_release_check import check_compose
|
|
from scripts.discover_unraid import discover
|
|
from services.observability.health import DependencyHealth, aggregate_health
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class ObservabilityTests(unittest.TestCase):
|
|
def test_trace_propagation_and_secret_safe_json_log(self) -> None:
|
|
trace = new_trace_id()
|
|
context = TraceContext(trace)
|
|
self.assertEqual(context.response_headers()["X-Trace-Id"], trace)
|
|
self.assertEqual(
|
|
trace_from_headers({"traceparent": "00-" + trace + "-0123456789abcdef-01"}).trace_id,
|
|
trace,
|
|
)
|
|
stream = io.StringIO()
|
|
record = StructuredLogger("test", stream=stream).emit(
|
|
"TEST_EVENT", trace, fields={"authorization": "never-log-me", "project": "Ludarium"}
|
|
)
|
|
self.assertEqual(record["fields"]["authorization"], "[REDACTED]")
|
|
self.assertNotIn("never-log-me", stream.getvalue())
|
|
self.assertEqual(
|
|
sanitize_fields({"nested": {"password": "secret"}})["nested"]["password"], "[REDACTED]"
|
|
)
|
|
|
|
def test_metrics_are_bounded_and_sensitive_labels_are_omitted(self) -> None:
|
|
metrics = MetricsRegistry()
|
|
metrics.inc("itworx.requests", labels={"route": "/health", "authorization": "secret"})
|
|
metrics.observe("duration", 0.25, labels={"route": "/health"})
|
|
rendered = metrics.render_prometheus()
|
|
self.assertIn('itworx_requests{route="/health"} 1', rendered)
|
|
self.assertIn('duration_count{route="/health"} 1', rendered)
|
|
self.assertNotIn("authorization", rendered)
|
|
|
|
def test_aggregate_health_is_deny_safe_and_isolates_optional_failure(self) -> None:
|
|
healthy = aggregate_health(
|
|
[
|
|
DependencyHealth("postgres", "healthy"),
|
|
DependencyHealth("redis", "unhealthy", required=False, authoritative=False),
|
|
DependencyHealth("gitea", "unhealthy", required=False),
|
|
]
|
|
)
|
|
self.assertEqual(healthy.status, "degraded")
|
|
recovered = aggregate_health(
|
|
[
|
|
DependencyHealth("postgres", "healthy"),
|
|
DependencyHealth("redis", "healthy", required=False, authoritative=False),
|
|
DependencyHealth("gitea", "healthy", required=False),
|
|
]
|
|
)
|
|
self.assertEqual(recovered.status, "healthy")
|
|
unavailable = aggregate_health([DependencyHealth("postgres", "unhealthy")])
|
|
self.assertEqual(unavailable.status, "unhealthy")
|
|
isolation = connector_isolation_report({"knowledge": "healthy", "gitea": "unhealthy"})
|
|
self.assertEqual(isolation["failedConnectors"], ["gitea"])
|
|
self.assertTrue(isolation["isolated"])
|
|
|
|
|
|
class BackupAndRecoveryTests(unittest.TestCase):
|
|
def test_hash_verified_clean_restore_and_secret_rejection(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory) / "source"
|
|
backup_root = Path(directory) / "backups"
|
|
restore_root = Path(directory) / "restore"
|
|
root.mkdir()
|
|
(root / "catalog.json").write_text('{"revision":"abc"}\n', encoding="utf-8")
|
|
manager = BackupManager(backup_root)
|
|
artifact = manager.create_archive(
|
|
root, name="release-001", include=[Path("catalog.json")]
|
|
)
|
|
self.assertEqual(manager.verify(artifact).sha256, artifact.sha256)
|
|
manager.restore_clean(artifact, restore_root)
|
|
self.assertEqual(
|
|
(restore_root / "catalog.json").read_text(encoding="utf-8"), '{"revision":"abc"}\n'
|
|
)
|
|
(root / "secrets").mkdir()
|
|
(root / "secrets" / "token").write_text("secret", encoding="utf-8")
|
|
with self.assertRaises(BackupError):
|
|
manager.create_archive(root, name="unsafe", include=[Path("secrets")])
|
|
|
|
def test_database_command_has_no_dsn_or_secret_argument(self) -> None:
|
|
command = build_pg_dump_command("itworx_mcp_hub", Path("hub.dump"))
|
|
self.assertEqual(command[0], "pg_dump")
|
|
self.assertNotIn("password", " ".join(command).lower())
|
|
self.assertNotIn("@", " ".join(command))
|
|
|
|
def test_recovery_probe_proves_required_restore_contract(self) -> None:
|
|
probe = RecoveryProbe(
|
|
login=lambda: True,
|
|
discovery=lambda: True,
|
|
tool_call=lambda: True,
|
|
negative_auth=lambda: True,
|
|
audit_continuity=lambda: True,
|
|
)
|
|
report = probe.run()
|
|
self.assertTrue(report.passed)
|
|
self.assertEqual(
|
|
set(report.checks),
|
|
{"login", "discovery", "toolCall", "negativeAuth", "auditContinuity"},
|
|
)
|
|
|
|
|
|
class DeploymentBoundaryTests(unittest.TestCase):
|
|
def test_discovery_is_sanitized_and_marks_missing_live_inputs(self) -> None:
|
|
inventory = discover()
|
|
self.assertTrue(inventory["safety"]["readOnlyDiscovery"])
|
|
self.assertFalse(inventory["safety"]["credentialsRead"])
|
|
self.assertEqual(inventory["safety"]["mutatedSystems"], [])
|
|
self.assertIn(
|
|
inventory["discoveryStatus"], {"external-input-required", "verified-local-only"}
|
|
)
|
|
self.assertEqual(inventory["endpoints"]["inspector"], "LAN-only; no public DNS")
|
|
|
|
def test_compose_observer_profile_has_no_new_public_surface(self) -> None:
|
|
path = ROOT / "infrastructure" / "compose" / "docker-compose.yml"
|
|
result = check_compose(path)
|
|
self.assertTrue(result["pass"])
|
|
self.assertNotIn("observability-collector", result["publishedServices"])
|
|
document = json.loads(
|
|
json.dumps(__import__("yaml").safe_load(path.read_text(encoding="utf-8")))
|
|
)
|
|
self.assertEqual(
|
|
document["services"]["observability-collector"]["profiles"], ["observability"]
|
|
)
|
|
self.assertTrue(document["networks"]["hub-observe"]["internal"])
|
|
self.assertEqual(
|
|
path.read_text(encoding="utf-8").count("docker_broker_service_token: {file:"),
|
|
1,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|