diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 1b3e423..daaac35 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `207d2a918641b199961d27de09b6c1fdf45a6e4b`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `a194da2defbd709ea58bd4c9503966dbd37b11f2`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index 6cac8ee..31d5ae4 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "207d2a918641b199961d27de09b6c1fdf45a6e4b", + "sourceRevision": "a194da2defbd709ea58bd4c9503966dbd37b11f2", "files": [ { "path": ".dockerignore", @@ -294,8 +294,8 @@ }, { "path": "backend/app/connectors/gitea_actions.py", - "sha256": "3c0be8c895079c007b9c04d2eaab152cc30d10ea8708b6d3914a86a3b1b35a65", - "bytes": 29286 + "sha256": "4d3229e03b5878a67e8b3826a62ec073a54a7279426b986f7e6dff027a9caec4", + "bytes": 29972 }, { "path": "backend/app/connectors/gpu_metrics.py", @@ -877,6 +877,11 @@ "sha256": "1da47a3f061ef0b89158c9f646bb7018ce88b81c0687d8b27abe65b74df67fea", "bytes": 4948 }, + { + "path": "backend/tests/fixtures/hub_security_base_publication_23280.json", + "sha256": "74e5cff171b4fa5aeeec2a58ce71147546c3e217a69c224f2d9227b007294cd1", + "bytes": 3090 + }, { "path": "backend/tests/test_ai_remediation_planner.py", "sha256": "784b83960cdff238e0ee25caaf83cfd1cb7c3cfc40976036b28bd256f6626e44", @@ -1024,8 +1029,8 @@ }, { "path": "backend/tests/test_deploy_broker.py", - "sha256": "5f7a8bd1724a81a5d7628f4b65e23687acb1a6af2ff5eb63e6d770d6e15914c0", - "bytes": 442256 + "sha256": "a83b6d74fa51b208e38c9e5b9fc667c3150b342fc99265e2d538e8442a13f5b1", + "bytes": 442377 }, { "path": "backend/tests/test_deployment_auto_rollback.py", @@ -1204,8 +1209,8 @@ }, { "path": "backend/tests/test_hub_candidate_import.py", - "sha256": "87762293c6088b5e3e71adfa4c109ec468d7931f3e10b8be6e04ce1d19f8d849", - "bytes": 47846 + "sha256": "9ae57b5a33652ca94bfdb4c69e65ec7dfe47650de4d6a39caa3026bfe8f3d354", + "bytes": 52139 }, { "path": "backend/tests/test_hub_candidate_transport.py", @@ -1227,6 +1232,11 @@ "sha256": "3ba8efe338a21b100e4bc20e4fbffe043eb57c411a4143c00332fdb368db5011", "bytes": 28482 }, + { + "path": "backend/tests/test_hub_security_publication_bindings.py", + "sha256": "7b086e64793faef0d98a2f62f36acbbc30d1f4b3363d4c6613db81f771358687", + "bytes": 3113 + }, { "path": "backend/tests/test_immutable_service_promotion.py", "sha256": "8220a58307a0dd3a2ab8c885111b00ae57658bbc246e9e826fb46fcfddf76ef2", @@ -1264,8 +1274,8 @@ }, { "path": "backend/tests/test_managed_validation_parallel.py", - "sha256": "8b3354bd6353831a081459b7398eac2ce4f8626251ff38422f6d217caf516ec3", - "bytes": 2367 + "sha256": "dd8c2ac42ef8c129cc4ed7b24d72ef42f3e92d45fb3165822b9c8ea547c1648e", + "bytes": 5935 }, { "path": "backend/tests/test_mcp_hub_candidate_preparation.py", @@ -1279,8 +1289,8 @@ }, { "path": "backend/tests/test_mcp_hub_governed_topology.py", - "sha256": "59872024ae0a2e552aafe459d42d90098e022877648c3d77d33ce102c595d234", - "bytes": 127375 + "sha256": "029cb69fec06a58bbf9125d0640e2b39aea0ec369d32da853ebf6a3f5fe0bcc5", + "bytes": 129320 }, { "path": "backend/tests/test_mcp_hub_legacy_registry_compatibility.py", @@ -1604,8 +1614,8 @@ }, { "path": "backend/tests/test_runner_admission_controller.py", - "sha256": "a03f9a18184149e210a3062578ade025559823050a355d55a760d861f36ae76c", - "bytes": 81756 + "sha256": "b5ddebc17d647b3bb0a7032cc146fdc015da3fd8732063b8088f39f02b6569da", + "bytes": 92461 }, { "path": "backend/tests/test_runner_admission_policy.py", @@ -1682,6 +1692,11 @@ "sha256": "878fe58ff4dbb696ce6234b403f08f003a22881dba394cc2de1fbddc307c36ef", "bytes": 3018 }, + { + "path": "backend/tests/test_runner_inventory_visibility.py", + "sha256": "06ff683bb9cd5b25053bce198e05a298b56e81352ce7230e13a8819a96f208f9", + "bytes": 3491 + }, { "path": "backend/tests/test_runner_profiles.py", "sha256": "45a8dba20d1e8d778dc90c20a997630037e2eaa1d462fa62f48671285aa82e0e", diff --git a/backend/app/connectors/gitea_actions.py b/backend/app/connectors/gitea_actions.py index 509806d..7dbe795 100644 --- a/backend/app/connectors/gitea_actions.py +++ b/backend/app/connectors/gitea_actions.py @@ -344,6 +344,7 @@ class GiteaActionsConnector(BaseConnector): "severity": severity, "generatedAt": datetime.now(UTC).isoformat(), "runners": [], + "inventory": {"scope": "all_registrations", "complete": False}, "queue": { "queued": 0, "inProgress": 0, @@ -404,7 +405,7 @@ class GiteaActionsConnector(BaseConnector): try: runner_payload = self._request( - "/api/v1/admin/actions/runners", {"disabled": "false"} + "/api/v1/admin/actions/runners", {"page": 1, "limit": 100} ) queued_payload = self._request( "/api/v1/admin/actions/jobs", @@ -654,7 +655,7 @@ class GiteaActionsConnector(BaseConnector): elif ( (oldest_age or 0) >= _WARNING_AGE_SECONDS or (oldest_active_age or 0) >= _ACTIVE_WARNING_AGE_SECONDS - or len(online_runners) < len(runners) + or len(online_runners) < sum(not item["disabled"] for item in runners) ): severity = "warning" @@ -702,6 +703,10 @@ class GiteaActionsConnector(BaseConnector): restart_reason = "" for label in [*blocked_labels, *stale_active_labels]: target = self._target_for_label(label, runners, targets) + # A disabled registration can be an operator pause or a fenced + # active lease. Neither is permission to restart its container. + if any(item["name"] == target and item["disabled"] for item in runners): + continue if target: restart_target = target restart_reason = ( @@ -727,11 +732,17 @@ class GiteaActionsConnector(BaseConnector): "severity": severity, "generatedAt": now.isoformat(), "runners": runners, + "inventory": { + "scope": "all_registrations", + "complete": recovery_runners is not None, + }, "summary": { "total": len(runners), "online": len(online_runners), "busy": sum(bool(item["busy"]) for item in runners), "offline": sum(item["status"] != "online" for item in runners), + "disabled": sum(item["disabled"] for item in runners), + "available": sum(not item["busy"] for item in online_runners), }, "queue": { "queued": display_count(queued_payload.get("total_count", 0)), diff --git a/backend/tests/fixtures/hub_security_base_publication_23280.json b/backend/tests/fixtures/hub_security_base_publication_23280.json new file mode 100644 index 0000000..a620a28 --- /dev/null +++ b/backend/tests/fixtures/hub_security_base_publication_23280.json @@ -0,0 +1,52 @@ +{ + "artifactName": "hub-release-security-bases-3237e4d8aec312d63d13b4e80664422a7520b5f9", + "candidateEmitted": false, + "preparedImages": [ + { + "archiveRef": "gitea.example.com/jens/itworx-mcp-hub/postgres:3237e4d8aec312d63d13b4e80664422a7520b5f9-security-run-23280-attempt-1", + "imageId": "sha256:227c8e37851f267cca5853042f86726995acadfbc84fd2d726879b6cce4d98a7", + "imageRef": "gitea.example.com/jens/itworx-mcp-hub/postgres:3237e4d8aec312d63d13b4e80664422a7520b5f9-security-run-23280-attempt-1@sha256:4e471418b9cf0805ed3c5cce716fb13ac96aa1fcabb6751fd4386b1db679e176", + "parentRef": "gitea.example.com/jens/itworx-mcp-hub/postgres:wp400-gosu-v2-20261002-a94e0b4d2029@sha256:a60d81e06987671ae903757f1b71cdfbbff4d481b42610e4de2b7dcfa32aad73", + "runtimeVerified": true, + "service": "postgres", + "sourceInputs": { + "evidence/WP-400/database-image-candidate-20261003/recipes/postgres/verify-gosu-metadata.sh": { + "materializedSha256": "0c33ed42751a48bb76908285519cef7a41b7e383b07418f641e799e183245a4b", + "sourceSha256": "0c33ed42751a48bb76908285519cef7a41b7e383b07418f641e799e183245a4b" + }, + "infrastructure/images/security-bases/postgres/Dockerfile": { + "materializedSha256": "fcd828b854c6aa2c83116a813ef83e7fa5431f9c2035c98871d90720f166adfb", + "sourceSha256": "fcd828b854c6aa2c83116a813ef83e7fa5431f9c2035c98871d90720f166adfb" + } + }, + "vulnerabilityAndSecretVerified": true + }, + { + "archiveRef": "gitea.example.com/jens/itworx-mcp-hub/ingestion-clamd:3237e4d8aec312d63d13b4e80664422a7520b5f9-security-run-23280-attempt-1", + "imageId": "sha256:f16f3f6d8a2a0a6c56d32d4e97e173ad8fdff08c88ac40ddc9890ab27cfdda35", + "imageRef": "gitea.example.com/jens/itworx-mcp-hub/ingestion-clamd:3237e4d8aec312d63d13b4e80664422a7520b5f9-security-run-23280-attempt-1@sha256:eeca204bb48b30e78cf103bff00a33babb45e7c289747d7469396551803a8d23", + "parentRef": "clamav/clamav:1.4.6@sha256:a5f03c12a79dbe9f6d8a527b6bb1ea053fa8dd061d3738a26897f055ee2d9303", + "runtimeVerified": true, + "service": "ingestion-clamd", + "sourceInputs": { + "infrastructure/images/security-bases/ingestion-clamd/Dockerfile": { + "materializedSha256": "0b806f1077af26b85a059eb2a40eeff27b661ef997d57fc82bb648fc311043d5", + "sourceSha256": "0b806f1077af26b85a059eb2a40eeff27b661ef997d57fc82bb648fc311043d5" + }, + "infrastructure/images/security-bases/ingestion-clamd/verify-pcre2.sh": { + "materializedSha256": "1ec848d1f52b37d3abfb56e1308383d4d2907810e65bad95368b5ecee60a5514", + "sourceSha256": "1ec848d1f52b37d3abfb56e1308383d4d2907810e65bad95368b5ecee60a5514" + } + }, + "vulnerabilityAndSecretVerified": true + } + ], + "reportCount": 5, + "repository": "jens/itworx_mcp_hub", + "revision": "3237e4d8aec312d63d13b4e80664422a7520b5f9", + "runAttempt": 1, + "runId": 23280, + "schema": "itworx.security-base-preparation/v1", + "status": "security-bases-prepared", + "tree": "8a97fdcd1fed62e48217326a29d18bc9805385ec" +} diff --git a/backend/tests/test_deploy_broker.py b/backend/tests/test_deploy_broker.py index 637d519..e9ca86d 100644 --- a/backend/tests/test_deploy_broker.py +++ b/backend/tests/test_deploy_broker.py @@ -8760,9 +8760,10 @@ def test_appops_uses_push_validation_without_a_deploy_runner_workflow(): assert "branches: [main]" in managed_validation assert "unraid-deploy" not in managed_validation - # Keep all jobs on the installed legacy fleet during specialized-runner bootstrap. - expected_runner = "linux-validation" + # The fleet is specialized: lightweight jobs use Checks, while the four + # execution lanes retain their installed legacy-compatible routing. for job in ("preflight", "backend", "frontend", "security", "polyglot", "full"): + expected_runner = "itworx-ci-checks" if job in {"preflight", "full"} else "linux-validation" match = re.search( rf"(?ms)^ {re.escape(job)}:\n(?P.*?)(?=^ [a-z][a-z0-9_-]*:\n|\Z)", managed_validation, diff --git a/backend/tests/test_hub_candidate_import.py b/backend/tests/test_hub_candidate_import.py index ccdaf90..ce86195 100644 --- a/backend/tests/test_hub_candidate_import.py +++ b/backend/tests/test_hub_candidate_import.py @@ -1184,6 +1184,81 @@ def test_custom_external_database_accepts_exact_custom_repo_digest( assert calls == [archive_ref, image_ref] +@pytest.mark.parametrize("service", ("postgres", "ingestion-clamd")) +def test_reviewed_external_variant_requires_its_exact_archive_and_digest( + modules, monkeypatch, service +): + importer, controller = modules.importer, modules.controller + image_ref = f"gitea.example.com/jens/itworx-mcp-hub/{service}:security-fixture@sha256:{'b' * 64}" + archive_ref = f"itworx-mcp-hub/{service}:security-fixture" + repo_digest = image_ref.split(":security-fixture", 1)[0] + "@sha256:" + "b" * 64 + approved = { + name: (binding,) for name, binding in importer.EXTERNAL_IMAGES.items() + } + approved[service] += ((image_ref, archive_ref, repo_digest),) + monkeypatch.setattr(controller, "MCP_HUB_APPROVED_EXTERNAL_IMAGE_BINDINGS", approved, raising=False) + image_id = "sha256:" + "c" * 64 + inspected = { + archive_ref: {"Id": image_id}, + image_ref: {"Id": image_id, "RepoDigests": [repo_digest]}, + } + monkeypatch.setattr(importer, "_inspect_image", inspected.get) + row = {"imageRef": image_ref, "archiveRef": archive_ref, "imageId": image_id} + assert importer._require_loaded_image(service, row, external=True)["Id"] == image_id + inspected[image_ref]["RepoDigests"] = [repo_digest[:-1] + "0"] + with pytest.raises(importer.CandidateRejected, match="Pinned external digest"): + importer._require_loaded_image(service, row, external=True) + + +@pytest.mark.parametrize("service", ("postgres", "ingestion-clamd")) +def test_external_variant_never_accepts_an_unreviewed_ref(modules, monkeypatch, service): + importer = modules.importer + image_id = "sha256:" + "c" * 64 + unknown_ref = f"gitea.example.com/jens/itworx-mcp-hub/{service}:unknown@sha256:{'b' * 64}" + monkeypatch.setattr(importer, "_inspect_image", lambda _ref: {"Id": image_id, "RepoDigests": []}) + with pytest.raises(importer.CandidateRejected, match="reviewed external image binding"): + importer._require_loaded_image( + service, {"imageRef": unknown_ref, "archiveRef": "unknown", "imageId": image_id}, external=True + ) + + +@pytest.mark.parametrize("service", ("postgres", "ingestion-clamd")) +def test_protected_model_external_variant_has_a_distinct_exact_archive_ref( + modules, monkeypatch, service +): + importer, controller = modules.importer, modules.controller + image_ref = f"gitea.example.com/jens/itworx-mcp-hub/{service}:security-fixture@sha256:{'b' * 64}" + archive_ref = f"itworx-mcp-hub/{service}:security-fixture" + approved = {name: (binding,) for name, binding in importer.EXTERNAL_IMAGES.items()} + approved[service] += ((image_ref, archive_ref, image_ref.split(":security-fixture", 1)[0] + "@sha256:" + "b" * 64),) + monkeypatch.setattr(controller, "MCP_HUB_APPROVED_EXTERNAL_IMAGE_BINDINGS", approved) + services = { + name: {"image": importer.EXTERNAL_IMAGES[name][0] if name in importer.EXTERNAL_IMAGES else "fixture:" + name} + for name in controller.MCP_HUB_SERVICES + } + ingestion = {"imageRef": services["connector-ingestion"]["image"], "archiveRef": "fixture:ingestion"} + services[service]["image"] = image_ref + references = importer._expected_image_refs({"services": services}, ingestion_binding=ingestion) + assert references[service] == (image_ref, archive_ref) + assert all(references[name][1] == binding[1] for name, binding in importer.EXTERNAL_IMAGES.items() if name != service) + services[service]["image"] = image_ref[:-1] + "0" + with pytest.raises(importer.CandidateRejected, match="fixed digest"): + importer._expected_image_refs({"services": services}, ingestion_binding=ingestion) + + +@pytest.mark.parametrize("service", ("postgres", "ingestion-clamd")) +def test_loaded_external_image_cannot_substitute_an_archive_alias(modules, monkeypatch, service): + importer = modules.importer + image_ref = importer.EXTERNAL_IMAGES[service][0] + monkeypatch.setattr(importer, "_inspect_image", lambda _ref: pytest.fail("must reject before inspecting a foreign alias")) + with pytest.raises(importer.CandidateRejected, match="reviewed external image binding"): + importer._require_loaded_image( + service, + {"imageRef": image_ref, "archiveRef": "foreign:alias", "imageId": "sha256:" + "c" * 64}, + external=True, + ) + + @pytest.mark.parametrize("service", ("postgres", "redis")) @pytest.mark.parametrize( "failure", ("upstream", "wrong_custom", "missing_repo_digests", "foreign_image_id") diff --git a/backend/tests/test_hub_security_publication_bindings.py b/backend/tests/test_hub_security_publication_bindings.py new file mode 100644 index 0000000..531f594 --- /dev/null +++ b/backend/tests/test_hub_security_publication_bindings.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from test_hub_candidate_import import modules # noqa: F401 - reuse importer isolation fixture + + +RECEIPT = json.loads( + (Path(__file__).parent / "fixtures/hub_security_base_publication_23280.json").read_text() +) + + +@pytest.mark.parametrize("row", RECEIPT["preparedImages"], ids=lambda row: row["service"]) +def test_actual_published_security_image_requires_exact_archive_id_and_digest(modules, monkeypatch, row): + importer, controller = modules.importer, modules.controller + service = row["service"] + repository, digest = row["imageRef"].split("@", 1) + repo_digest = repository.rsplit(":", 1)[0] + "@" + digest + assert RECEIPT["revision"] == "3237e4d8aec312d63d13b4e80664422a7520b5f9" + assert RECEIPT["runId"] == 23280 and RECEIPT["runAttempt"] == 1 + assert RECEIPT["candidateEmitted"] is False and RECEIPT["reportCount"] == 5 + assert row["runtimeVerified"] and row["vulnerabilityAndSecretVerified"] + inspected = { + row["archiveRef"]: {"Id": row["imageId"]}, + row["imageRef"]: {"Id": row["imageId"], "RepoDigests": [repo_digest]}, + } + monkeypatch.setattr(importer, "_inspect_image", inspected.get) + assert importer._require_loaded_image(service, row, external=True)["Id"] == row["imageId"] + binding = controller.mcp_hub_external_image_binding(service, row["imageRef"]) + assert binding == (row["imageRef"], row["archiveRef"], repo_digest) + old_binding = controller.mcp_hub_external_image_binding(service, row["parentRef"]) + assert old_binding is not None + assert old_binding[1] != row["archiveRef"] + with pytest.raises(importer.CandidateRejected, match="reviewed external image binding"): + importer._require_loaded_image(service, {**row, "archiveRef": old_binding[1]}, external=True) + inspected[row["archiveRef"]]["Id"] = "sha256:" + "0" * 64 + with pytest.raises(importer.CandidateRejected, match="Loaded Docker image ID differs"): + importer._require_loaded_image(service, row, external=True) + inspected[row["archiveRef"]]["Id"] = row["imageId"] + inspected[row["imageRef"]]["RepoDigests"] = [old_binding[2]] + with pytest.raises(importer.CandidateRejected, match="Pinned external digest"): + importer._require_loaded_image(service, row, external=True) + + +@pytest.mark.parametrize("row", RECEIPT["preparedImages"], ids=lambda row: row["service"]) +def test_actual_security_pin_preserves_only_reviewed_forward_predecessors(modules, row): + controller = modules.controller + allowed = controller._mcp_hub_observed_external_ref_allowed + service, new, old = row["service"], row["imageRef"], row["parentRef"] + assert allowed(service, new, old) + assert allowed(service, new, new) + assert not allowed(service, old, new) + assert not allowed(service, new, old + "-drift") + assert not allowed(service, new[:-1] + "0", old) + if service == "postgres": + assert allowed(service, new, controller.MCP_HUB_POSTGRES_LEGACY_IMAGE) + assert allowed(service, new, controller.MCP_HUB_POSTGRES_PREVIOUS_IMAGE) diff --git a/backend/tests/test_managed_validation_parallel.py b/backend/tests/test_managed_validation_parallel.py index 1f62e24..d42343c 100644 --- a/backend/tests/test_managed_validation_parallel.py +++ b/backend/tests/test_managed_validation_parallel.py @@ -1,6 +1,12 @@ import re +import importlib.util +import os +import subprocess +import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml" @@ -61,3 +67,68 @@ def test_backend_budget_is_bounded_without_weakening_validation(): assert "continue-on-error" not in backend_body assert "--ignore" not in backend_body assert "--deselect" not in backend_body + + +def _job_body(job): + text = WORKFLOW.read_text(encoding="utf-8") + match = re.search(rf"(?ms)^ {re.escape(job)}:\n(?P.*?)(?=^ [a-z][a-z0-9_-]*:\n|\Z)", text) + assert match is not None + return match["body"] + + +def test_light_checks_use_existing_single_checks_profile(): + spec = importlib.util.spec_from_file_location( + "appops_light_checks_profiles", ROOT / "ops/gitea-deploy-broker/runner_profiles.py" + ) + profiles = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = profiles + spec.loader.exec_module(profiles) + try: + for job in ("preflight", "full"): + label = re.search(r"(?m)^ runs-on: (\S+)\s*$", _job_body(job)).group(1) + assert tuple(profile.id for profile in profiles.profiles_for_labels((label,))) == ("checks",) + assert profiles.RUNNER_PROFILES["checks"].memory_mib == 1536 + for job in ("backend", "frontend", "security", "polyglot"): + assert " runs-on: linux-validation\n" in _job_body(job) + finally: + sys.modules.pop(spec.name, None) + + +def _bash_script(job): + body = _job_body(job) + script = body.split(" run: |\n", 1)[1] + return "\n".join(line[10:] if line.startswith(" ") else line for line in script.splitlines()) + + +@pytest.mark.skipif(os.name == "nt", reason="Execute actual workflow Bash on Linux") +@pytest.mark.parametrize("failed_lane", [None, "PREFLIGHT", "BACKEND", "FRONTEND", "SECURITY", "POLYGLOT"]) +def test_light_full_gate_executes_all_required_lane_results(tmp_path, failed_lane): + environment = {"PATH": os.defpath, **{name: "success" for name in ("PREFLIGHT", "BACKEND", "FRONTEND", "SECURITY", "POLYGLOT")}} + if failed_lane: + environment[failed_lane] = "failure" + result = subprocess.run(["bash", "-c", _bash_script("full")], cwd=tmp_path, env=environment, + capture_output=True, text=True, timeout=10) + assert (result.returncode == 0) is (failed_lane is None) + assert ("All managed validation lanes passed." in result.stdout) is (failed_lane is None) + + +@pytest.mark.skipif(os.name == "nt", reason="Execute actual workflow Bash/Git on Linux") +@pytest.mark.parametrize("change", ["clean", "whitespace", "conflict"]) +def test_light_preflight_executes_git_and_rejects_bad_source(tmp_path, change): + environment = {"PATH": os.defpath, "HOME": str(tmp_path), "GITHUB_OUTPUT": str(tmp_path / "output")} + for args in (["init", "--quiet"], ["config", "user.name", "CI fixture"], + ["config", "user.email", "ci@example.invalid"]): + subprocess.run(["git", *args], cwd=tmp_path, env=environment, check=True, capture_output=True) + readme = tmp_path / "README.md" + readme.write_text("Clean source\n") + subprocess.run(["git", "add", "README.md"], cwd=tmp_path, env=environment, check=True, capture_output=True) + subprocess.run(["git", "commit", "--quiet", "-m", "fixture"], cwd=tmp_path, env=environment, check=True, capture_output=True) + if change == "whitespace": + readme.write_text("Trailing whitespace \n") + elif change == "conflict": + readme.write_text("<<<<<<< fixture\n") + result = subprocess.run(["bash", "-c", _bash_script("preflight")], cwd=tmp_path, env=environment, + capture_output=True, text=True, timeout=10) + assert (result.returncode == 0) is (change == "clean") + if change == "clean": + assert "runtime_change=true" in (tmp_path / "output").read_text() diff --git a/backend/tests/test_mcp_hub_governed_topology.py b/backend/tests/test_mcp_hub_governed_topology.py index 570b3cf..fb0e1c1 100644 --- a/backend/tests/test_mcp_hub_governed_topology.py +++ b/backend/tests/test_mcp_hub_governed_topology.py @@ -2234,6 +2234,37 @@ def test_observed_baseline_is_separate_evidence_and_preserves_unknown_controller if os.name != "nt": assert stat.S_IMODE(evidence_path.stat().st_mode) == 0o600 +@pytest.mark.parametrize("service", ("postgres", "ingestion-clamd")) +def test_reviewed_security_transition_preserves_the_exact_live_baseline( + controller, monkeypatch, tmp_path, service +): + project, _candidate, model, _backup, runtime, _images, state_path, prior = ( + _observed_live_fixture(controller, monkeypatch, tmp_path) + ) + original_state = state_path.read_bytes() + name = f"itworx-mcp-hub-{service}-1" + observed_ref = runtime[name]["Config"]["Image"] + security_ref = f"gitea.example.com/jens/itworx-mcp-hub/{service}:security-fixture@sha256:{'e' * 64}" + bindings = dict(controller.MCP_HUB_APPROVED_EXTERNAL_IMAGE_BINDINGS) + bindings[service] += ((security_ref, "fixture:security", security_ref),) + monkeypatch.setattr(controller, "MCP_HUB_APPROVED_EXTERNAL_IMAGE_BINDINGS", bindings) + transitions = dict(getattr(controller, "MCP_HUB_APPROVED_OBSERVED_EXTERNAL_TRANSITIONS", {})) + transitions[service] = {security_ref: (observed_ref, security_ref)} + monkeypatch.setattr(controller, "MCP_HUB_APPROVED_OBSERVED_EXTERNAL_TRANSITIONS", transitions, raising=False) + model["services"][service]["image"] = security_ref + + verified, facts, observed_model = controller.mcp_hub_verify_observed_baseline(project, prior, model) + + assert verified[name]["Config"]["Image"] == observed_ref + assert facts[name]["image_ref"] == observed_ref + assert observed_model["services"][service]["image"] == observed_ref + assert state_path.read_bytes() == original_state + assert controller._mcp_hub_observed_external_ref_allowed(service, security_ref, observed_ref) + assert not controller._mcp_hub_observed_external_ref_allowed(service, observed_ref, security_ref) + assert not controller._mcp_hub_observed_external_ref_allowed(service, security_ref, observed_ref + "-unexpected") + assert not controller._mcp_hub_observed_external_ref_allowed(service, security_ref[:-1] + "0", observed_ref) + + def test_backup_receipt_validator_rejects_an_unpersisted_boolean_claim( controller, monkeypatch, tmp_path ): diff --git a/backend/tests/test_runner_admission_controller.py b/backend/tests/test_runner_admission_controller.py index 9d64e57..1e2b68a 100644 --- a/backend/tests/test_runner_admission_controller.py +++ b/backend/tests/test_runner_admission_controller.py @@ -316,6 +316,69 @@ def test_safe_live_metric_fluctuations_do_not_restart_stable_headroom(controller assert reason is None +@pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"]) +def test_cached_post_start_sample_preserves_stability_without_authorizing_enable( + controller_module, monkeypatch, change +): + controller, now, host, capacity = _host_gate_sample(controller_module, monkeypatch) + state = {} + controller._fresh_mutation_gate("backend", ["backend"], state) + now[0] = 160 + host["swapSampleAt"] = 160 + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is None + established = copy.deepcopy(state["hostStability"]) + now[0] = 161 + assert controller._fresh_mutation_gate( + "backend", ["backend"], state, minimum_sample_at=160 + )[0] == "runner_admission_host_telemetry_not_refreshed" + assert state["hostStability"] == established + + now[0] = 162 + host["swapSampleAt"] = 162 + assert controller._fresh_mutation_gate( + "backend", ["backend"], state, minimum_sample_at=160 + )[0] is None + if change == "load": + host["loadRatio"] = 0.71 + elif change == "memory": + host["availableMemoryBytes"] = 15 * 1024**3 - 1 + elif change == "cache": + host["cacheFreeBytes"] = 0 + elif change == "swap": + host["swapCritical"] = True + elif change == "stale": + now[0] = 253 + elif change == "future": + host["swapSampleAt"] = 163 + elif change == "pressure": + capacity["decision"]["pressure"] = "critical" + elif change == "guard": + monkeypatch.setattr(controller, "_host_evidence", lambda: ("blocked", "runner_capacity_pressure_critical", capacity)) + assert controller._fresh_mutation_gate( + "backend", ["backend"], state, minimum_sample_at=162 + )[0] is not None + assert state["hostStability"] is None + + +def test_cached_post_start_wait_never_shortens_required_stability(controller_module, monkeypatch): + controller, now, host, _ = _host_gate_sample(controller_module, monkeypatch) + state = {} + controller._fresh_mutation_gate("backend", ["backend"], state) + now[0] = 130 + host["swapSampleAt"] = 130 + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" + now[0] = 131 + assert controller._fresh_mutation_gate( + "backend", ["backend"], state, minimum_sample_at=130 + )[0] == "runner_admission_host_telemetry_not_refreshed" + now[0] = 140 + host["swapSampleAt"] = 140 + assert controller._fresh_mutation_gate( + "backend", ["backend"], state, minimum_sample_at=130 + )[0] == "runner_admission_host_headroom_not_stable" + assert state["hostStability"]["since"] == 100 + + @pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"]) def test_unsafe_sample_restarts_the_entire_stability_window(controller_module, monkeypatch, change): module = controller_module @@ -1539,6 +1602,142 @@ def test_busy_controller_owned_runner_gets_a_durable_disable_but_job_is_not_stop assert claims == ["backend"] +@pytest.mark.parametrize("busy_flag,active_inventory", [(True, False), (False, True), (True, True)]) +def test_busy_disabled_drain_does_not_starve_other_profile( + controller_module, monkeypatch, busy_flag, active_inventory +): + module = controller_module + waiting = {"id": "waiting-backend", "labels": ["itworx-ci-backend"], "queued_at": time.time() - 30} + controller, state, observation, _inventory, containers = _lease_fixture( + module, enabled=("frontend",), busy=("frontend",), queue=(waiting,) + ) + profile = module.RUNNER_PROFILES["frontend"] + runner_id = observation["profileRunnerIds"]["frontend"] + row = observation["runnerRows"][profile.container_name] + row.update(disabled=True, busy=busy_flag) + state["profiles"]["frontend"].update(enabledByController=False, drainOwned=True) + if not active_inventory: + observation["activeJobs"][runner_id] = [] + stops = [] + monkeypatch.setattr(controller, "_stop_drained_profile", lambda *args, **kwargs: stops.append(kwargs) or "runner_admission_drain_stop_deferred") + + assert controller._drain_one_idle_profile(state, observation) is None + assert stops == [] + assert containers[profile.container_name]["running"] is True + assert state["profiles"]["frontend"]["drainOwned"] is True + assert observation["claims"] == ["frontend"] + choice = module.select_admission( + observation["queue"], available_profiles=["backend"], active_claims=observation["claims"] + ) + assert choice.profile_id == "backend" + + +@pytest.mark.parametrize("blocked", [None, "resource", "host", "capacity"]) +@pytest.mark.parametrize("projection_status", ["healthy", "degraded"]) +def test_cycle_can_select_second_slot_during_busy_drain_but_keeps_guards( + controller_module, monkeypatch, blocked, projection_status +): + module = controller_module + now = time.time() + waiting = {"id": "waiting-backend", "labels": ["itworx-ci-backend"], "queued_at": now - 30} + controller, state, observation, _inventory, _containers = _lease_fixture( + module, enabled=("frontend",), busy=("frontend",), queue=(waiting,) + ) + profile = module.RUNNER_PROFILES["frontend"] + observation["runnerRows"][profile.container_name]["disabled"] = True + state["profiles"]["frontend"].update(enabledByController=False, drainOwned=True) + state["terminalHistoryUpdatedAt"] = now + observation.update(specialization="active", at=now, resourceProof={"ready": True}, hostReason=None) + if blocked == "resource": + observation.update(resourceProof=None, resourceReason="runner_admission_resource_attestation_unready") + elif blocked == "host": + observation["hostReason"] = "runner_capacity_pressure_critical" + elif blocked == "capacity": + observation["claims"].append("build") + selected = [] + monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) + monkeypatch.setattr(controller, "_read_state", lambda: state) + monkeypatch.setattr(controller, "_save_state", lambda _state: None) + monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation) + monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False) + monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False) + monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False) + monkeypatch.setattr(controller, "_record_idle_state", lambda *_args: False) + monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": projection_status, "reason": None}) + monkeypatch.setattr(controller, "_write_projection", lambda _projection: None) + monkeypatch.setattr(controller, "_stop_drained_profile", lambda *_args, **_kwargs: pytest.fail("busy job must be retained")) + monkeypatch.setattr(controller, "_continue_pending", lambda *_args, **kwargs: selected.append(kwargs["reserved_profile_id"]) or "test_selected") + + result = controller.cycle() + + assert selected == (["backend"] if blocked is None else []) + assert observation["claims"] == (["frontend", "build"] if blocked == "capacity" else ["frontend"]) + assert state["profiles"]["frontend"]["drainOwned"] is True + if blocked is not None: + assert state["pending"] is None + assert result["status"] != "test_selected" + assert result["statistics"]["reason"] == result["status"] + + +@pytest.mark.parametrize("stage", ["reserved", "started"]) +@pytest.mark.parametrize("blocked", [None, "operator", "fresh_queue", "guard"]) +def test_pending_start_does_not_starve_guarded_idle_lease_cleanup( + controller_module, monkeypatch, stage, blocked +): + module = controller_module + now = time.time() + waiting = {"id": "queued-backend", "labels": [module.RUNNER_PROFILES["backend"].runner_label], "queued_at": now - 600} + controller, state, observation, inventory, containers = _lease_fixture( + module, enabled=("checks",), queue=(waiting,) + ) + checks = module.RUNNER_PROFILES["checks"] + checks_id = observation["profileRunnerIds"]["checks"] + pending = {"profileId": "backend", "runnerId": observation["profileRunnerIds"]["backend"], "jobId": waiting["id"], "createdAt": module._utc_iso(now - 200), "stage": stage} + state.update(pending=pending, terminalHistoryUpdatedAt=module._utc_iso(now)) + state["profiles"]["checks"]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1 + observation.update(specialization="active", at=now, resourceProof={"ready": True}, claims=["checks", "backend"]) + if stage == "started": + containers[module.RUNNER_PROFILES["backend"].container_name].update(running=True, health="healthy") + if blocked == "operator": + state["profiles"]["checks"]["operatorEnabled"] = True + queue_rows = [_raw_job(module, waiting)] + if blocked == "fresh_queue": + queued_check = {"id": "queued-checks", "labels": [checks.runner_label], "queued_at": now - 10} + observation["queue"].append(queued_check) + queue_rows.append(_raw_job(module, queued_check)) + _install_lease_io(monkeypatch, controller, module, inventory, containers, queue_rows=queue_rows) + if blocked == "guard": + monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("blocked", "guard")) + patches, continued = [], [] + def disable(runner_id, value): + patches.append((runner_id, value)) + next(row for row in inventory if row["id"] == runner_id)["disabled"] = value + monkeypatch.setattr(controller, "_set_disabled", disable) + monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) + monkeypatch.setattr(controller, "_read_state", lambda: state) + monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation) + monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False) + monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False) + monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False) + monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": "healthy"}) + monkeypatch.setattr(controller, "_write_projection", lambda *_args: None) + monkeypatch.setattr(controller, "_continue_pending", lambda *_args, **_kwargs: continued.append(True) or "runner_admission_host_headroom_insufficient") + result = controller.cycle() + assert state["pending"] is pending + if blocked is None: + assert result["status"] == "runner_admission_drained_idle_profile" + assert patches == [(checks_id, True)] and continued == [] + assert containers[checks.container_name]["running"] is False + else: + assert patches == [] and containers[checks.container_name]["running"] is True + expected = { + "operator": "runner_admission_host_headroom_insufficient", + "fresh_queue": "runner_admission_drain_deferred_queue", + "guard": "runner_admission_drain_state_changed", + } + assert result["status"] == expected[blocked] + + def test_cycle_fences_a_busy_owned_lease_even_without_a_resource_proof( controller_module, monkeypatch ): diff --git a/backend/tests/test_runner_inventory_visibility.py b/backend/tests/test_runner_inventory_visibility.py new file mode 100644 index 0000000..784a994 --- /dev/null +++ b/backend/tests/test_runner_inventory_visibility.py @@ -0,0 +1,93 @@ +"""Complete display inventory must never authorize recovery of fenced runners.""" + +from datetime import UTC, datetime, timedelta + +from backend.app.core.config import settings +from backend.tests.test_runner_scheduler_liveness_recovery import _configure + + +def _snapshot( + monkeypatch, *, busy=False, queued=False, apply_filter=True, incomplete=False +): + connector = _configure(monkeypatch) + monkeypatch.setitem( + settings.apis, + "GITEA_RUNNER_TARGETS_JSON", + '{"linux-validation":"itworx-ci-backend"}', + ) + runners = [ + { + "id": 71 + index, + "name": f"itworx-ci-{profile}", + "status": "online" if busy and profile == "backend" else "offline", + "busy": busy and profile == "backend", + "disabled": True, + "labels": ["linux-validation"], + "token": "fixture-secret", + } + for index, profile in enumerate( + ("checks", "backend", "frontend", "security", "browser", "build") + ) + ] + requests = [] + + def request(path, params=None): + requests.append((path, params)) + if path.endswith("/runners"): + rows = [] if apply_filter and params.get("disabled") == "false" else runners + return {"total_count": len(rows) + int(incomplete), "runners": rows} + jobs = [] + if queued and params.get("status") == "queued": + jobs = [ + { + "id": 9, + "status": "queued", + "runner_id": 0, + "labels": ["linux-validation"], + "created_at": ( + datetime.now(UTC) - timedelta(minutes=20) + ).isoformat(), + } + ] + return {"total_count": len(jobs), "jobs": jobs} + + monkeypatch.setattr(connector, "_request", request) + return connector.snapshot(refresh=True), requests + + +def test_standby_fleet_remains_visible_and_does_not_appear_broken(monkeypatch): + result, requests = _snapshot(monkeypatch) + assert len(result["runners"]) == 6 + assert result["summary"]["disabled"] == 6 + assert result["summary"]["available"] == 0 + assert result["inventory"] == {"scope": "all_registrations", "complete": True} + assert result["severity"] == "healthy" + assert result["recommendedAction"] is None + assert "fixture-secret" not in repr(result) + assert requests[0][1] == {"page": 1, "limit": 100} + + +def test_fenced_busy_runner_is_counted_without_becoming_claimable(monkeypatch): + result, _ = _snapshot(monkeypatch, busy=True) + assert result["summary"]["busy"] == 1 + assert result["summary"]["online"] == 0 + assert result["summary"]["available"] == 0 + assert result["summary"]["disabled"] == 6 + assert result["runners"][1]["busy"] is True + assert result["runners"][1]["disabled"] is True + + +def test_disabled_runner_never_receives_restart_guidance_for_a_stalled_queue( + monkeypatch, +): + result, _ = _snapshot(monkeypatch, queued=True, apply_filter=False) + assert result["severity"] == "critical" + assert result["queue"]["queued"] == 1 + assert result["recommendedAction"] is None + + +def test_incomplete_runner_sample_is_not_reported_as_complete(monkeypatch): + result, _ = _snapshot(monkeypatch, incomplete=True) + assert len(result["runners"]) == 6 + assert result["inventory"] == {"scope": "all_registrations", "complete": False} + assert result["recommendedAction"] is None