From 2de6e5c85e7841977ad419c1948ac354f58286eb Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Fri, 2 Oct 2026 16:00:16 +0000 Subject: [PATCH] Publish curated Unraid-AppOps-Gateway source from 3c5fb0c6d1f4e693e4a0fa50e7151509b4450786 --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 16 +- .../services/controller_artifact_promoter.py | 3 +- .../tests/test_appops_live_state_recovery.py | 262 +++++++++ .../tests/test_runner_fleet_cutover_safety.py | 509 ++++++++++++++++++ 5 files changed, 786 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_appops_live_state_recovery.py create mode 100644 backend/tests/test_runner_fleet_cutover_safety.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 87a5722..657e609 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `a898651d5411fca177825aa908a157a34f4ce4a8`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `3c5fb0c6d1f4e693e4a0fa50e7151509b4450786`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index c34aa58..b8fd65a 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "a898651d5411fca177825aa908a157a34f4ce4a8", + "sourceRevision": "3c5fb0c6d1f4e693e4a0fa50e7151509b4450786", "files": [ { "path": ".dockerignore", @@ -489,8 +489,8 @@ }, { "path": "backend/app/services/controller_artifact_promoter.py", - "sha256": "557cc8dfc7fd5f8625dc36eab9bdef730b7a4cc8fa9bf273954b4aabae190ab0", - "bytes": 16381 + "sha256": "a193123dd59de03f19cc14e51e9551e0ab316b741cc8b417ed530f86c539dc13", + "bytes": 16370 }, { "path": "backend/app/services/controller_bootstrap_import.py", @@ -797,6 +797,11 @@ "sha256": "80c9444a354c1bee6114ae004b0c7d9c1e72834afd2b2dc04ed7c6b738119150", "bytes": 47199 }, + { + "path": "backend/tests/test_appops_live_state_recovery.py", + "sha256": "33b5fa25e441128f84c4e518cef803fa9229ffe04a6ec4e980a3bb2707175959", + "bytes": 12368 + }, { "path": "backend/tests/test_appops_self_upgrade_controller.py", "sha256": "ad6d8fba2213c1c8018bc9ff1c2ebd61684792c27fa00a07195c5d6aea99d69f", @@ -1207,6 +1212,11 @@ "sha256": "a442a54748caf77f912d05b7ed9d68645db502d1a2a96cd1838501343bccdf90", "bytes": 5966 }, + { + "path": "backend/tests/test_runner_fleet_cutover_safety.py", + "sha256": "85f93b1e70114ea1b86bdc4adeb05ea18b91e2886c5ed0dd44474991ee3ddda9", + "bytes": 15917 + }, { "path": "backend/tests/test_runner_fleet_scheduler_recovery.py", "sha256": "567c59526c430c61c7823a984fb8c2a7eb17c5614e3e0a28a2b25b9af85fc55f", diff --git a/backend/app/services/controller_artifact_promoter.py b/backend/app/services/controller_artifact_promoter.py index e7d88ef..2c8feb5 100644 --- a/backend/app/services/controller_artifact_promoter.py +++ b/backend/app/services/controller_artifact_promoter.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import re import time from datetime import datetime @@ -20,7 +19,7 @@ GITEA_REPOSITORY = "Unraid-AppOps-Gateway" REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)" HELPER_NAME = "itworx-controller-artifact-promoter" CONTROLLER_CONTAINER = "gitea-deploy-control" -CONTROLLER_VERSION = "2.1.5" +CONTROLLER_VERSION = "2.1.13" HELPER_LABEL = "itworx.controller.promoter" _HELPER_TIMEOUT_SECONDS = 240 _REVISION = re.compile(r"^[0-9a-f]{40}$") diff --git a/backend/tests/test_appops_live_state_recovery.py b/backend/tests/test_appops_live_state_recovery.py new file mode 100644 index 0000000..c3f91c3 --- /dev/null +++ b/backend/tests/test_appops_live_state_recovery.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import os +import sys +import types +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +REPOSITORY = "jens/unraid-appops-gateway" +CONTAINER = "Unraid-AppOps-Gateway" +OBSERVED = "5" * 40 +RECORDED = "6" * 40 +ATTEMPT = "a" * 40 +IMAGE = "sha256:" + "5" * 64 +OLD_IMAGE = "sha256:" + "6" * 64 + + +def receipt(revision=RECORDED, image=OLD_IMAGE): + tree = "b" * 40 + digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest() + built = "2026-10-02T00:00:00.000Z" + return { + "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY, + "revision": revision, "tree": tree, "treeDigest": digest, + "build": {"id": revision[:12], "repository": REPOSITORY, + "revision": revision, "tree": tree, "treeDigest": digest, + "builtAt": built}, + "containers": {CONTAINER: {"image_id": image, + "image_revision": revision, "image_source": REPOSITORY, + "image_tree": tree, "image_version": revision[:12], + "image_created": built}}, "live": None, + } + + +@pytest.fixture +def recovery(monkeypatch, tmp_path): + if "fcntl" not in sys.modules: + shim = types.ModuleType("fcntl") + shim.LOCK_EX, shim.LOCK_UN = 2, 8 + shim.flock = lambda *_args: None + monkeypatch.setitem(sys.modules, "fcntl", shim) + spec = importlib.util.spec_from_file_location( + "appops_recovery_test", ROOT / "ops/gitea-deploy-broker/deploy.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + state_root = tmp_path / "state" + state_root.mkdir(mode=0o700) + monkeypatch.setattr(module, "STATE_ROOT", state_root) + monkeypatch.setattr(module, "require_host_root", lambda: None) + monkeypatch.setattr(module, "_root_owned", lambda _path: True) + if os.name == "nt": + # Windows does not expose POSIX write bits; keep real inode/type checks. + real_lstat = Path.lstat + def posix_permissions(path): + observed = real_lstat(path) + return types.SimpleNamespace(st_mode=observed.st_mode & ~0o022, + st_dev=observed.st_dev, st_ino=observed.st_ino) + monkeypatch.setattr(Path, "lstat", posix_permissions) + fsyncs = [] + monkeypatch.setattr(module, "_fsync_directory", lambda path: fsyncs.append(path)) + project = {"slug": "unraid-appops-gateway", "branch": "main", + "gitea_repository_id": 9, "containers": [CONTAINER], + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], "verify_oci_revision": True, + "environment": {"APPOPS_DATA_PATH": str(module.APPOPS_DATA_ROOT)}, + "source_dir": str(tmp_path), "_repository_name": REPOSITORY} + (tmp_path / "docker-compose.yml").write_text("services: {}\n") + public = {key: value for key, value in project.items() if not key.startswith("_")} + monkeypatch.setattr(module, "load_projects", lambda: {REPOSITORY: copy.deepcopy(public)}) + def validate(_repo, branch, revision): + assert branch == "main" + if revision not in {OBSERVED, ATTEMPT}: + raise module.DeployError("revision is not a protected ancestor") + return revision + monkeypatch.setattr(module, "validate_commit", validate) + source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" + labels = {"org.opencontainers.image.revision": OBSERVED, + "org.opencontainers.image.source": source, + "com.docker.compose.project": "unraid-appops-gateway"} + container = {"Id": "c" * 64, "Image": IMAGE, + "State": {"Running": True, "Health": {"Status": "healthy"}}, + "Config": {"Image": "appops:observed", "Labels": labels}, + "NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}}, + "Mounts": [{"Destination": "/data", "Type": "bind", "RW": True, + "Source": str(module.APPOPS_DATA_ROOT)}]} + image = {"Id": IMAGE, "Config": {"Labels": copy.deepcopy(labels)}} + monkeypatch.setattr(module, "inspect_container", lambda name: copy.deepcopy(container)) + monkeypatch.setattr(module, "inspect_image", lambda name: copy.deepcopy(image)) + monkeypatch.setattr(module, "compose_project_container_names", lambda name: {CONTAINER}) + def output(command, **_kwargs): + assert command == ["docker", "ps", "-a", "--format", "{{.Names}}"] + return CONTAINER + "\n" + monkeypatch.setattr(module, "output", output) + def forbidden(*_args, **_kwargs): + pytest.fail("recovery attempted a host mutation") + monkeypatch.setattr(module, "run", forbidden) + prior = {"schema_version": 3, "repository": REPOSITORY, + "slug": project["slug"], "compose_project": project["compose_project"], + "revision": RECORDED, "live_revision": RECORDED, + "status": "failed", "live_status": "unknown", "live_legacy_provenance": False, + "live_artifacts": {"containers": {CONTAINER: {"image_id": OLD_IMAGE}}}, + "live_receipt": receipt(), "rollback_snapshot": {"revision": "7" * 40}, + "last_attempt": {"revision": ATTEMPT, "status": "failed", + "message": module._APPOPS_RECOVERY_FAILURE}} + path = state_root / "unraid-appops-gateway.json" + def save(): + path.write_text(json.dumps(prior), encoding="utf-8") + path.chmod(0o600) + save() + return types.SimpleNamespace(m=module, project=project, repo=tmp_path, + path=path, prior=prior, container=container, image=image, + public=public, save=save, fsyncs=fsyncs) + + +def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery): + r = recovery + before = r.path.read_bytes() + compose = (r.repo / "docker-compose.yml").read_bytes() + result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + state = json.loads(r.path.read_bytes()) + backup = r.path.parent / result["backupFile"] + assert backup.read_bytes() == before + assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest() + assert state["last_attempt"] == r.prior["last_attempt"] + assert state["live_revision"] == OBSERVED + assert state["live_legacy_provenance"] is True + assert state["live_status"] == "healthy" + assert state["live_receipt"] == state["rollback_snapshot"] == {} + assert state["live_artifacts"]["compose_sha256"] == "" + assert state["live_artifacts"]["containers"][CONTAINER]["image_id"] == IMAGE + assert state["recovery"]["retired_receipt_revision"] == RECORDED + assert (r.repo / "docker-compose.yml").read_bytes() == compose + assert r.fsyncs == [r.path.parent, r.path.parent] + + +@pytest.mark.parametrize("field,value", [ + ("schema_version", 2), ("status", "success"), ("live_status", "healthy"), + ("live_legacy_provenance", None), ("live_legacy_provenance", 0), + ("repository", "jens/other"), ("live_revision", OBSERVED), + ("rollback_snapshot", None), + ("live_receipt", None), ("live_receipt", {}), ("live_receipt", "invalid"), + ("live_artifacts", None), ("live_artifacts", {}), +]) +def test_rejects_ineligible_recorded_state(recovery, field, value): + r = recovery + r.prior[field] = value + r.save() + before = r.path.read_bytes() + with pytest.raises(r.m.DeployError): + r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + assert r.path.read_bytes() == before + assert not list(r.path.parent.glob("*.bak")) + + +@pytest.mark.parametrize("failure", ["wrong-failure", "non-ancestor", "same-image", "receipt-image", "receipt-revision"]) +def test_rejects_unproven_historical_drift(recovery, failure): + r = recovery + if failure == "wrong-failure": + r.prior["last_attempt"]["message"] = "Deployment failed" + elif failure == "non-ancestor": + r.prior["last_attempt"]["revision"] = "d" * 40 + elif failure == "same-image": + r.prior["live_artifacts"]["containers"][CONTAINER]["image_id"] = IMAGE + elif failure == "receipt-image": + r.prior["live_receipt"]["containers"][CONTAINER]["image_id"] = IMAGE + else: + r.prior["live_receipt"]["revision"] = OBSERVED + r.save() + before = r.path.read_bytes() + with pytest.raises(r.m.DeployError): + r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + assert r.path.read_bytes() == before + + +@pytest.mark.parametrize("failure", ["unhealthy", "volume", "readonly", "mount-path", "port", "image", "source", "revision", "extra-container", "config", "owner"]) +def test_rejects_wrong_actual_runtime_or_target(recovery, monkeypatch, failure): + r = recovery + if failure == "unhealthy": + r.container["State"]["Health"]["Status"] = "unhealthy" + elif failure in {"volume", "readonly", "mount-path"}: + key, value = {"volume": ("Type", "volume"), "readonly": ("RW", False), + "mount-path": ("Source", "/other")}[failure] + r.container["Mounts"][0][key] = value + elif failure == "port": + r.container["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostPort"] = "9999" + elif failure == "image": + r.image["Id"] = OLD_IMAGE + elif failure in {"source", "revision"}: + r.image["Config"]["Labels"]["org.opencontainers.image." + failure] = "wrong" + elif failure == "extra-container": + monkeypatch.setattr(r.m, "compose_project_container_names", lambda name: {CONTAINER, "other"}) + elif failure == "config": + r.public["branch"] = "other" + else: + monkeypatch.setattr(r.m, "_root_owned", lambda path: False) + before = r.path.read_bytes() + with pytest.raises(r.m.DeployError): + r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + assert r.path.read_bytes() == before + + +@pytest.mark.parametrize("change", ["bytes", "inode", "runtime", "config"]) +def test_rechecks_after_durable_backup_before_replace(recovery, monkeypatch, change): + r = recovery + def after_backup(_path): + if change == "bytes": + r.path.write_bytes(b'{"external": true}') + elif change == "inode": + replacement = r.path.with_suffix(".replacement") + replacement.write_bytes(r.path.read_bytes()) + replacement.chmod(0o600) + replacement.replace(r.path) + elif change == "runtime": + r.container["Id"] = "d" * 64 + else: + r.public["branch"] = "changed" + monkeypatch.setattr(r.m, "_fsync_directory", after_backup) + with pytest.raises(r.m.DeployError): + r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + state = json.loads(r.path.read_bytes()) + assert "recovery" not in state + assert len(list(r.path.parent.glob("*.bak"))) == 1 + assert not list(r.path.parent.glob("*.tmp")) + + +def test_next_normal_delivery_keeps_recovery_audit(recovery): + r = recovery + result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + audit = json.loads(r.path.read_bytes())["recovery"] + r.m.write_state(REPOSITORY, r.project, OBSERVED, "success", "Verified normal deployment", + live_receipt=receipt(OBSERVED, IMAGE), live_legacy_provenance=False) + state = json.loads(r.path.read_bytes()) + assert state["recovery"] == audit + assert (r.path.parent / result["backupFile"]).exists() + assert state["live_receipt"]["revision"] == OBSERVED + assert state["live_legacy_provenance"] is False + assert state["last_attempt"]["message"] == "Verified normal deployment" + + +def test_post_replace_fsync_failure_reports_applied_unknown_and_retains_backup(recovery, monkeypatch): + r = recovery + calls = 0 + def fail_after_replace(_path): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("synthetic directory fsync failure") + monkeypatch.setattr(r.m, "_fsync_directory", fail_after_replace) + before = r.path.read_bytes() + with pytest.raises(r.m.DeployError, match="state was replaced but durability is unknown"): + r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + state = json.loads(r.path.read_bytes()) + assert state["live_revision"] == OBSERVED + assert (r.path.parent / state["recovery"]["backup_file"]).read_bytes() == before + with pytest.raises(r.m.DeployError, match="exact failed pre-mutation"): + r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) diff --git a/backend/tests/test_runner_fleet_cutover_safety.py b/backend/tests/test_runner_fleet_cutover_safety.py new file mode 100644 index 0000000..e2cd8a8 --- /dev/null +++ b/backend/tests/test_runner_fleet_cutover_safety.py @@ -0,0 +1,509 @@ +from __future__ import annotations + +import importlib.util +import json +import shlex +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py" +ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs" +UNSET = object() +SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh" + +PRIMARY = "itworx-security-runner" +SECONDARY = "itworx-validation-runner-2" +LEGACY = "itworx-unraid-deploy" +OTHER_FIXED_RUNNERS = ( + "itworx-validation-runner-3", + "itworx-docker-runner", + "itworx-browser-runner", + "itworx-validation-runner-burst-1", +) + + +@pytest.fixture +def upgrade_module(): + spec = importlib.util.spec_from_file_location( + "appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _runner( + runner_id: int, + name: str, + *, + busy: object = False, + disabled: object = False, +) -> dict[str, object]: + return { + "id": runner_id, + "name": name, + "busy": busy, + "disabled": disabled, + } + + +def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]: + runners = [ + _runner(1, PRIMARY, disabled=True), + _runner(2, SECONDARY, disabled=True), + ] + if include_legacy: + runners.append(_runner(3, LEGACY, disabled=True)) + return runners + + +class ServiceStub: + def __init__( + self, + runners: list[dict[str, object]] | None = None, + jobs: object = UNSET, + *, + inventory_error: Exception | None = None, + request_error: Exception | None = None, + ) -> None: + self.runners = list(runners if runners is not None else _safe_runners()) + self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs + self.inventory_error = inventory_error + self.request_error = request_error + self.requests: list[tuple[str, dict[str, object]]] = [] + + def _gitea_runner_inventory(self) -> list[dict[str, object]]: + if self.inventory_error is not None: + raise self.inventory_error + return self.runners + + def _gitea_request( + self, path: str, params: dict[str, object] | None = None + ) -> object: + self.requests.append((path, dict(params or {}))) + if self.request_error is not None: + raise self.request_error + return self.jobs + + +def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]: + return module.require_cutover_quiescence(service) + + +def _assert_rejected(module: Any, service: ServiceStub) -> None: + with pytest.raises(RuntimeError): + _require_quiescent(module, service) + + +def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled( + upgrade_module, +): + service = ServiceStub( + jobs={ + "jobs": [ + {"id": 101, "runner_id": 1, "status": "in_progress"}, + ], + "total_count": 1, + } + ) + + _assert_rejected(upgrade_module, service) + + assert len(service.requests) == 1 + assert service.requests[0][0] == ACTIVE_JOBS_PATH + assert service.requests[0][1].get("status") == "in_progress" + assert service.requests[0][1].get("limit") == 100 + + +@pytest.mark.parametrize( + "runners", + [ + [_runner(2, SECONDARY, disabled=True)], + [_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)], + [ + _runner(1, PRIMARY, busy=True, disabled=True), + _runner(2, SECONDARY, disabled=True), + ], + ], + ids=("primary-missing", "primary-not-fenced", "primary-busy"), +) +def test_primary_runner_must_be_unique_present_disabled_and_idle( + upgrade_module, runners +): + _assert_rejected(upgrade_module, ServiceStub(runners=runners)) + + +@pytest.mark.parametrize( + "runners", + [ + [_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)], + [ + _runner(1, PRIMARY, disabled=True), + _runner(2, SECONDARY, busy=True, disabled=True), + ], + ], + ids=("secondary-not-fenced", "secondary-busy"), +) +def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners): + _assert_rejected(upgrade_module, ServiceStub(runners=runners)) + + +def test_duplicate_primary_registration_is_rejected(upgrade_module): + runners = _safe_runners() + runners.append(_runner(11, PRIMARY, disabled=True)) + + _assert_rejected(upgrade_module, ServiceStub(runners=runners)) + + +@pytest.mark.parametrize("bad_value", [None, "false", 0, 1]) +def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value): + busy_runners = _safe_runners() + busy_runners[0]["busy"] = bad_value + _assert_rejected(upgrade_module, ServiceStub(runners=busy_runners)) + + disabled_runners = _safe_runners() + disabled_runners[0]["disabled"] = bad_value + _assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners)) + + +@pytest.mark.parametrize( + "legacy", + [ + _runner(3, LEGACY, disabled=False), + _runner(3, LEGACY, busy=True, disabled=True), + ], + ids=("legacy-enabled", "legacy-busy"), +) +def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy): + runners = _safe_runners() + runners.append(legacy) + + _assert_rejected(upgrade_module, ServiceStub(runners=runners)) + + +@pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS) +def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name): + runners = _safe_runners() + runners.append(_runner(10, name, busy=True, disabled=False)) + + _assert_rejected(upgrade_module, ServiceStub(runners=runners)) + + +def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module): + runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")] + _assert_rejected(upgrade_module, ServiceStub(runners=runners)) + + +@pytest.mark.parametrize( + "jobs", + [ + None, + [], + {}, + {"jobs": []}, + {"jobs": [], "total_count": 1}, + {"jobs": [], "total_count": True}, + {"jobs": [], "total_count": "0"}, + {"jobs": None, "total_count": 0}, + {"jobs": {}, "total_count": 0}, + {"jobs": [None], "total_count": 1}, + {"jobs": ["not-a-job"], "total_count": 1}, + {"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1}, + {"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1}, + ], + ids=( + "response-null", + "response-list", + "jobs-field-missing", + "jobs-count-missing", + "jobs-count-mismatch", + "jobs-count-bool", + "jobs-count-not-integer", + "jobs-null", + "jobs-not-list", + "job-null", + "job-not-object", + "job-status-missing", + "job-status-mismatch", + ), +) +def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs): + _assert_rejected(upgrade_module, ServiceStub(jobs=jobs)) + + +def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module): + jobs = { + "jobs": [ + {"id": index + 1, "runner_id": 90, "status": "in_progress"} + for index in range(100) + ], + "total_count": 100, + } + + _assert_rejected( + upgrade_module, + ServiceStub( + runners=[ + *_safe_runners(), + _runner(90, "known-unrelated-runner", disabled=True), + ], + jobs=jobs, + ), + ) + + +@pytest.mark.parametrize( + "runner_id", + [None, True, False, 0, -1, "1", 999], + ids=("missing", "true", "false", "zero", "negative", "string", "unknown"), +) +def test_active_job_with_invalid_or_unknown_runner_id_fails_closed( + upgrade_module, runner_id +): + jobs = { + "jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}], + "total_count": 1, + } + + _assert_rejected(upgrade_module, ServiceStub(jobs=jobs)) + + +def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false( + upgrade_module, +): + runners = [ + *_safe_runners(), + _runner(10, "itworx-docker-runner", busy=False, disabled=True), + ] + service = ServiceStub( + runners=runners, + jobs={ + "jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}], + "total_count": 1, + }, + ) + + _assert_rejected(upgrade_module, service) + + +@pytest.mark.parametrize("where", ["inventory", "jobs"]) +def test_gitea_api_errors_fail_closed(upgrade_module, where): + service = ServiceStub( + inventory_error=OSError("inventory unavailable") + if where == "inventory" + else None, + request_error=OSError("jobs unavailable") if where == "jobs" else None, + ) + + expected_message = ( + "inventory unavailable" if where == "inventory" else "jobs unavailable" + ) + with pytest.raises(OSError, match=expected_message): + _require_quiescent(upgrade_module, service) + + +def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module): + service = ServiceStub( + runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")], + jobs={ + "jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}], + "total_count": 1, + }, + ) + _assert_rejected(upgrade_module, service) + + +def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch): + def unexpected_shell(*_args, **_kwargs): + raise AssertionError("cutover guard must not execute Docker or shell commands") + + monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell) + unrelated = _runner(90, "known-unrelated-runner", disabled=True) + service = ServiceStub( + runners=[*_safe_runners(include_legacy=True), unrelated], + jobs={"jobs": [], "total_count": 0}, + ) + + result = _require_quiescent(upgrade_module, service) + + assert result == { + "status": "quiescent", + "fencedRunners": [PRIMARY, SECONDARY, LEGACY], + "activeFleetJobs": 0, + } + assert service.requests == [ + ( + ACTIVE_JOBS_PATH, + { + "status": "in_progress", + "page": 1, + "limit": 100, + "sort": "id", + "order": "asc", + }, + ) + ] + + +def _bash_function(name: str) -> str: + script = SCRIPT_PATH.read_text(encoding="utf-8") + start = script.index(f"{name}() {{") + end = script.index("\n}", start) + 2 + return script[start:end] + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") +@pytest.mark.parametrize( + ("proof", "executor_status", "allowed"), + [ + ("", 0, False), + ("not-json", 0, False), + ( + json.dumps( + { + "status": "quiescent", + "fencedRunners": [PRIMARY, SECONDARY], + "activeFleetJobs": 1, + } + ), + 0, + False, + ), + ( + json.dumps( + { + "status": "quiescent", + "fencedRunners": [PRIMARY, SECONDARY], + "activeFleetJobs": 0, + } + ), + 1, + False, + ), + ( + json.dumps( + { + "status": "quiescent", + "fencedRunners": [PRIMARY, SECONDARY], + "activeFleetJobs": 0, + } + ), + 0, + True, + ), + ], + ids=( + "old-executor-empty", + "malformed-proof", + "active-job", + "executor-error", + "quiescent", + ), +) +def test_cutover_shell_requires_real_successful_quiescence_proof( + proof, executor_status, allowed +): + # The first python3 invocation stands in for the installed executor; the second + # invokes the actual JSON parser inside check_cutover. + body = "\n".join( + [ + "ROOT=/synthetic", + f"MOCK_PROOF={shlex.quote(proof)}", + f"MOCK_EXECUTOR_STATUS={executor_status}", + "python3() {", + ' if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then', + ' printf "%s" "$MOCK_PROOF"', + ' return "$MOCK_EXECUTOR_STATUS"', + " fi", + ' command python3 "$@"', + "}", + _bash_function("check_cutover"), + "check_cutover", + ] + ) + result = subprocess.run( + ["bash"], input=body.encode(), capture_output=True, check=False + ) + assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") +@pytest.mark.parametrize( + ("inventory", "docker_status", "allowed"), + [("", 1, False), ("running-container-id", 0, False), ("", 0, True)], + ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"), +) +def test_dind_idle_shell_requires_successful_empty_docker_inventory( + inventory, docker_status, allowed +): + body = "\n".join( + [ + "DIND=synthetic-dind", + f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}", + f"MOCK_DOCKER_STATUS={docker_status}", + 'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }', + _bash_function("check_dind_idle"), + "check_dind_idle", + ] + ) + result = subprocess.run( + ["bash"], input=body.encode(), capture_output=True, check=False + ) + assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") +@pytest.mark.parametrize( + ("cutover_started", "dispatch_started"), + [(0, 0), (1, 0), (1, 1)], + ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"), +) +def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started): + body = "\n".join( + [ + "backup=/synthetic-backup", + "COMPOSE_FILE=/synthetic-compose", + "PRIMARY_CONFIG=/synthetic-primary", + "SECONDARY_CONFIG=/synthetic-secondary", + "ROOT=/synthetic-root", + "guard_changed=0", + "host_guard_changed=0", + f"cutover_started={cutover_started}", + f"dispatch_started={dispatch_started}", + 'new_runners=("new-container")', + 'registered=("new-registration")', + 'fenced=("itworx-security-runner:false")', + "created_paths=()", + "exec 3>&1", + 'cp() { printf "cp:%s\\n" "$*"; }', + 'docker() { printf "docker:%s\\n" "$*" >&3; }', + 'delete_runner() { printf "delete:%s\\n" "$*"; }', + 'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }', + 'stop_capacity_daemon() { printf "stop-daemon\\n"; }', + 'rm() { printf "rm:%s\\n" "$*"; }', + _bash_function("restore"), + "false", + "restore", + ] + ) + result = subprocess.run( + ["bash"], input=body.encode(), capture_output=True, check=False + ) + output = result.stdout.decode(errors="replace") + errors = result.stderr.decode(errors="replace") + assert result.returncode == 1, errors + if dispatch_started: + assert output == "" + assert "runner_fleet_recovery_needed" in errors + else: + assert "cp:" in output + assert "delete:new-registration" in output + assert "restore-fence:itworx-security-runner:false" in output + assert ("stop-daemon" in output) is bool(cutover_started) + assert ("docker:compose " in output) is bool(cutover_started)