Publish curated Unraid-AppOps-Gateway source from 21729207a161d87393cfb73e79582bc373aa19cf

This commit is contained in:
Public Source Publisher committed 2026-10-08 01:20:28 +00:00
1 parent ab3fa553f0
commit 76d7249b5e
6 files changed
+1497 -37

No files matched your search

+1 -1
View File
@@ -1,3 +1,3 @@
# Public source export # Public source export
This source snapshot was generated from the private canonical repository at revision `bc001a6df859f7c9836fa145afb739d1ceebc524`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. This source snapshot was generated from the private canonical repository at revision `21729207a161d87393cfb73e79582bc373aa19cf`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
+9 -9
View File
@@ -1,6 +1,6 @@
{ {
"schemaVersion": 1, "schemaVersion": 1,
"sourceRevision": "bc001a6df859f7c9836fa145afb739d1ceebc524", "sourceRevision": "21729207a161d87393cfb73e79582bc373aa19cf",
"files": [ "files": [
{ {
"path": ".dockerignore", "path": ".dockerignore",
@@ -969,13 +969,13 @@
}, },
{ {
"path": "backend/tests/test_controller_hub_release_volume.py", "path": "backend/tests/test_controller_hub_release_volume.py",
"sha256": "4fe32bd783e2535724d75ff892e9b842ecba0a3b3ec6b141de45ab31d96bd945", "sha256": "b629dadd5616d5afd5ff8ad33662d93da829424226f0bb36bda71d9c621049cf",
"bytes": 118768 "bytes": 144585
}, },
{ {
"path": "backend/tests/test_controller_root_procedure_contract.py", "path": "backend/tests/test_controller_root_procedure_contract.py",
"sha256": "964b3e21828a9d815d05ffb0e51acf6e9e45efe56af9fa3b70f39200d3a5c336", "sha256": "2e6a8645674c32f16d251b5cdcdc4f75d72759c4253aebc312b58e766b1b6581",
"bytes": 5125 "bytes": 6445
}, },
{ {
"path": "backend/tests/test_controller_runtime_capabilities.py", "path": "backend/tests/test_controller_runtime_capabilities.py",
@@ -994,8 +994,8 @@
}, },
{ {
"path": "backend/tests/test_deploy_broker.py", "path": "backend/tests/test_deploy_broker.py",
"sha256": "6b1ad652cc7f06ce94105846e15110741fb9514e833101ae670279d5a1711deb", "sha256": "b0e96b1cd2954dc2a6452f5f7c50c5fbae541dc0b4de1e60af11349654594902",
"bytes": 415988 "bytes": 438851
}, },
{ {
"path": "backend/tests/test_deployment_auto_rollback.py", "path": "backend/tests/test_deployment_auto_rollback.py",
@@ -1584,8 +1584,8 @@
}, },
{ {
"path": "backend/tests/test_runner_fleet_cutover_safety.py", "path": "backend/tests/test_runner_fleet_cutover_safety.py",
"sha256": "85f93b1e70114ea1b86bdc4adeb05ea18b91e2886c5ed0dd44474991ee3ddda9", "sha256": "2eb1ec46b706a00a3bb7a5fc6d0d135fccc2d82324ba03296286e3b30a0747a9",
"bytes": 15917 "bytes": 29912
}, },
{ {
"path": "backend/tests/test_runner_fleet_scheduler_recovery.py", "path": "backend/tests/test_runner_fleet_scheduler_recovery.py",
@@ -318,7 +318,10 @@ def _prepare_promotion_doubles(monkeypatch, tmp_path, *, config_valid=True):
raise module.VolumeTransitionError("staged_config_invalid") raise module.VolumeTransitionError("staged_config_invalid")
monkeypatch.setattr(module, "_validate_staged_configuration", validate_staged) monkeypatch.setattr(module, "_validate_staged_configuration", validate_staged)
monkeypatch.setattr(module, "_install_release_leaf", lambda *_args, **_kwargs: None) monkeypatch.setattr(
module, "_install_release_leaf",
lambda revision, *_args, **_kwargs: events.append(f"release_installed:{revision}"),
)
monkeypatch.setattr(module, "_read_root_compose", lambda: (ORIGINAL_COMPOSE, 0o644)) monkeypatch.setattr(module, "_read_root_compose", lambda: (ORIGINAL_COMPOSE, 0o644))
monkeypatch.setattr(module, "_mirror_snapshot_tar", lambda: b"mirror") monkeypatch.setattr(module, "_mirror_snapshot_tar", lambda: b"mirror")
monkeypatch.setattr( monkeypatch.setattr(
@@ -329,6 +332,10 @@ def _prepare_promotion_doubles(monkeypatch, tmp_path, *, config_valid=True):
module, "_verify_compose_image", module, "_verify_compose_image",
lambda *_args, **_kwargs: events.append("compose_verified"), lambda *_args, **_kwargs: events.append("compose_verified"),
) )
monkeypatch.setattr(
module, "_verify_promotion_source_attestation",
lambda *_args, **_kwargs: events.append("source_attested"),
)
monkeypatch.setattr( monkeypatch.setattr(
module, "ensure_backup_volume", module, "ensure_backup_volume",
lambda **_kwargs: events.append("backup_volume_prepared"), lambda **_kwargs: events.append("backup_volume_prepared"),
@@ -360,6 +367,73 @@ def test_promotion_prepares_backup_volume_after_source_stage_controller_and_conf
assert events.index("compose_rendered") < events.index("backup_volume_prepared") assert events.index("compose_rendered") < events.index("backup_volume_prepared")
assert events.count("compose_verified") == 2 assert events.count("compose_verified") == 2
assert events.index("compose_verified") < events.index("backup_volume_prepared") assert events.index("compose_verified") < events.index("backup_volume_prepared")
assert events.count("source_attested") == 4
attestation_after_backup = events.index(
"source_attested", events.index("backup_volume_prepared") + 1
)
assert attestation_after_backup < events.index("release_installed:" + "b" * 40)
def test_promotion_rejects_config_leaf_change_during_staged_validation(monkeypatch, tmp_path):
events = _prepare_promotion_doubles(monkeypatch, tmp_path)
original_config = ({}, {"": 0o755})
changed_config = ({"changed.json": (b"changed", 0o600)}, {"": 0o755})
reads = 0
def read_config():
nonlocal reads
reads += 1
if reads == 1:
return original_config
events.append("config_changed")
return changed_config
monkeypatch.setattr(module, "_read_config_leaf", read_config)
with pytest.raises(module.VolumeTransitionError, match="controller_volume_config_changed"):
module.prepare_promotion(
tmp_path / "stage",
source_revision="d" * 40,
source_tree="e" * 40,
controller_version="2.2.0",
image_id=IMAGE_ID,
)
assert reads == 2
assert events.index("config_validated") < events.index("config_changed")
assert not any(event.startswith("release_installed:") for event in events)
def test_promotion_rejects_legacy_config_source_change_before_config_install(monkeypatch, tmp_path):
events = _prepare_promotion_doubles(monkeypatch, tmp_path)
monkeypatch.setattr(module, "_read_config_leaf", lambda: None)
monkeypatch.setattr(
module, "_config_tree",
lambda entries, _directories, **_kwargs: (
({"changed.json": (b"changed", 0o600)}, {"": 0o755})
if "changed.json" in entries else ({}, {"": 0o755})
),
)
identity = {
"sourceRevision": "b" * 40,
"sourceTree": "c" * 40,
"controllerVersion": "2.1.0",
}
monkeypatch.setattr(
module, "_capture_controller_source",
lambda *_args: ({"changed.json": (b"changed", 0o600)}, {"": 0o755}, identity),
)
with pytest.raises(module.VolumeTransitionError, match="controller_volume_config_changed"):
module.prepare_promotion(
tmp_path / "stage",
source_revision="d" * 40,
source_tree="e" * 40,
controller_version="2.2.0",
image_id=IMAGE_ID,
)
assert not any(event.startswith("release_installed:") for event in events)
def test_promotion_does_not_prepare_backup_volume_when_staged_configuration_fails( def test_promotion_does_not_prepare_backup_volume_when_staged_configuration_fails(
@@ -1226,16 +1300,489 @@ def test_drifted_tag_requires_healthy_current_controller_provenance(compose, mon
elif damage == "interpolation": elif damage == "interpolation":
source_reference = "${CONTROLLER_IMAGE}" source_reference = "${CONTROLLER_IMAGE}"
original = f"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: {source_reference}\n".encode() original = f"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: {source_reference}\n".encode()
if damage: with pytest.raises(module.VolumeTransitionError, match="image_reference_(changed|invalid)"):
with pytest.raises(module.VolumeTransitionError, match="image_reference_(changed|invalid)"): module.rendered_model(original, REVISION, IMAGE_ID, current_container=current)
module.rendered_model(original, REVISION, IMAGE_ID, current_container=current)
else:
rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID, current_container=current))
assert rendered["services"][module.SERVICE]["image"] == IMAGE_ID
assert rendered["services"][module.SERVICE]["pull_policy"] == "never"
assert not any("up" in arguments or "pull" in arguments for arguments, _ in calls) assert not any("up" in arguments or "pull" in arguments for arguments, _ in calls)
def _tag_drift_container():
return {
"Id": "d" * 64,
"Image": IMAGE_ID,
"Name": f"/{module.CONTAINER}",
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Config": {
"Image": "docker.gitea.com/runner-images:ubuntu-latest",
"Labels": {
"com.docker.compose.project": "gitea-deploy-runners",
"com.docker.compose.service": "deploy-control",
"com.docker.compose.oneoff": "False",
},
},
"Mounts": [
{
"Type": "bind",
"Source": module.CONTROLLER_DIR.as_posix(),
"Destination": module.EXECUTABLE_TARGET,
"RW": False,
},
{
"Type": "bind",
"Source": module._expected_config_path().as_posix(),
"Destination": module.CONFIG_TARGET,
"RW": False,
},
{
"Type": "volume",
"Name": module.VOLUME,
"Source": module.MOUNTPOINT.as_posix(),
"Destination": module.TARGET,
"RW": True,
},
],
}
_DEFAULT_SOURCE_IMAGE_ID = object()
def _install_tag_drift_docker_fixture(
monkeypatch, *, source_image_id=_DEFAULT_SOURCE_IMAGE_ID, source_reference=None,
expected_image_output=IMAGE_ID, mutate=None
):
source_ref = "docker.gitea.com/runner-images:ubuntu-latest"
if source_reference is not None:
source_ref = source_reference
if source_image_id is _DEFAULT_SOURCE_IMAGE_ID:
source_image_id = "sha256:" + "c" * 64
container = _tag_drift_container()
if mutate is not None:
mutate(container)
compose_model = model()
compose_model["services"][module.SERVICE]["image"] = source_ref
def run(arguments, *, data=None, **_kwargs):
if arguments[:2] == ["docker", "inspect"]:
return json.dumps([container]).encode("utf-8")
if arguments[:3] == ["docker", "image", "inspect"]:
reference = arguments[-1]
if reference == source_ref:
if source_image_id is None:
return b""
return (source_image_id + "\n").encode("ascii")
if reference == IMAGE_ID:
return (expected_image_output + "\n").encode("ascii") if expected_image_output else b""
return b"sha256:" + b"c" * 64 + b"\n"
if arguments[-2:] == ["--format", "json"]:
return json.dumps(compose_model).encode("utf-8")
return b""
monkeypatch.setattr(module, "_run", run)
return container, source_ref, source_image_id
def test_observed_runner_tag_drift_proof_pins_both_rendered_models(monkeypatch):
container, source_ref, source_image_id = _install_tag_drift_docker_fixture(monkeypatch)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
proof = module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
rollback = json.loads(module.rendered_model(
original, REVISION, IMAGE_ID, source_tag_drift_proof=proof
))
forward = json.loads(module.rendered_model(
original, "c" * 40, IMAGE_ID, source_tag_drift_proof=proof
))
assert proof.source_reference == source_ref
assert proof.source_image_id == source_image_id
for rendered in (rollback, forward):
assert rendered["services"][module.SERVICE]["image"] == IMAGE_ID
assert rendered["services"][module.SERVICE]["pull_policy"] == "never"
@pytest.mark.parametrize(
("mutate", "source_image_id", "expected_error"),
[
(lambda item: item["Config"].__setitem__("Image", "wrong:tag"), None, "source_tag_drift_container_invalid"),
(lambda item: item["Config"]["Labels"].__setitem__("com.docker.compose.project", "other"), None, "source_tag_drift_container_invalid"),
(lambda item: item["Config"]["Labels"].__setitem__("com.docker.compose.service", "runner"), None, "source_tag_drift_container_invalid"),
(lambda item: item["Config"]["Labels"].__setitem__("com.docker.compose.oneoff", "True"), None, "source_tag_drift_container_invalid"),
(lambda item: item["State"].__setitem__("Running", False), None, "container_not_running"),
(lambda item: item.__setitem__("Name", "/renamed-controller"), None, "container_image_changed"),
(lambda item: item["Mounts"].__setitem__(0, {"Type": "bind", "Source": "/unsafe", "Destination": module.EXECUTABLE_TARGET, "RW": False}), None, "executable_mount_invalid"),
(lambda item: item["Mounts"][1].__setitem__("RW", True), None, "config_alias_invalid"),
(None, None, "image_reference_invalid"),
(None, IMAGE_ID, "source_tag_drift_not_observed"),
(None, "not-an-image-id", "image_reference_invalid"),
],
)
def test_observed_runner_tag_drift_rejects_unattested_or_non_drift_cases(
monkeypatch, mutate, source_image_id, expected_error
):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(
monkeypatch, source_image_id=source_image_id, mutate=mutate
)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
with pytest.raises(module.VolumeTransitionError, match=expected_error):
module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
def test_observed_runner_tag_drift_rejects_wrong_source_tag(monkeypatch):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(
monkeypatch, source_reference="docker.gitea.com/runner-images@sha256:" + "c" * 64
)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images@sha256:" + b"c" * 64 + b"\n"
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_reference_invalid"):
module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
def test_observed_runner_tag_drift_rejects_missing_local_expected_image(monkeypatch):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(
monkeypatch, expected_image_output=""
)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_expected_image_missing"):
module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
def test_rendered_model_rejects_altered_drift_proof(monkeypatch):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(monkeypatch)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
proof = module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
altered = proof._replace(compose_sha256="0" * 64)
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_proof_invalid"):
module.rendered_model(original, REVISION, IMAGE_ID, source_tag_drift_proof=altered)
@pytest.mark.parametrize(
("mutate", "expected_error"),
[
(lambda item: item.__setitem__("Id", "e" * 64), "container_changed_during_tag_drift_check"),
(lambda item: item.__setitem__("Image", "sha256:" + "e" * 64), "container_image_changed"),
(lambda item: item["Config"].__setitem__("Image", "other:tag"), "source_tag_drift_container_invalid"),
(lambda item: item["Config"]["Labels"].__setitem__("com.docker.compose.project", "other"), "source_tag_drift_container_invalid"),
(lambda item: item["Mounts"][1].__setitem__("RW", True), "config_alias_invalid"),
(lambda item: item["Mounts"].append({"Type": "bind", "Source": "/other", "Destination": "/extra", "RW": False}), "source_tag_drift_proof_invalid"),
(lambda item: item["Config"].__setitem__("User", "unexpected"), "source_tag_drift_proof_invalid"),
],
)
def test_rendered_model_revalidates_live_container_and_mount_proof(
monkeypatch, mutate, expected_error
):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(monkeypatch)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
proof = module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
mutate(container)
with pytest.raises(module.VolumeTransitionError, match=expected_error):
module.rendered_model(original, REVISION, IMAGE_ID, source_tag_drift_proof=proof)
def test_rendered_model_rejects_tag_that_resolves_back_to_expected_image(monkeypatch):
container, source_ref, _observed_id = _install_tag_drift_docker_fixture(monkeypatch)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
proof = module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
original_run = module._run
def moved_tag(arguments, **kwargs):
if arguments[:3] == ["docker", "image", "inspect"] and arguments[-1] == source_ref:
return (IMAGE_ID + "\n").encode("ascii")
return original_run(arguments, **kwargs)
monkeypatch.setattr(module, "_run", moved_tag)
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_changed"):
module.rendered_model(original, REVISION, IMAGE_ID, source_tag_drift_proof=proof)
def test_rendered_model_rejects_missing_expected_image_after_proof(monkeypatch):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(monkeypatch)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
proof = module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
original_run = module._run
def missing_expected(arguments, **kwargs):
if arguments[:3] == ["docker", "image", "inspect"] and arguments[-1] == IMAGE_ID:
return b""
return original_run(arguments, **kwargs)
monkeypatch.setattr(module, "_run", missing_expected)
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_expected_image_missing"):
module.rendered_model(original, REVISION, IMAGE_ID, source_tag_drift_proof=proof)
def test_rendered_model_rejects_reuse_with_changed_captured_compose_bytes(monkeypatch):
container, _source_ref, _observed_id = _install_tag_drift_docker_fixture(monkeypatch)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
proof = module._attest_observed_runner_tag_drift(original, container, IMAGE_ID)
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_proof_invalid"):
module.rendered_model(original + b"# changed captured source\n", REVISION,
IMAGE_ID, source_tag_drift_proof=proof)
def test_observed_runner_tag_drift_rechecks_same_container_around_image_inspection(monkeypatch):
source_ref = "docker.gitea.com/runner-images:ubuntu-latest"
first = _tag_drift_container()
second = _tag_drift_container()
second["Id"] = "e" * 64
compose_model = model()
compose_model["services"][module.SERVICE]["image"] = source_ref
inspect_count = 0
def run(arguments, *, data=None, **_kwargs):
nonlocal inspect_count
if arguments[:2] == ["docker", "inspect"]:
inspect_count += 1
return json.dumps([first if inspect_count == 1 else second]).encode("utf-8")
if arguments[:3] == ["docker", "image", "inspect"]:
reference = arguments[-1]
return (("sha256:" + "c" * 64) if reference == source_ref else IMAGE_ID).encode("ascii")
if arguments[-2:] == ["--format", "json"]:
return json.dumps(compose_model).encode("utf-8")
return b""
monkeypatch.setattr(module, "_run", run)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
with pytest.raises(module.VolumeTransitionError, match="container_changed_during_tag_drift_check"):
module._attest_observed_runner_tag_drift(original, first, IMAGE_ID)
def test_observed_runner_tag_drift_rejects_mount_change_during_tag_inspection(monkeypatch):
source_ref = "docker.gitea.com/runner-images:ubuntu-latest"
first = _tag_drift_container()
second = _tag_drift_container()
second["Mounts"][1]["RW"] = True
compose_model = model()
compose_model["services"][module.SERVICE]["image"] = source_ref
inspect_count = 0
def run(arguments, *, data=None, **_kwargs):
nonlocal inspect_count
if arguments[:2] == ["docker", "inspect"]:
inspect_count += 1
return json.dumps([first if inspect_count == 1 else second]).encode("utf-8")
if arguments[:3] == ["docker", "image", "inspect"]:
reference = arguments[-1]
return (("sha256:" + "c" * 64) if reference == source_ref else IMAGE_ID).encode("ascii")
if arguments[-2:] == ["--format", "json"]:
return json.dumps(compose_model).encode("utf-8")
return b""
monkeypatch.setattr(module, "_run", run)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
with pytest.raises(module.VolumeTransitionError, match="config_alias_invalid"):
module._attest_observed_runner_tag_drift(original, first, IMAGE_ID)
def test_observed_runner_tag_drift_rejects_config_change_during_tag_inspection(monkeypatch):
source_ref = "docker.gitea.com/runner-images:ubuntu-latest"
first = _tag_drift_container()
second = _tag_drift_container()
second["Config"]["User"] = "unexpected"
compose_model = model()
compose_model["services"][module.SERVICE]["image"] = source_ref
inspect_count = 0
def run(arguments, *, data=None, **_kwargs):
nonlocal inspect_count
if arguments[:2] == ["docker", "inspect"]:
inspect_count += 1
return json.dumps([first if inspect_count == 1 else second]).encode("utf-8")
if arguments[:3] == ["docker", "image", "inspect"]:
reference = arguments[-1]
return (("sha256:" + "c" * 64) if reference == source_ref else IMAGE_ID).encode("ascii")
if arguments[-2:] == ["--format", "json"]:
return json.dumps(compose_model).encode("utf-8")
return b""
monkeypatch.setattr(module, "_run", run)
original = b"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: docker.gitea.com/runner-images:ubuntu-latest\n"
with pytest.raises(module.VolumeTransitionError, match="container_changed_during_tag_drift_check"):
module._attest_observed_runner_tag_drift(original, first, IMAGE_ID)
def test_cli_explicit_tag_drift_switch_is_accepted_only_for_prepare(monkeypatch, tmp_path):
calls = []
monkeypatch.setattr(sys, "argv", [
"prepare_controller_hub_release_volume.py", "prepare",
"--stage-root", str(tmp_path), "--source-revision", REVISION,
"--source-tree", REVISION, "--controller-version", "2.1.13",
"--image-id", IMAGE_ID, "--allow-observed-runner-tag-drift",
])
monkeypatch.setattr(
module, "prepare_promotion",
lambda *args, **kwargs: calls.append(kwargs) or tmp_path / "backup",
)
module.main()
assert len(calls) == 1
assert {key: value for key, value in calls[0].items() if key != "on_phase"} == {
"source_revision": REVISION,
"source_tree": REVISION,
"controller_version": "2.1.13",
"image_id": IMAGE_ID,
"allow_observed_runner_tag_drift": True,
}
assert callable(calls[0]["on_phase"])
def test_cli_rejects_tag_drift_switch_for_non_prepare_operation(monkeypatch):
monkeypatch.setattr(sys, "argv", [
"prepare_controller_hub_release_volume.py", "activate",
"--backup", "/unused", "--allow-observed-runner-tag-drift",
])
with pytest.raises(SystemExit, match="source_tag_recovery_operation_invalid"):
module.main()
@pytest.mark.parametrize("allow_recovery", [False, True])
def test_promotion_image_proof_failures_precede_persistent_volume_writes(
monkeypatch, tmp_path, allow_recovery
):
original = b"captured source compose with exact original bytes\n"
current = _tag_drift_container()
old_revision = "c" * 40
config_tree = ({"projects.json": (b"{}", 0o600)}, {"": 0o755})
compose_model = model()
compose_model["services"][module.SERVICE]["image"] = "docker.gitea.com/runner-images:ubuntu-latest"
proof = module._ObservedRunnerTagDriftProof(
current["Id"], IMAGE_ID, module.OBSERVED_RUNNER_IMAGE_REF,
"sha256:" + "c" * 64, module._hash(original),
module._tag_drift_config_identity(current), (),
)
stage_validation = []
writes = []
monkeypatch.setattr(module, "_validate_private_stage", lambda *_args: None)
monkeypatch.setattr(module, "ensure_volume", lambda **_kwargs: None)
monkeypatch.setattr(module, "ensure_backup_volume", lambda **_kwargs: None)
monkeypatch.setattr(module, "_controller_image_inspect", lambda *_args, **_kwargs: current)
monkeypatch.setattr(module, "_snapshot_attested_controller", lambda *_args: (
{"controller-release.json": (b"captured release", 0o644)},
{"": 0o755},
{"sourceRevision": old_revision, "sourceTree": "d" * 40},
))
monkeypatch.setattr(module, "_read_root_compose", lambda: (original, 0o600))
monkeypatch.setattr(module, "_attest_observed_runner_tag_drift", lambda *_args: proof)
monkeypatch.setattr(module, "_validate_previous_private_tls", lambda *_args: None)
monkeypatch.setattr(module, "_release_tree", lambda *_args: ({"deploy.py": (b"old", 0o644)}, {"": 0o755}))
monkeypatch.setattr(module, "_config_tree", lambda *_args, **_kwargs: config_tree)
monkeypatch.setattr(module, "_read_config_leaf", lambda: None)
monkeypatch.setattr(module, "_prepare_merged_tree", lambda *_args: ({"deploy.py": (b"new", 0o644)}, {"": 0o755}))
monkeypatch.setattr(module, "_validate_staged_configuration", lambda *_args: stage_validation.append(True))
monkeypatch.setattr(module, "_mirror_snapshot_tar", lambda: b"mirror")
monkeypatch.setattr(module, "_source_compose_model", lambda _original: (
compose_model, compose_model["services"][module.SERVICE]
))
if allow_recovery:
def invalidate_after_stage(*_args):
assert stage_validation
raise module.VolumeTransitionError("controller_volume_source_tag_drift_changed")
monkeypatch.setattr(module, "_verify_tag_drift_proof", invalidate_after_stage)
else:
monkeypatch.setattr(
module, "_verify_source_compose_image",
lambda *_args: (_ for _ in ()).throw(
module.VolumeTransitionError("controller_volume_image_reference_changed")
),
)
monkeypatch.setattr(module, "_verify_compose_image", lambda *_args: None)
monkeypatch.setattr(module, "_install_config_tree", lambda *_args: writes.append("config"))
monkeypatch.setattr(module, "_install_release_leaf", lambda *_args: writes.append("release"))
with pytest.raises(module.VolumeTransitionError):
module.prepare_promotion(
tmp_path, source_revision=REVISION, source_tree="e" * 40,
controller_version="2.1.13", image_id=IMAGE_ID,
allow_observed_runner_tag_drift=allow_recovery,
)
assert stage_validation
assert writes == []
def test_promotion_revalidates_drift_proof_immediately_before_first_volume_write(
monkeypatch, tmp_path
):
original = b"captured source compose with exact original bytes\n"
current = _tag_drift_container()
old_revision = "c" * 40
config_tree = ({"projects.json": (b"{}", 0o600)}, {"": 0o755})
stage_validation = []
proof = module._ObservedRunnerTagDriftProof(
current["Id"], IMAGE_ID, module.OBSERVED_RUNNER_IMAGE_REF,
"sha256:" + "c" * 64, module._hash(original),
module._tag_drift_config_identity(current), (),
)
writes = []
proof_checks = 0
image_checks = 0
def source_model(_original):
compose_model = model()
compose_model["services"][module.SERVICE]["image"] = module.OBSERVED_RUNNER_IMAGE_REF
return compose_model, compose_model["services"][module.SERVICE]
monkeypatch.setattr(module, "_validate_private_stage", lambda *_args: None)
monkeypatch.setattr(module, "ensure_volume", lambda **_kwargs: None)
monkeypatch.setattr(module, "ensure_backup_volume", lambda **_kwargs: None)
monkeypatch.setattr(module, "_controller_image_inspect", lambda *_args, **_kwargs: current)
monkeypatch.setattr(module, "_snapshot_attested_controller", lambda *_args: (
{"controller-release.json": (b"captured release", 0o644)},
{"": 0o755},
{"sourceRevision": old_revision, "sourceTree": "d" * 40},
))
monkeypatch.setattr(module, "_read_root_compose", lambda: (original, 0o600))
monkeypatch.setattr(module, "_attest_observed_runner_tag_drift", lambda *_args: proof)
monkeypatch.setattr(module, "_validate_previous_private_tls", lambda *_args: None)
monkeypatch.setattr(module, "_release_tree", lambda *_args: ({"deploy.py": (b"old", 0o644)}, {"": 0o755}))
monkeypatch.setattr(module, "_config_tree", lambda *_args, **_kwargs: config_tree)
monkeypatch.setattr(module, "_read_config_leaf", lambda: config_tree)
monkeypatch.setattr(module, "_prepare_merged_tree", lambda *_args: ({"deploy.py": (b"new", 0o644)}, {"": 0o755}))
monkeypatch.setattr(module, "_validate_staged_configuration", lambda *_args: stage_validation.append(True))
monkeypatch.setattr(module, "_mirror_snapshot_tar", lambda: b"mirror")
monkeypatch.setattr(module, "_source_compose_model", source_model)
monkeypatch.setattr(module, "_compose", lambda *_args, **_kwargs: b"")
def verify_tag_proof(*_args):
nonlocal proof_checks
proof_checks += 1
assert stage_validation
if proof_checks == 4:
raise module.VolumeTransitionError("controller_volume_source_tag_drift_changed")
def verify_image(*_args):
nonlocal image_checks
image_checks += 1
monkeypatch.setattr(module, "_verify_tag_drift_proof", verify_tag_proof)
monkeypatch.setattr(module, "_verify_compose_image", verify_image)
monkeypatch.setattr(module, "_install_config_tree", lambda *_args: writes.append("config"))
monkeypatch.setattr(module, "_install_release_leaf", lambda *_args: writes.append("release"))
with pytest.raises(module.VolumeTransitionError, match="source_tag_drift_changed"):
module.prepare_promotion(
tmp_path, source_revision=REVISION, source_tree="e" * 40,
controller_version="2.1.13", image_id=IMAGE_ID,
allow_observed_runner_tag_drift=True,
)
assert proof_checks == 4
assert image_checks == 2
assert writes == []
def test_config_snapshot_has_exact_allowlist_and_broker_token(): def test_config_snapshot_has_exact_allowlist_and_broker_token():
entries = { entries = {
"projects.json": (b"{}", 0o600), "projects.json": (b"{}", 0o600),
@@ -81,6 +81,25 @@ def test_windows_wrapper_supplies_private_archive_and_exact_image_identity():
assert '[[ "$archive_parent" =~ ^/tmp/itworx-controller-artifact-promoter-' in installer assert '[[ "$archive_parent" =~ ^/tmp/itworx-controller-artifact-promoter-' in installer
def test_observed_runner_tag_recovery_switch_is_fixed_and_prepare_only():
wrapper = (ROOT / "scripts/upgrade_deployment_controller.ps1").read_text(encoding="utf-8")
installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8")
preparer = (ROOT / "scripts/prepare_controller_hub_release_volume.py").read_text(encoding="utf-8")
assert "[switch]$AllowObservedRunnerTagDrift" in wrapper
assert '$recoveryArgument = if ($AllowObservedRunnerTagDrift)' in wrapper
assert "--allow-observed-runner-tag-drift" in wrapper
assert "OBSERVED_RUNNER_IMAGE_REF = 'docker.gitea.com/runner-images:ubuntu-latest'" in preparer
assert "--allow-observed-runner-tag-drift)" in installer
assert "recovery_arguments+=(--allow-observed-runner-tag-drift)" in installer
prepare_start = installer.index('backup="$(python3 "${work}/${PREPARER_NAME}" prepare')
prepare_stop = installer.index('python3 "${work}/${PREPARER_NAME}" stop --backup "$backup"')
prepare_command = installer[prepare_start:prepare_stop]
assert '"${recovery_arguments[@]}"' in prepare_command
assert "parser.add_argument('--allow-observed-runner-tag-drift', action='store_true')" in preparer
assert "if args.allow_observed_runner_tag_drift and args.operation != 'prepare':" in preparer
def test_state_reconciliation_is_committed_only_after_volume_verification(): def test_state_reconciliation_is_committed_only_after_volume_verification():
installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8") installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8")
verify = installer.index('python3 "${work}/${PREPARER_NAME}" verify --backup "$backup"') verify = installer.index('python3 "${work}/${PREPARER_NAME}" verify --backup "$backup"')
+511
View File
@@ -2445,6 +2445,30 @@ def _workmesh_product_project(tmp_path: Path) -> dict[str, object]:
} }
def _agenttestlab_project(tmp_path: Path, containers: list[str] | None = None) -> dict[str, object]:
return {
"slug": "agenttestlab",
"branch": "main",
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/agenttestlab",
"compose_project": "agenttestlab",
"compose_dir": "",
"compose_files": ["compose.inspect.yaml"],
"containers": list(containers or [
"agenttestlab-postgres-1",
"agenttestlab-redis-1",
"agenttestlab-api-1",
"agenttestlab-worker-1",
"agenttestlab-agenttestlab-1",
]),
"exact_source": True,
"env_source": "/srv/example/appdata/agenttestlab/deploy.env",
"health_timeout": 300,
"verify_oci_revision": False,
"_repository_name": "jens/agenttestlab",
"_repository_path": str(tmp_path),
}
def _workmesh_legacy_reattest_setup(tmp_path: Path): def _workmesh_legacy_reattest_setup(tmp_path: Path):
revision = "a" * 40 revision = "a" * 40
project = _workmesh_product_project(tmp_path) project = _workmesh_product_project(tmp_path)
@@ -3100,6 +3124,17 @@ def test_workmesh_topology_transition_requires_exact_registry_and_recorded_recei
assert deploy_module.workmesh_rollback_topology(wrong, previous) is wrong assert deploy_module.workmesh_rollback_topology(wrong, previous) is wrong
def test_snapshot_topology_preserves_workmesh_legacy_container_name_contract(
deploy_module, tmp_path
):
project = _workmesh_product_project(tmp_path)
snapshot = {"containers": [{"name": name} for name in deploy_module.WORKMESH_LEGACY_CONTAINERS]}
restored = deploy_module.snapshot_rollback_topology(project, snapshot)
assert restored["containers"] == deploy_module.WORKMESH_LEGACY_CONTAINERS
def test_workmesh_product_deploy_removes_only_reviewed_compose_orphan( def test_workmesh_product_deploy_removes_only_reviewed_compose_orphan(
deploy_module, tmp_path, monkeypatch deploy_module, tmp_path, monkeypatch
): ):
@@ -3272,6 +3307,482 @@ def test_workmesh_product_snapshot_retains_authorized_five_container_rollback(
assert deploy_module.durable_rollback_snapshot(project, "jens/workmesh", new_revision) == snapshot assert deploy_module.durable_rollback_snapshot(project, "jens/workmesh", new_revision) == snapshot
def test_agenttestlab_deploy_accepts_only_the_reviewed_five_or_six_container_topology(
deploy_module, tmp_path, monkeypatch
):
project = _agenttestlab_project(tmp_path, deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS)
commands = []
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {})
monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "prepare_compose_migration", lambda *_: [])
monkeypatch.setattr(deploy_module, "run", lambda command, **_: commands.append(command))
observed = [
set(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS),
set(deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS),
set(deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS) | {"agenttestlab-unreviewed-1"},
]
monkeypatch.setattr(
deploy_module, "compose_project_container_names", lambda *_: observed.pop(0)
)
deploy_module.deploy_project(project, "a" * 40, "jens/agenttestlab")
deploy_module.deploy_project(project, "b" * 40, "jens/agenttestlab")
with pytest.raises(deploy_module.DeployError, match="topology changed"):
deploy_module.deploy_project(project, "c" * 40, "jens/agenttestlab")
assert commands == [
["docker", "compose", "config", "--quiet"],
["docker", "compose", "build"],
["docker", "compose", "up", "--detach", "--no-build"],
["docker", "compose", "config", "--quiet"],
["docker", "compose", "build"],
["docker", "compose", "up", "--detach", "--no-build"],
["docker", "compose", "config", "--quiet"],
["docker", "compose", "build"],
]
def test_agenttestlab_transition_preserves_registry_and_accepts_verified_post_deploy_retry(
deploy_module, tmp_path, monkeypatch
):
entry = _agenttestlab_project(tmp_path)
entry.pop("_repository_name")
entry.pop("_repository_path")
original = {"jens/agenttestlab": entry, "jens/other": {"slug": "other", "policy": ["unchanged"]}}
original_bytes = json.dumps(original).encode()
registry = tmp_path / "projects.json"
registry.write_bytes(original_bytes)
registry.chmod(0o600)
revision = "b" * 40
protected_head = revision
live_revision = "a" * 40
tree = "c" * 40
state = {
"schema_version": 3,
"repository": "jens/agenttestlab",
"live_revision": live_revision,
"live_status": "healthy",
"live_receipt": {
"build": {"tree": tree},
"containers": {name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS},
},
"live_artifacts": {
"containers": {name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS},
},
}
project = {**entry, "_repository_name": "jens/agenttestlab", "_repository_path": str(tmp_path)}
monkeypatch.setattr(deploy_module, "CONFIG_PATH", registry)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path)
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
monkeypatch.setattr(deploy_module, "_fsync_directory", lambda _path: None)
monkeypatch.setattr(deploy_module, "load_projects", lambda: json.loads(registry.read_text()))
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
def git_or_compose_output(command, **_kwargs):
if command[-2:] == ["rev-parse", "refs/heads/main"]:
return protected_head
return "\n".join(deploy_module.AGENTTESTLAB_SERVICES)
monkeypatch.setattr(deploy_module, "output", git_or_compose_output)
monkeypatch.setattr(deploy_module, "read_state", lambda *_: state)
monkeypatch.setattr(deploy_module, "state_live_artifacts", lambda _state: state["live_artifacts"])
monkeypatch.setattr(
deploy_module, "compose_project_container_names",
lambda *_: set(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS),
)
monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_: True)
monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {})
verified = []
monkeypatch.setattr(
deploy_module,
"verified_current_receipt",
lambda _repo, target, _revision, _tree: verified.append(list(target["containers"])) or {"verified": True},
)
result = deploy_module.transition_agenttestlab_registry(
original, project, tmp_path, revision
)
expected = copy.deepcopy(original)
expected["jens/agenttestlab"]["containers"] = deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS.copy()
assert result == "product-topology-registered"
assert json.loads(registry.read_bytes()) == expected
assert expected["jens/agenttestlab"]["env_source"] == entry["env_source"]
assert expected["jens/other"] == original["jens/other"]
backup = deploy_module._agenttestlab_registry_backup_path()
assert backup.read_bytes() == original_bytes
monkeypatch.setattr(
deploy_module,
"_root_only_backup",
lambda path, payload: None
if path.read_bytes() == payload
else pytest.fail("idempotent transition changed the saved registry backup"),
)
product_project = {**expected["jens/agenttestlab"], "_repository_name": "jens/agenttestlab"}
result = deploy_module.transition_agenttestlab_registry(
expected, product_project, tmp_path, revision
)
assert result == "already-registered"
restored_artifacts = []
monkeypatch.setattr(
deploy_module,
"deployment_artifacts",
lambda target: restored_artifacts.append(list(target["containers"])) or {
"containers": {name: {"image_id": "sha256:" + "1" * 64}
for name in target["containers"]}
},
)
failed_state = deploy_module.resolve_failed_live_state(
state,
{"revision": live_revision, "receipt": state["live_receipt"]},
product_project,
production_started=True,
failure_stage="deployment",
restore_error=None,
)
assert restored_artifacts == [deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS]
assert set(failed_state["live_artifacts"]["containers"]) == set(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS)
assert json.loads(registry.read_bytes()) == expected
later_revision = "d" * 40
protected_head = later_revision
state["live_revision"] = later_revision
state["live_receipt"]["containers"] = {
name: {"image_id": "sha256:" + "2" * 64}
for name in deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS
}
state["live_artifacts"]["containers"] = copy.deepcopy(state["live_receipt"]["containers"])
monkeypatch.setattr(
deploy_module, "compose_project_container_names",
lambda *_: set(deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS),
)
result = deploy_module.transition_agenttestlab_registry(
expected, product_project, tmp_path, later_revision
)
assert result == "already-registered"
assert registry.read_bytes() != original_bytes
assert json.loads(registry.read_bytes()) == expected
assert verified == [
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS,
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS,
deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS,
]
saved = deploy_module.agenttestlab_registry_rollback_backup(expected)
deploy_module.restore_agenttestlab_registry(expected, saved)
assert saved == original_bytes
assert registry.read_bytes() != original_bytes
assert json.loads(registry.read_bytes()) == original
legacy_project = {
**original["jens/agenttestlab"],
"_repository_name": "jens/agenttestlab",
"_repository_path": str(tmp_path),
}
state["live_revision"] = live_revision
state["live_receipt"]["containers"] = {
name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS
}
state["live_artifacts"]["containers"] = copy.deepcopy(state["live_receipt"]["containers"])
monkeypatch.setattr(
deploy_module, "compose_project_container_names",
lambda *_: set(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS),
)
result = deploy_module.transition_agenttestlab_registry(
original, legacy_project, tmp_path, later_revision
)
assert result == "product-topology-registered"
assert json.loads(registry.read_bytes()) == expected
assert backup.read_bytes() == original_bytes
def test_agenttestlab_transition_rejects_wrong_target_source_and_rendered_services(
deploy_module, tmp_path, monkeypatch
):
entry = _agenttestlab_project(tmp_path)
entry.pop("_repository_name")
entry.pop("_repository_path")
projects = {"jens/agenttestlab": entry}
original_bytes = (json.dumps(projects, indent=2) + "\n").encode()
registry = tmp_path / "projects.json"
registry.write_bytes(original_bytes)
project = {**entry, "_repository_name": "jens/agenttestlab"}
revision = "b" * 40
tree = "c" * 40
state = {
"live_revision": "a" * 40,
"live_status": "healthy",
"live_receipt": {
"build": {"tree": tree},
"containers": {name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS},
},
"live_artifacts": {
"containers": {name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS},
},
}
compose_services = [*deploy_module.AGENTTESTLAB_SERVICES]
source_matches = True
monkeypatch.setattr(deploy_module, "CONFIG_PATH", registry)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path)
monkeypatch.setattr(deploy_module, "_fsync_directory", lambda _path: None)
monkeypatch.setattr(deploy_module, "load_projects", lambda: json.loads(registry.read_text()))
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
def output_stub(command, **_kwargs):
if command[-2:] == ["rev-parse", "refs/heads/main"]:
return revision
return "\n".join(compose_services)
monkeypatch.setattr(deploy_module, "output", output_stub)
monkeypatch.setattr(deploy_module, "read_state", lambda *_: state)
monkeypatch.setattr(deploy_module, "state_live_artifacts", lambda _state: state["live_artifacts"])
monkeypatch.setattr(
deploy_module, "compose_project_container_names",
lambda *_: set(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS),
)
monkeypatch.setattr(
deploy_module, "exact_source_matches_revision",
lambda *_: source_matches,
)
monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {})
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: {"verified": True})
with pytest.raises(deploy_module.DeployError, match="fixed main target"):
deploy_module.transition_agenttestlab_registry(
projects,
{**project, "_repository_name": "other/agenttestlab"},
tmp_path,
revision,
)
with pytest.raises(deploy_module.DeployError, match="fixed main target"):
deploy_module.transition_agenttestlab_registry(
projects,
{**project, "source_dir": str(tmp_path)},
tmp_path,
revision,
)
source_matches = False
with pytest.raises(deploy_module.DeployError, match="prepared source"):
deploy_module.transition_agenttestlab_registry(projects, project, tmp_path, revision)
source_matches = True
compose_services.append("unreviewed")
with pytest.raises(deploy_module.DeployError, match="six-service topology"):
deploy_module.transition_agenttestlab_registry(projects, project, tmp_path, revision)
compose_services.pop()
state["live_artifacts"]["containers"][deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS[0]]["image_id"] = "sha256:" + "2" * 64
with pytest.raises(deploy_module.DeployError, match="artifact identity"):
deploy_module.transition_agenttestlab_registry(projects, project, tmp_path, revision)
assert registry.read_bytes() == original_bytes
assert not deploy_module._agenttestlab_registry_backup_path().exists()
def test_agenttestlab_automatic_failure_and_explicit_rollback_keep_distinct_registry_semantics(
deploy_module, tmp_path, monkeypatch
):
legacy = _agenttestlab_project(tmp_path)
product = {**legacy, "containers": deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS.copy()}
previous_receipt = {"containers": {
name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS
}}
assert deploy_module.rollback_topology(product, {"live_receipt": previous_receipt})["containers"] == deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS
assert deploy_module.rollback_topology(
product,
{"live_receipt": {"containers": {
name: {"image_id": "sha256:" + "2" * 64}
for name in deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS
}}},
) is product
observed = []
monkeypatch.setattr(
deploy_module,
"deployment_artifacts",
lambda target: observed.append(list(target["containers"])) or {
"containers": {name: {"image_id": "sha256:" + "1" * 64}
for name in target["containers"]}
},
)
restored = deploy_module.resolve_failed_live_state(
{"live_revision": "a" * 40, "live_status": "healthy", "live_receipt": previous_receipt},
{"revision": "a" * 40, "receipt": previous_receipt},
product,
production_started=True,
failure_stage="deployment",
restore_error=None,
)
assert observed == [deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS]
assert set(restored["live_artifacts"]["containers"]) == set(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS)
@pytest.mark.parametrize("drift", [None, "image", "build", "image_ref"])
def test_agenttestlab_snapshot_restore_allows_only_container_id_recreation(
deploy_module, tmp_path, monkeypatch, drift
):
previous_revision = "a" * 40
failed_revision = "d" * 40
legacy = _agenttestlab_project(tmp_path)
legacy["_repository_name"] = "jens/agenttestlab"
receipt = _receipt(legacy, previous_revision, tree="c" * 40)
project = {**legacy, "containers": deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS.copy()}
project["_repository_path"] = str(tmp_path)
snapshot = {
"revision": previous_revision,
"receipt": receipt,
"containers": [
{
"name": name,
"container_id": receipt["containers"][name]["container_id"],
"image_id": receipt["containers"][name]["image_id"],
"image_ref": receipt["containers"][name]["image_ref"],
"rollback_ref": f"gitea-deploy-rollback/agenttestlab-{index}:{previous_revision[:12]}",
}
for index, name in enumerate(deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS)
],
}
observed_projects = []
observed_receipts = []
monkeypatch.setattr(deploy_module, "run", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "inspect_container", lambda _name: None)
snapshot_images = {item["image_ref"]: item["image_id"] for item in snapshot["containers"]}
monkeypatch.setattr(
deploy_module, "inspect_image", lambda reference: {"Id": snapshot_images[reference]}
)
monkeypatch.setattr(deploy_module, "repository_path_from_project", lambda _project: tmp_path)
monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args, **_kwargs: None)
monkeypatch.setattr(
deploy_module,
"remove_attempt_only_compose_containers",
lambda target, _snapshot: observed_projects.append(list(target["containers"])),
)
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {})
monkeypatch.setattr(
deploy_module,
"restore_snapshot_runtime",
lambda target, *_args, **_kwargs: observed_projects.append(list(target["containers"])),
)
monkeypatch.setattr(
deploy_module,
"validate_recorded_receipt",
lambda target, saved, *_args: observed_projects.append(list(target["containers"]))
or saved["build"],
)
monkeypatch.setattr(deploy_module, "cleanup_rollback_image_tags", lambda *_args: None)
def verify(target, _build, *, expected_image_ids):
observed_projects.append(list(target["containers"]))
observed = copy.deepcopy(receipt)
for item in observed["containers"].values():
item["container_id"] = "3" * 64
if drift == "image":
observed["containers"][deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS[0]]["image_id"] = "sha256:" + "9" * 64
elif drift == "build":
observed["build"]["builtAt"] = "2026-10-02T12:00:00.000Z"
elif drift == "image_ref":
observed["containers"][deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS[0]]["image_ref"] = "unexpected:production"
observed_receipts.append(observed)
return observed
monkeypatch.setattr(deploy_module, "verify_deployment", verify)
if drift is None:
deploy_module.restore_compose_snapshot(project, snapshot, "jens/agenttestlab", failed_revision)
assert project["_rollback_receipt"] == observed_receipts[0]
assert project["_rollback_receipt"]["containers"][
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS[0]
]["container_id"] == "3" * 64
assert observed_projects == [
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS,
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS,
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS,
deploy_module.AGENTTESTLAB_LEGACY_CONTAINERS,
]
else:
with pytest.raises(deploy_module.DeployError, match="Rollback runtime no longer matches"):
deploy_module.restore_compose_snapshot(
project, snapshot, "jens/agenttestlab", failed_revision
)
assert "_rollback_receipt" not in project
def test_agenttestlab_registry_replacement_failure_leaves_original_bytes_intact(
deploy_module, tmp_path, monkeypatch
):
original = {"jens/agenttestlab": _agenttestlab_project(tmp_path)}
original["jens/agenttestlab"].pop("_repository_name")
original["jens/agenttestlab"].pop("_repository_path")
original_bytes = (json.dumps(original, indent=2) + "\n").encode()
registry = tmp_path / "projects.json"
registry.write_bytes(original_bytes)
registry.chmod(0o600)
expected = copy.deepcopy(original)
expected["jens/agenttestlab"]["containers"] = deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS.copy()
replace = os.replace
calls = 0
def fail_registry_replace(source, destination):
nonlocal calls
calls += 1
if calls == 1:
raise OSError("synthetic atomic replacement failure")
replace(source, destination)
monkeypatch.setattr(deploy_module, "CONFIG_PATH", registry)
monkeypatch.setattr(deploy_module, "load_projects", lambda: json.loads(registry.read_text()))
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
monkeypatch.setattr(deploy_module.os, "replace", fail_registry_replace)
with pytest.raises(OSError, match="synthetic atomic replacement failure"):
deploy_module._replace_agenttestlab_registry_atomically(
original, original_bytes, expected
)
assert registry.read_bytes() == original_bytes
assert list(tmp_path.glob(".projects.agenttestlab.*.tmp")) == []
def test_agenttestlab_registry_replacement_rejects_group_readable_registry(
deploy_module, tmp_path, monkeypatch
):
if os.name == "nt":
pytest.skip("POSIX registry permission checks are unavailable on Windows")
original = {"jens/agenttestlab": _agenttestlab_project(tmp_path)}
original["jens/agenttestlab"].pop("_repository_name")
original["jens/agenttestlab"].pop("_repository_path")
original_bytes = (json.dumps(original, indent=2) + "\n").encode()
registry = tmp_path / "projects.json"
registry.write_bytes(original_bytes)
registry.chmod(0o644)
expected = copy.deepcopy(original)
expected["jens/agenttestlab"]["containers"] = deploy_module.AGENTTESTLAB_PRODUCT_CONTAINERS.copy()
monkeypatch.setattr(deploy_module, "CONFIG_PATH", registry)
monkeypatch.setattr(deploy_module, "load_projects", lambda: json.loads(registry.read_text()))
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
with pytest.raises(deploy_module.DeployError, match="private root-owned file"):
deploy_module._replace_agenttestlab_registry_atomically(
original, original_bytes, expected
)
assert registry.read_bytes() == original_bytes
assert registry.stat().st_mode & 0o777 == 0o644
def test_deployment_environment_exports_verified_build_date(deploy_module, tmp_path): def test_deployment_environment_exports_verified_build_date(deploy_module, tmp_path):
project = _project(tmp_path) project = _project(tmp_path)
repository = "jens/unraid-appops-gateway" repository = "jens/unraid-appops-gateway"
+402 -19
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import importlib.util import importlib.util
import hashlib
import json import json
import shlex import shlex
import shutil import shutil
@@ -19,6 +20,7 @@ SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh"
PRIMARY = "itworx-security-runner" PRIMARY = "itworx-security-runner"
SECONDARY = "itworx-validation-runner-2" SECONDARY = "itworx-validation-runner-2"
LEGACY = "itworx-unraid-deploy" LEGACY = "itworx-unraid-deploy"
WINDOWS_NATIVE = "itworx-windows11-native"
OTHER_FIXED_RUNNERS = ( OTHER_FIXED_RUNNERS = (
"itworx-validation-runner-3", "itworx-validation-runner-3",
"itworx-docker-runner", "itworx-docker-runner",
@@ -358,6 +360,280 @@ def _bash_function(name: str) -> str:
return script[start:end] return script[start:end]
def _labels_digest(labels: list[object]) -> str:
canonical = json.dumps(
labels, sort_keys=True, separators=(",", ":"), ensure_ascii=False
).encode("utf-8")
return hashlib.sha256(canonical).hexdigest()
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
("rows", "total_count", "expected_status", "expected_snapshot"),
[
(
[
{
"id": 41,
"name": WINDOWS_NATIVE,
"busy": False,
"disabled": False,
"labels": ["windows-validation"],
},
{
"id": 42,
"name": "itworx-security-runner",
"busy": False,
"disabled": True,
"labels": [],
},
],
2,
0,
f"41:false:{_labels_digest(['windows-validation'])}",
),
(
[
{
"id": 42,
"name": "itworx-security-runner",
"busy": False,
"disabled": True,
"labels": [],
}
],
1,
3,
"",
),
(
[
{
"id": 41,
"name": WINDOWS_NATIVE,
"busy": False,
"disabled": False,
"labels": [],
}
],
2,
1,
"",
),
(
[
{
"id": 41,
"name": WINDOWS_NATIVE,
"busy": False,
"disabled": False,
"labels": [],
},
{
"id": 41,
"name": "other-runner",
"busy": False,
"disabled": False,
"labels": [],
},
],
2,
1,
"",
),
(
[
{
"id": 41,
"name": WINDOWS_NATIVE,
"busy": False,
"disabled": False,
"labels": [],
},
{
"id": 42,
"name": WINDOWS_NATIVE,
"busy": False,
"disabled": True,
"labels": [],
},
],
2,
1,
"",
),
(
[
{
"id": index + 1,
"name": WINDOWS_NATIVE if index == 0 else f"other-{index}",
"busy": False,
"disabled": False,
"labels": [],
}
for index in range(100)
],
100,
1,
"",
),
(
[
{
"id": 41,
"name": WINDOWS_NATIVE,
"busy": "false",
"disabled": False,
"labels": [],
}
],
1,
1,
"",
),
],
ids=(
"known-registration",
"optional-registration-absent",
"incomplete-inventory",
"duplicate-id",
"duplicate-name",
"bounded-inventory-limit",
"malformed-inventory-row",
),
)
def test_windows_snapshot_requires_a_complete_unique_inventory(
tmp_path, rows, total_count, expected_status, expected_snapshot
):
token_file = tmp_path / "token"
token_file.write_text("synthetic-token", encoding="utf-8")
inventory = json.dumps({"total_count": total_count, "runners": rows})
body = "\n".join(
[
f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}",
f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
f"MOCK_INVENTORY={shlex.quote(inventory)}",
'curl() { printf "%s" "$MOCK_INVENTORY"; }',
_bash_function("windows_native_snapshot"),
"windows_native_snapshot",
]
)
result = subprocess.run(
["bash"], input=body.encode(), capture_output=True, check=False
)
assert result.returncode == expected_status, result.stderr.decode(errors="replace")
assert result.stdout.decode(errors="replace").strip() == expected_snapshot
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
def test_windows_snapshot_fails_closed_on_inventory_api_error(tmp_path):
token_file = tmp_path / "token"
token_file.write_text("synthetic-token", encoding="utf-8")
body = "\n".join(
[
f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}",
f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
'curl() { return 22; }',
_bash_function("windows_native_snapshot"),
"windows_native_snapshot",
]
)
result = subprocess.run(
["bash"], input=body.encode(), capture_output=True, check=False
)
assert result.returncode == 1
assert result.stdout == b""
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
("prior_disabled", "expected_patches"),
[("false", ["patch:41:true", "patch:41:false"]), ("true", [])],
ids=("restore-prior-enabled", "preserve-prior-paused"),
)
def test_windows_admission_capture_and_restore_preserve_prior_state(
prior_disabled, expected_patches
):
labels = _labels_digest(["windows-validation"])
body = "\n".join(
[
f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
'windows_native_saved=0; windows_native_id=""; windows_native_prior_disabled=""; windows_native_labels_digest=""',
'mock_id=41; mock_disabled="' + prior_disabled + '"; mock_labels="' + labels + '"',
'windows_native_snapshot() { printf "%s:%s:%s\\n" "$mock_id" "$mock_disabled" "$mock_labels"; }',
'set_runner_disabled_by_id() { printf "patch:%s:%s\\n" "$1" "$2"; mock_disabled="$2"; }',
_bash_function("capture_windows_native_admission"),
_bash_function("restore_windows_native_admission"),
"capture_windows_native_admission",
'[[ "$windows_native_saved" == 1 ]] || exit 91',
'[[ "$mock_disabled" == true ]] || exit 92',
"restore_windows_native_admission",
'printf "final:%s:saved:%s\\n" "$mock_disabled" "$windows_native_saved"',
]
)
result = subprocess.run(
["bash"], input=body.encode(), capture_output=True, check=False
)
assert result.returncode == 0, result.stderr.decode(errors="replace")
output = result.stdout.decode(errors="replace").splitlines()
assert output[:-1] == expected_patches
assert output[-1] == f"final:{prior_disabled}:saved:0"
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
"snapshot",
[
"99:true:{digest}",
"41:true:{other_digest}",
],
ids=("registration-replaced", "labels-drifted"),
)
def test_windows_admission_restore_rejects_identity_or_label_drift(snapshot):
expected_digest = _labels_digest(["windows-validation"])
other_digest = _labels_digest(["changed-label"])
rendered = snapshot.format(digest=expected_digest, other_digest=other_digest)
body = "\n".join(
[
'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false',
f'windows_native_labels_digest="{expected_digest}"',
f'MOCK_SNAPSHOT="{rendered}"',
'windows_native_snapshot() { printf "%s\\n" "$MOCK_SNAPSHOT"; }',
'set_runner_disabled_by_id() { echo unexpected-patch; }',
_bash_function("restore_windows_native_admission"),
'if restore_windows_native_admission; then exit 93; else status=$?; fi',
'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"',
]
)
result = subprocess.run(
["bash"], input=body.encode(), capture_output=True, check=False
)
assert result.returncode == 0, result.stderr.decode(errors="replace")
assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1"
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
def test_windows_admission_restore_failure_keeps_recovery_intent():
labels = _labels_digest(["windows-validation"])
body = "\n".join(
[
'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false',
f'windows_native_labels_digest="{labels}"',
f'mock_snapshot="41:true:{labels}"',
'windows_native_snapshot() { printf "%s\\n" "$mock_snapshot"; }',
'set_runner_disabled_by_id() { return 1; }',
_bash_function("restore_windows_native_admission"),
'if restore_windows_native_admission; then exit 94; else status=$?; fi',
'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"',
]
)
result = subprocess.run(
["bash"], input=body.encode(), capture_output=True, check=False
)
assert result.returncode == 0, result.stderr.decode(errors="replace")
assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1"
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize( @pytest.mark.parametrize(
("proof", "executor_status", "allowed"), ("proof", "executor_status", "allowed"),
@@ -458,20 +734,25 @@ def test_dind_idle_shell_requires_successful_empty_docker_inventory(
assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") def _restore_exit_harness(
@pytest.mark.parametrize( *,
("cutover_started", "dispatch_started"), cutover_started: int,
[(0, 0), (1, 0), (1, 1)], dispatch_started: int,
ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"), exit_status: int,
) fail_first_cp: bool = False,
def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started): fail_windows_patch: bool = False,
):
labels = _labels_digest(["windows-validation"])
body = "\n".join( body = "\n".join(
[ [
"set -Eeuo pipefail",
"backup=/synthetic-backup", "backup=/synthetic-backup",
"COMPOSE_FILE=/synthetic-compose", "COMPOSE_FILE=/synthetic-compose",
"PRIMARY_CONFIG=/synthetic-primary", "PRIMARY_CONFIG=/synthetic-primary",
"SECONDARY_CONFIG=/synthetic-secondary", "SECONDARY_CONFIG=/synthetic-secondary",
"ROOT=/synthetic-root", "ROOT=/synthetic-root",
"RUNNER_GUARD_SCRIPT=/synthetic-runner-guard",
"HOST_GUARD_SCRIPT=/synthetic-host-guard",
"guard_changed=0", "guard_changed=0",
"host_guard_changed=0", "host_guard_changed=0",
f"cutover_started={cutover_started}", f"cutover_started={cutover_started}",
@@ -480,30 +761,132 @@ def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_start
'registered=("new-registration")', 'registered=("new-registration")',
'fenced=("itworx-security-runner:false")', 'fenced=("itworx-security-runner:false")',
"created_paths=()", "created_paths=()",
"windows_native_saved=1",
"windows_native_id=41",
"windows_native_prior_disabled=false",
f'windows_native_labels_digest="{labels}"',
"mock_windows_disabled=true",
f"fail_first_cp={int(fail_first_cp)}",
f"fail_windows_patch={int(fail_windows_patch)}",
"cp_calls=0",
"exec 3>&1", "exec 3>&1",
'cp() { printf "cp:%s\\n" "$*"; }', 'cp() { printf "cp:%s\\n" "$*" >&3; ((cp_calls += 1)); if ((fail_first_cp && cp_calls == 1)); then return 1; fi; }',
'docker() { printf "docker:%s\\n" "$*" >&3; }', 'docker() { printf "docker:%s\\n" "$*" >&3; }',
'delete_runner() { printf "delete:%s\\n" "$*"; }', 'delete_runner() { printf "delete:%s\\n" "$*" >&3; }',
'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }', 'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2" >&3; }',
'stop_capacity_daemon() { printf "stop-daemon\\n"; }', 'set_runner_disabled_by_id() { printf "windows-patch:%s:%s\\n" "$1" "$2" >&3; if ((fail_windows_patch)); then return 1; fi; mock_windows_disabled="$2"; }',
'rm() { printf "rm:%s\\n" "$*"; }', 'windows_native_snapshot() { printf "windows-snapshot\\n" >&3; printf "41:%s:%s\\n" "$mock_windows_disabled" "$windows_native_labels_digest"; }',
'stop_capacity_daemon() { printf "stop-daemon\\n" >&3; }',
'rm() { printf "rm:%s\\n" "$*" >&3; }',
_bash_function("restore_managed_failure"),
_bash_function("restore_windows_native_admission"),
_bash_function("restore"), _bash_function("restore"),
"false", "trap restore EXIT",
"restore", f"exit {exit_status}",
] ]
) )
result = subprocess.run( return subprocess.run(
["bash"], input=body.encode(), capture_output=True, check=False ["bash"], input=body.encode(), capture_output=True, check=False
) )
output = result.stdout.decode(errors="replace")
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
("cutover_started", "dispatch_started"),
[(0, 0), (1, 0), (1, 1)],
ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
)
def test_exit_cleanup_restores_windows_after_managed_recovery(
cutover_started, dispatch_started
):
result = _restore_exit_harness(
cutover_started=cutover_started,
dispatch_started=dispatch_started,
exit_status=1,
)
output = result.stdout.decode(errors="replace").splitlines()
errors = result.stderr.decode(errors="replace") errors = result.stderr.decode(errors="replace")
assert result.returncode == 1, errors assert result.returncode == 1, errors
windows_patch_index = output.index("windows-patch:41:false")
assert not any(
line.startswith("docker:") for line in output[windows_patch_index + 1 :]
)
if dispatch_started: if dispatch_started:
assert output == "" assert output == [
"windows-snapshot",
"windows-patch:41:false",
"windows-snapshot",
]
assert "runner_fleet_recovery_needed" in errors assert "runner_fleet_recovery_needed" in errors
else: else:
assert "cp:" in output assert any(line.startswith("cp:") for line in output)
assert "delete:new-registration" in output assert "delete:new-registration" in output
assert "restore-fence:itworx-security-runner:false" in output assert "restore-fence:itworx-security-runner:false" in output
assert ("stop-daemon" in output) is bool(cutover_started) assert ("stop-daemon" in output) is bool(cutover_started)
assert ("docker:compose " in output) is bool(cutover_started) compose_indexes = [
index
for index, line in enumerate(output)
if line.startswith("docker:compose ")
]
assert bool(compose_indexes) is bool(cutover_started)
if compose_indexes:
assert compose_indexes[-1] < windows_patch_index
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
def test_copy_failure_still_restores_windows_after_managed_cleanup_aborts():
result = _restore_exit_harness(
cutover_started=1,
dispatch_started=0,
exit_status=1,
fail_first_cp=True,
)
output = result.stdout.decode(errors="replace").splitlines()
errors = result.stderr.decode(errors="replace")
assert result.returncode == 75
assert output == [
"cp:-a -- /synthetic-backup/docker-compose.yml /synthetic-compose",
"windows-snapshot",
"windows-patch:41:false",
"windows-snapshot",
]
assert "managed rollback did not complete" in errors
assert not any(line.startswith("docker:") for line in output)
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
def test_exit_cleanup_restores_windows_on_success():
result = _restore_exit_harness(
cutover_started=0,
dispatch_started=1,
exit_status=0,
)
assert result.returncode == 0, result.stderr.decode(errors="replace")
assert result.stdout.decode(errors="replace").splitlines() == [
"windows-snapshot",
"windows-patch:41:false",
"windows-snapshot",
]
script = SCRIPT_PATH.read_text(encoding="utf-8")
dispatch_index = script.index("dispatch_started=1")
restore_index = script.index("if ! restore_windows_native_admission; then", dispatch_index)
ready_index = script.index("printf 'ci_runner_fleet_v2_ready", restore_index)
trap_removed_index = script.index("trap - EXIT", ready_index)
assert restore_index < ready_index < trap_removed_index
assert "docker " not in script[restore_index:trap_removed_index]
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
def test_post_dispatch_windows_restore_failure_marks_recovery_needed():
result = _restore_exit_harness(
cutover_started=1,
dispatch_started=1,
exit_status=1,
fail_windows_patch=True,
)
assert result.returncode == 75
assert (
"runner_fleet_recovery_needed: Windows-native admission state could not be restored"
in result.stderr.decode(errors="replace")
)