Files
Unraid-AppOps-Gateway-Public/backend/tests/test_controller_artifact_bootstrap_failed_child_recovery.py
T

315 lines
10 KiB
Python

from __future__ import annotations
import copy
import subprocess
import pytest
from scripts import bootstrap_controller_artifact_promoter as bootstrap
from backend.tests import test_controller_artifact_bootstrap as original
FAILED_REVISION = "1" * 40
FAILED_TREE = "2" * 40
FAILED_ID = "3" * 64
class Git(original.FakeProtectedGit):
def verify_failed_modern_ancestor(self, revision, current_revision):
if (
revision != FAILED_REVISION
or current_revision != self.revision
or not self.retired_tree_valid
):
raise bootstrap.BootstrapError(
"controller_promoter_failed_helper_not_ancestor"
)
return FAILED_TREE
def read_blob(self, revision, path):
if revision == FAILED_REVISION:
self.blobs_read.append((revision, path))
assert path == bootstrap.MODERN_HELPER_SOURCE_PATH
return self.helper_source.encode()
return super().read_blob(revision, path)
class Docker(original.FakeDocker):
def inspect_container(self, reference):
if reference == FAILED_ID:
return self.existing_child
return super().inspect_container(reference)
def remove_retired_failed_container(self, identifier):
assert identifier == FAILED_ID
self.removed.append(identifier)
self.existing_child = None
def inputs():
source = (original.ROOT / bootstrap.MODERN_HELPER_SOURCE_PATH).read_text()
child = original._retired_failed_child(source)
baseline = bootstrap.MODERN_CONTROLLER_BASELINE
spec = bootstrap.render_modern_helper_spec(
source,
revision=FAILED_REVISION,
image_id=baseline.image_id,
current_controller_id=baseline.container_id,
current_controller_revision=baseline.revision,
current_controller_tree=baseline.tree,
current_controller_version=baseline.version,
)
child["Id"] = FAILED_ID
child["Config"].update(
{
"Hostname": original.HOST_HOSTNAME,
"Cmd": spec["Cmd"],
"Labels": {**original._controller_labels(), **spec["Labels"]},
"Env": ["PATH=/usr/bin"],
"WorkingDir": "/tmp",
"Entrypoint": None,
"Domainname": "",
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
"OnBuild": None,
}
)
child["State"].update({"Paused": False, "Restarting": False, "Dead": False})
child["HostConfig"].update(copy.deepcopy(bootstrap.FAILED_HELPER_HOST_DEFAULTS))
child["HostConfig"]["RestartPolicy"] = {"Name": "no", "MaximumRetryCount": 0}
docker = Docker(original._legacy_helper_spec(), existing_child=child)
docker.containers[bootstrap.CONTROLLER_NAME] = (
original._modern_controller_source_inspect(baseline.revision)
)
docker.images[baseline.image_id]["Config"].update(
{
"Env": ["PATH=/usr/bin"],
"WorkingDir": "/tmp",
"Entrypoint": None,
"Shell": ["/bin/bash", "-c"],
}
)
return docker, Git(source), child
def run(docker, git):
return original._run_bootstrap(
docker,
git,
health_sequence=[
original._modern_baseline_health(),
original._modern_baseline_health(),
original._modern_baseline_health(),
original._modern_baseline_health(),
original._health(),
original._health(),
],
)
def test_noncurrent_protected_ancestor_failure_releases_only_its_verified_name():
docker, git, child = inputs()
result = run(docker, git)
assert result["status"] == "promoted"
assert docker.removed == [FAILED_ID]
assert (
git.blobs_read.count((FAILED_REVISION, bootstrap.MODERN_HELPER_SOURCE_PATH))
== 1
)
assert (
docker.created[0]["Labels"]["itworx.controller.bootstrap.target.id"]
== bootstrap.MODERN_CONTROLLER_ID
)
@pytest.mark.parametrize(
"change",
[
"running",
"succeeded",
"bool_exit",
"paused",
"dead",
"restarting",
"source",
"target",
"command",
"entrypoint",
"environment",
"mount",
"host_capability",
"controller",
"appops",
"ancestor",
],
)
def test_noncurrent_failed_child_recovery_refuses_unattested_identity_or_baseline(
change,
):
docker, git, child = inputs()
if change == "running":
child["State"].update(Running=True, Status="running")
elif change == "succeeded":
child["State"]["ExitCode"] = 0
elif change == "bool_exit":
child["State"]["ExitCode"] = True
elif change in {"paused", "dead", "restarting"}:
child["State"][change.capitalize()] = True
elif change == "source":
child["Config"]["Labels"]["itworx.source.repository"] = "other/repository"
elif change == "target":
child["Config"]["Labels"]["itworx.controller.bootstrap.target.id"] = "9" * 64
elif change == "command":
child["Config"]["Cmd"][2] += "\necho changed"
elif change == "entrypoint":
child["Config"]["Entrypoint"] = ["/bin/sh"]
elif change == "environment":
child["Config"]["Env"].append("CUSTOM_VARIABLE=extra")
elif change == "mount":
child["Mounts"][2]["RW"] = True
elif change == "host_capability":
child["HostConfig"]["CapAdd"] = ["SYS_ADMIN"]
elif change == "controller":
docker.containers[bootstrap.CONTROLLER_NAME]["Id"] = "9" * 64
elif change == "appops":
docker.containers[original.APPOPS_CONTAINER_ID]["Image"] = "sha256:" + "9" * 64
else:
git.retired_tree_valid = False
with pytest.raises(bootstrap.BootstrapError):
run(docker, git)
assert docker.removed == docker.created == docker.started == []
def test_noncurrent_child_identity_drift_on_final_named_reread_refuses_delete():
docker, git, child = inputs()
docker.named_inspect_override = copy.deepcopy(child)
docker.named_inspect_override["Id"] = "9" * 64
docker.named_inspect_override_at = 3
with pytest.raises(bootstrap.BootstrapError):
run(docker, git)
assert not docker.removed
def test_dynamic_failed_child_delete_retains_nonforce_and_no_volume_policy(monkeypatch):
client = bootstrap.DockerSocketClient()
calls = []
monkeypatch.setattr(
client,
"_request",
lambda method, path, **kw: (calls.append((method, path)) or (204, b"")),
)
client.remove_retired_failed_container(FAILED_ID)
assert calls == [("DELETE", f"/containers/{FAILED_ID}?force=false&v=false")]
@pytest.mark.parametrize(
"change",
["cpu", "dns", "masked_paths", "logging", "boolean_default", "unknown_option"],
)
def test_failed_child_requires_complete_known_host_defaults(change):
docker, git, child = inputs()
if change == "cpu":
child["HostConfig"]["CpuShares"] = 100
elif change == "dns":
child["HostConfig"]["Dns"] = ["192.0.2.1"]
elif change == "masked_paths":
child["HostConfig"]["MaskedPaths"] = []
elif change == "logging":
child["HostConfig"]["LogConfig"]["Type"] = "none"
elif change == "boolean_default":
child["HostConfig"]["CpuShares"] = False
else:
child["HostConfig"]["UnreviewedOption"] = None
with pytest.raises(bootstrap.BootstrapError):
run(docker, git)
assert not docker.removed
def test_raw_docker_api_nil_mount_omission_matches_cli_null():
docker, git, child = inputs()
child["HostConfig"].pop("Mounts")
assert run(docker, git)["status"] == "promoted"
def test_appops_drift_on_final_baseline_reread_refuses_cleanup():
docker, git, child = inputs()
docker.appops_inspect_override = copy.deepcopy(
docker.containers[original.APPOPS_CONTAINER_ID]
)
docker.appops_inspect_override["State"]["Running"] = False
docker.appops_inspect_override_at = 3
with pytest.raises(bootstrap.BootstrapError):
run(docker, git)
assert not docker.removed
def test_protected_main_drift_immediately_before_delete_refuses_cleanup():
docker, git, child = inputs()
git.fail_current_main_at = 5
with pytest.raises(bootstrap.BootstrapError):
run(docker, git)
assert not docker.removed
@pytest.mark.parametrize(
"identifier", ["", "short", "A" * 64, "3" * 63, "../containers/other"]
)
def test_delete_adapter_rejects_noncanonical_full_container_identity(
monkeypatch, identifier
):
client = bootstrap.DockerSocketClient()
monkeypatch.setattr(
client,
"_request",
lambda *_args, **_kw: pytest.fail("unexpected Docker mutation"),
)
with pytest.raises(bootstrap.BootstrapError):
client.remove_retired_failed_container(identifier)
@pytest.mark.parametrize("scenario", ["ancestor", "unrelated", "replace_ref"])
def test_git_ancestor_proof_reads_original_protected_objects(
tmp_path, monkeypatch, scenario
):
def git(*args):
return (
subprocess.run(
["git", "-C", str(tmp_path), *args],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
.stdout.decode()
.strip()
)
git("init", "-q", "-b", "main")
git("config", "user.name", "Test")
git("config", "user.email", "test@example.invalid")
path = tmp_path / "tracked"
path.write_text("protected ancestor")
git("add", ".")
git("commit", "-qm", "ancestor")
old = git("rev-parse", "HEAD")
tree = git("rev-parse", "HEAD^{tree}")
path.write_text("new protected main")
git("commit", "-qam", "main")
current = git("rev-parse", "HEAD")
monkeypatch.setattr(bootstrap, "PROTECTED_REPO", str(tmp_path / ".git"))
proof = bootstrap.ProtectedGitRepository()
if scenario == "replace_ref":
git("replace", old, current)
assert git("rev-parse", f"{old}^{{tree}}") != tree
if scenario == "unrelated":
git("checkout", "-qb", "side", old)
path.write_text("unrelated child")
git("commit", "-qam", "side")
old = git("rev-parse", "HEAD")
with pytest.raises(bootstrap.BootstrapError):
proof.verify_failed_modern_ancestor(old, current)
else:
assert proof.verify_failed_modern_ancestor(old, current) == tree