Publish curated Unraid-AppOps-Gateway source from 87aa7dda6889fd951d202754d76daaaa1e98e26b

This commit is contained in:
Public Source Publisher committed 2026-10-10 14:24:16 +00:00
1 parent 07bb0bbb90
commit ad3a9736cc
5 files changed
+891 -6

No files matched your search

+354 -2
View File
@@ -1181,6 +1181,66 @@ def test_specialized_compose_validation_accepts_real_compose_normalization(
assert specialize._validate_composition(b"normalized candidate")
def test_specialized_dind_healthcheck_replaces_captured_tcp_endpoint(specialize, monkeypatch):
base = _base_v1_compose_model(specialize)
base["services"]["runner-dind"]["healthcheck"] = {
"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"],
"interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s",
}
rendered = json.loads(specialize._render_specialized_compose(base))
check = rendered["services"]["runner-dind"]["healthcheck"]
assert check == {
"test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"],
"interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s",
}
assert base["services"]["runner-dind"]["healthcheck"]["test"][1].startswith("DOCKER_HOST=tcp:")
monkeypatch.setattr(specialize, "_compose_model", lambda _raw: rendered)
assert specialize._validate_composition(b"reviewed model")
@pytest.mark.parametrize("check", [None, {"disable": True}, {"test": ["NONE"]},
{"disable": True, "test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"]},
{"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]},
{"test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]},
{"test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]},
])
def test_specialized_compose_rejects_healthcheck_endpoint_drift(specialize, monkeypatch, check):
model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize)))
model["services"]["runner-dind"]["healthcheck"] = check
monkeypatch.setattr(specialize, "_compose_model", lambda _raw: model)
with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_compose_invalid"):
specialize._validate_composition(b"foreign healthcheck")
@pytest.mark.parametrize("check", [None, {"Test": ["NONE"]},
{"Test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]},
{"Test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]},
{"Test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]},
])
def test_specialized_dind_runtime_rejects_healthcheck_drift(specialize, check):
base = _base_v1_compose_model(specialize)
model = json.loads(specialize._render_specialized_compose(base))
service = model["services"]["runner-dind"]
item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID,
running=True, restart_policy="no")
item["Config"].update(User="0", Cmd=service["command"],
Env=[f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"],
Healthcheck={"Test": service["healthcheck"]["test"]})
item["HostConfig"].update(CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={})
item["Mounts"] = [{"Type": "bind", "Source": mount["source"],
"Destination": mount["target"], "RW": not mount["read_only"]}
for mount in service["volumes"]]
assert specialize._validate_dind_outer(item, model)
changed = copy.deepcopy(item)
changed["Config"]["Healthcheck"] = check
assert specialize._validate_dind_outer(changed, model) is False
changed_model = copy.deepcopy(model)
changed_model["services"]["runner-dind"]["healthcheck"] = {
"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"],
}
assert specialize._validate_dind_outer(item, changed_model) is False
def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize):
base = _base_v1_compose_model(specialize)
item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID,
@@ -1208,6 +1268,7 @@ def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces(
image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no")
item["Config"].update(User="0", Cmd=service["command"])
item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"]
item["Config"]["Healthcheck"] = {"Test": service["healthcheck"]["test"]}
item["HostConfig"].update(
CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={}
)
@@ -1358,6 +1419,123 @@ def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(speciali
specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow")
def _resource_guard_source(kind):
prefix = '''#!/bin/bash
set -eu
RUNNER="legacy-primary"
SECONDARY_RUNNER="legacy-secondary"
RUNNER_DIND="private-dind"
DEPLOY_RUNNER="deploy-runner"
RUNNER_PAUSE_FILE="/nonexistent-test-pause"
RUNNER_MARKER="/nonexistent-test-marker"
controller() { printf "controller:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = --state ]; then printf "%s" "$GUARD_STATE"; fi; }
timeout() { shift; "$@"; }
docker() { printf "docker:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = inspect ]; then printf legacy; fi; }
enforce_limits() { printf "enforce:%s\\n" "$*" >> "$EVENTS"; }
'''
if kind == "runner":
return prefix + '''# The sandbox is the aggregate resource envelope for all jobs and services.
if [ -n "$(timeout 10 docker inspect -f '{{.Id}}' "$RUNNER_DIND" 2>/dev/null || true)" ]; then
dind_limits="$(timeout 10 docker inspect -f '{{.HostConfig.NanoCpus}}' "$RUNNER_DIND")"
if [ "$dind_limits" != "expected" ]; then
timeout 20 docker update --restart unless-stopped "$RUNNER_DIND"
fi
if [ "$dind_limits" = "legacy" ]; then
timeout 20 docker update --cpuset-cpus 0-5 "$RUNNER_DIND"
fi
fi
enforce_limits "$DEPLOY_RUNNER"
stack_recovery=0
:
unexpected=0
for name in "$DEPLOY_RUNNER"; do
[ "$name" = "$DEPLOY_RUNNER" ] && continue
done
'''
return prefix + '''# ITWORX_RUNNER_CAPACITY_V2_BEGIN
:
# ITWORX_RUNNER_CAPACITY_V2_END
if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then
enforce_limits "$RUNNER" no
enforce_limits "$SECONDARY_RUNNER" no
enforce_limits "$RUNNER_DIND" no
else
enforce_limits "$RUNNER" unless-stopped
enforce_limits "$SECONDARY_RUNNER" unless-stopped
enforce_limits "$RUNNER_DIND" unless-stopped
fi
# The deploy runner is deliberately tiny and is required for deployment and
enforce_limits "$DEPLOY_RUNNER"
enforce_limits tdarr
enforce_limits controller
enforce_limits containment-pressure
runner_healthy=1
if [ "$runner_healthy" -eq 0 ]; then
if [ "$sandbox_healthy" -eq 1 ] && \\
timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then
:
fi
fi
'''
@pytest.mark.parametrize("kind", ["runner", "host"])
def test_resource_writer_is_gated_before_any_legacy_mutation(specialize, kind):
source = _resource_guard_source(kind)
if kind == "runner":
patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow")
writer = "# The sandbox is the aggregate resource envelope"
else:
patched = specialize._patch_host_guard(source, "controller --state")
writer = 'if [ -e "$RUNNER_PAUSE_FILE" ]'
assert patched.index('specialization_state="$(controller --state') < patched.index(writer)
assert 'if [ "$specialization_state" = "legacy" ]; then\n ' + writer in patched
assert 'enforce_limits "$DEPLOY_RUNNER"' in patched
@pytest.mark.skipif(os.name != "posix", reason="requires actual Bash stub execution")
@pytest.mark.parametrize("kind", ["runner", "host"])
@pytest.mark.parametrize("state", ["legacy", "blocked", "active"])
def test_guard_bash_skips_only_legacy_resource_writer_in_cutover(specialize, tmp_path, kind, state):
source = _resource_guard_source(kind)
if kind == "runner":
patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow")
else:
patched = specialize._patch_host_guard(source, "controller --state")
events = tmp_path / "events"
result = subprocess.run(["bash", "-c", patched], capture_output=True, check=False, timeout=5,
env={"PATH": "/usr/bin:/bin", "EVENTS": str(events), "GUARD_STATE": state})
assert result.returncode == 0
rows = events.read_text().splitlines()
assert rows[0] == "controller:--state"
legacy_writes = [row for row in rows if row.startswith("docker:update")
or row.startswith("enforce:legacy-") or row.startswith("enforce:private-dind")]
assert bool(legacy_writes) is (state == "legacy")
assert "enforce:deploy-runner" in rows
if kind == "host":
assert {"enforce:tdarr", "enforce:controller", "enforce:containment-pressure"}.issubset(rows)
@pytest.mark.parametrize("kind", ["runner", "host"])
def test_resource_writer_shape_drift_is_rejected(specialize, kind):
source = _resource_guard_source(kind)
anchor = ("# The sandbox is the aggregate resource envelope for all jobs and services.\n"
if kind == "runner" else 'if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then\n')
patch = (lambda text: specialize._patch_runner_guard(text, "controller --state", "controller --allow")) if kind == "runner" else (
lambda text: specialize._patch_host_guard(text, "controller --state"))
with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_shape_invalid"):
patch(source.replace(anchor, anchor + anchor, 1))
def test_historical_orphan_transform_does_not_acquire_new_resource_gate(specialize):
source = _resource_guard_source("runner")
historical = specialize._patch_runner_guard_historical(source, "controller --state", "controller --allow")
patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow")
writer = historical.index("# The sandbox is the aggregate resource envelope")
assert 'specialization_state="$(controller --state' not in historical[:writer]
assert 'specialization_state="$(controller --state' in patched[:patched.index("# The sandbox is the aggregate resource envelope")]
def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch):
script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes()
monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest())
@@ -1621,7 +1799,9 @@ def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, g
monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable)
journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(),
originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None},
originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupRootSubtree=[])
originalContainers=[], originalRegistrations=[], cgroupCreated=True,
cgroupFamilyInitiallyPresent=False, cgroupRootSubtree=[])
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose")
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
@@ -1656,6 +1836,174 @@ def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, g
assert not marker.exists() and not durable.exists()
@pytest.fixture
def rollback_empty_family(specialize, tmp_path, monkeypatch):
compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal"
compose.write_bytes(b"fixed compose")
marker.write_bytes(b"owned marker")
durable.write_bytes(b"owned journal")
for name, value in (("COMPOSE_PATH", compose), ("SPECIALIZED_STATE_PATH", marker),
("MIGRATION_JOURNAL_PATH", durable)):
monkeypatch.setattr(specialize, name, value)
journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(),
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
"originalContainers": [], "originalRegistrations": [], "guardFiles": [],
"cgroupCreated": True, "cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False,
"cgroupRootSubtree": ["cpu", "cpuset", "hugetlb", "io", "memory", "pids"]}
family = {"exists": True, "groupCount": 1, "hasProcesses": False,
"familyCpuMax": "max 100000", "familyMemoryMax": "max",
"familySubtreeControllers": list(journal["cgroupRootSubtree"]),
"rootSubtreeControllers": list(journal["cgroupRootSubtree"])}
rebound = {"created": False, "rootSubtreeBefore": list(journal["cgroupRootSubtree"])}
events = []
@contextmanager
def maintenance():
yield lambda: events.append("barrier")
monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance)
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: None)
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose")
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: [])
monkeypatch.setattr(specialize, "_inspect", lambda _name: None)
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: events.append("snapshot") or family)
def ensure(**kwargs):
assert kwargs == {"allow_existing": True}
events.append("ensure")
return rebound
def rollback(**kwargs):
assert kwargs == {"created": True, "initial_controllers": tuple(journal["cgroupRootSubtree"])}
events.append("rollback")
return {"removed": True, "familyExists": False}
monkeypatch.setattr(specialize, "_ensure_cgroup_family_details", ensure)
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", rollback)
return journal, family, rebound, events, marker, durable
@pytest.mark.parametrize("pending", [False, True])
@pytest.mark.parametrize("cpu", ["max", "max 100000"])
def test_rollback_rebinds_only_owned_empty_default_family_before_strict_removal(
specialize, rollback_empty_family, pending, cpu,
):
journal, family, _rebound, events, marker, durable = rollback_empty_family
journal.update(cgroupCreated=not pending, cgroupEnsurePending=pending)
family["familyCpuMax"] = cpu
original = copy.deepcopy(journal)
assert specialize._rollback_before_activation(journal) is True
assert journal == original
snapshot = events.index("snapshot")
assert events[snapshot:snapshot + 6] == ["snapshot", "barrier", "ensure", "barrier", "barrier", "rollback"]
assert not marker.exists() and not durable.exists()
@pytest.mark.parametrize("absent", [False, True])
def test_rollback_never_ensures_absent_or_already_bounded_family(specialize, rollback_empty_family, absent):
journal, family, _rebound, events, _marker, _durable = rollback_empty_family
if absent:
family.clear()
family["exists"] = False
else:
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
assert specialize._rollback_before_activation(journal) is True
assert "ensure" not in events and "rollback" in events
@pytest.mark.parametrize("mutation", ["initially_present", "initially_unknown", "foreign_cpu", "foreign_memory",
"partial_default", "processes", "unknown_processes", "children", "bool_count", "newly_created", "root_changed"])
def test_empty_family_rebind_refusal_retains_marker_and_journal(specialize, rollback_empty_family, mutation):
journal, family, rebound, events, marker, durable = rollback_empty_family
if mutation == "initially_present":
journal["cgroupFamilyInitiallyPresent"] = True
elif mutation == "initially_unknown":
journal.pop("cgroupFamilyInitiallyPresent")
elif mutation == "foreign_cpu":
family["familyCpuMax"] = "400000 100000"
elif mutation == "foreign_memory":
family["familyMemoryMax"] = str(8 * 1024**3)
elif mutation == "partial_default":
family["familyCpuMax"] = "800000 100000"
elif mutation == "processes":
family["hasProcesses"] = True
elif mutation == "unknown_processes":
family.pop("hasProcesses")
elif mutation == "children":
family["groupCount"] = 2
elif mutation == "bool_count":
family["groupCount"] = True
elif mutation == "newly_created":
rebound["created"] = True
else:
rebound["rootSubtreeBefore"] = ["cpu", "memory", "pids"]
assert specialize._rollback_before_activation(journal) is False
assert "rollback" not in events
assert ("ensure" in events) is (mutation in {"newly_created", "root_changed"})
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
def test_rebind_barrier_loss_prevents_cgroup_removal_and_keeps_durable_state(
specialize, rollback_empty_family, monkeypatch,
):
journal, _family, _rebound, events, marker, durable = rollback_empty_family
@contextmanager
def maintenance():
def barrier():
if events and events[-1] == "ensure":
raise RuntimeError("maintenance identity changed")
events.append("barrier")
yield barrier
monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance)
assert specialize._rollback_before_activation(journal) is False
assert "ensure" in events and "rollback" not in events
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
@pytest.mark.parametrize("bounded", [False, True])
def test_family_rollback_accepts_only_required_controllers_for_minimal_initial_root(
specialize, rollback_empty_family, bounded,
):
journal, family, rebound, events, _marker, _durable = rollback_empty_family
initial = ["cpu", "memory", "pids"]
journal["cgroupRootSubtree"] = list(initial)
family["rootSubtreeControllers"] = list(initial)
family["familySubtreeControllers"] = list(initial)
rebound["rootSubtreeBefore"] = list(initial)
if bounded:
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
assert specialize._rollback_before_activation(journal) is True
assert ("ensure" in events) is not bounded
assert "rollback" in events
@pytest.mark.parametrize("bounded", [False, True])
@pytest.mark.parametrize("mutation", ["missing", "null", "string", "duplicate", "unknown", "nonstring",
"required_missing", "unowned_extra"])
def test_family_subtree_drift_refuses_before_ensure_or_removal(
specialize, rollback_empty_family, bounded, mutation,
):
journal, family, _rebound, events, marker, durable = rollback_empty_family
if bounded:
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
if mutation == "missing":
family.pop("familySubtreeControllers")
elif mutation == "null":
family["familySubtreeControllers"] = None
elif mutation == "string":
family["familySubtreeControllers"] = "cpu memory pids"
elif mutation == "duplicate":
family["familySubtreeControllers"].append("cpu")
elif mutation == "unknown":
family["familySubtreeControllers"].append("foreign-controller")
elif mutation == "nonstring":
family["familySubtreeControllers"].append(["cpu"])
elif mutation == "required_missing":
family["familySubtreeControllers"].remove("pids")
else:
journal["cgroupRootSubtree"] = ["cpu", "memory", "pids"]
assert specialize._rollback_before_activation(journal) is False
assert "ensure" not in events and "rollback" not in events
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch):
marker, journal = tmp_path / "marker", tmp_path / "journal"
marker.write_bytes(b"owned marker")
@@ -1727,7 +2075,9 @@ def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_fini
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup)
journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}],
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
"originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupRootSubtree": []}
"originalContainers": [], "originalRegistrations": [], "cgroupCreated": True,
"cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": []}
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
assert specialize._rollback_before_activation(journal) is True
assert events == ["cleanup", "cgroup", "guards-restored"]
assert not marker.exists() and not durable.exists()
@@ -3059,6 +3409,7 @@ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup(
"guardFiles": [],
"cgroupCreated": True,
"cgroupEnsurePending": False,
"cgroupFamilyInitiallyPresent": False,
"cgroupRootSubtree": ["cpu", "memory", "pids"],
"backupDirectory": str(tmp_path / "specialized-test"),
}
@@ -3118,6 +3469,7 @@ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup(
monkeypatch.setattr(specialize, "_run", run)
monkeypatch.setattr(specialize, "_run_compose", run_compose)
monkeypatch.setattr(specialize, "_runner_rows", lambda: [])
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
monkeypatch.setattr(
specialize,
"rollback_host_cgroup_family",