240 lines
12 KiB
Python
240 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from test_runner_fleet_specialize import specialize # noqa: F401
|
|
|
|
|
|
@pytest.mark.parametrize("cause", [
|
|
"resource_cgroup_process_mapping_invalid", "resource_cgroup_probe_failed",
|
|
"resource_cgroup_probe_lock_busy", "resource_cgroup_probe_lock_untrusted",
|
|
"resource_cgroup_probe_lock_store_untrusted", "resource_cgroup_probe_lock_unavailable",
|
|
"resource_cgroup_command_failed", "resource_cgroup_command_unavailable",
|
|
"resource_cgroup_probe_io_failed", "resource_cgroup_probe_path_invalid",
|
|
"secret-value", "resource_cgroup_probe_failed\nsecret-value",
|
|
])
|
|
def test_rollback_diagnostic_preserves_only_fixed_codes(specialize, monkeypatch, tmp_path, cause):
|
|
root = tmp_path / "backups"
|
|
root.mkdir(mode=0o700)
|
|
backup = root / "specialized-owned"
|
|
backup.mkdir(mode=0o700)
|
|
monkeypatch.setattr(specialize, "_SPECIALIZED_BACKUP_ROOT", root)
|
|
journal = {"backupDirectory": str(backup), "sourceRevision": "a" * 40,
|
|
"sourceTree": "b" * 40, "stage": "state_prepared"}
|
|
error = specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable")
|
|
error.__cause__ = RuntimeError(cause)
|
|
specialize._record_migration_failure(journal, error)
|
|
raw = (backup / "migration-failure.json").read_bytes()
|
|
record = json.loads(raw)
|
|
assert record == {"schemaVersion": 1, "sourceRevision": "a" * 40, "sourceTree": "b" * 40,
|
|
"stage": "state_prepared", "failureCode": str(error),
|
|
"causeCode": cause if cause in specialize._RESOURCE_FAILURE_CODES else str(error)}
|
|
assert b"secret-value" not in raw
|
|
|
|
|
|
@pytest.mark.parametrize("code", [
|
|
"runner_fleet_specialization_container_health_timeout",
|
|
"runner_fleet_specialization_dind_population_invalid",
|
|
"runner_fleet_specialization_cgroup_family_result_invalid",
|
|
])
|
|
def test_dind_preparation_codes_survive_both_finite_boundaries(specialize, monkeypatch, capsys, code):
|
|
def fail():
|
|
raise specialize.RunnerFleetSpecializeError(code)
|
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
|
captured = capsys.readouterr()
|
|
assert captured.out == ""
|
|
assert captured.err == f"APPOPS_SPECIALIZATION_FAILURE:{code}\n"
|
|
result = SimpleNamespace(returncode=1, stdout="", stderr=captured.err)
|
|
assert specialize._script_failure_code(result) == code
|
|
|
|
|
|
@pytest.mark.parametrize("defect", ["foreign", "symlink", "stage", "revision", "write"])
|
|
def test_rollback_diagnostic_refuses_untrusted_storage_and_never_prevents_rollback(
|
|
specialize, monkeypatch, tmp_path, defect,
|
|
):
|
|
root = tmp_path / "backups"
|
|
root.mkdir(mode=0o700)
|
|
backup = root / "specialized-owned"
|
|
backup.mkdir(mode=0o700)
|
|
monkeypatch.setattr(specialize, "_SPECIALIZED_BACKUP_ROOT", root)
|
|
journal = {"backupDirectory": str(backup), "sourceRevision": "a" * 40,
|
|
"sourceTree": "b" * 40, "stage": "state_prepared"}
|
|
if defect == "foreign":
|
|
journal["backupDirectory"] = str(tmp_path)
|
|
elif defect == "symlink":
|
|
monkeypatch.setattr(specialize, "_trusted_directory", lambda _path: (_ for _ in ()).throw(OSError("private-secret")))
|
|
elif defect == "stage":
|
|
journal["stage"] = "secret-value"
|
|
elif defect == "revision":
|
|
journal["sourceRevision"] = "secret-value"
|
|
else:
|
|
monkeypatch.setattr(specialize, "_write_private_atomic", lambda *_a, **_k: (_ for _ in ()).throw(OSError("private-secret")))
|
|
specialize._record_migration_failure(journal, RuntimeError("secret-value"))
|
|
assert not (backup / "migration-failure.json").exists()
|
|
|
|
|
|
def test_child_fixed_failure_is_finite_and_does_not_echo_exception(specialize, monkeypatch, capsys):
|
|
def fail():
|
|
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_entrypoint_untrusted")
|
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
|
captured = capsys.readouterr()
|
|
assert captured.out == ""
|
|
assert captured.err == "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_entrypoint_untrusted\n"
|
|
|
|
|
|
@pytest.mark.parametrize("private", ["token-secret", "runner_fleet_specialization_arbitrary_secret",
|
|
"runner_fleet_specialization_entrypoint_untrusted\nsecret"])
|
|
def test_unknown_child_details_are_redacted(specialize, monkeypatch, capsys, private):
|
|
def fail():
|
|
raise specialize.RunnerFleetSpecializeError(private)
|
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
|
assert capsys.readouterr().err == "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_script_execution_failed\n"
|
|
assert specialize.public_specialization_failure(RuntimeError(private)) == "runner_fleet_specialization_script_execution_failed"
|
|
|
|
|
|
@pytest.mark.parametrize("fault", [None, "stdout", "multiline", "secret", "code-prefix", "exit", "too-large", "unknown"])
|
|
def test_parent_adopts_only_exact_single_finite_envelope(specialize, fault):
|
|
code = "runner_fleet_specialization_entrypoint_untrusted"
|
|
result = SimpleNamespace(returncode=1, stdout="", stderr=f"APPOPS_SPECIALIZATION_FAILURE:{code}\n")
|
|
if fault == "stdout":
|
|
result.stdout = "sensitive log"
|
|
elif fault == "multiline":
|
|
result.stderr += "secret\n"
|
|
elif fault == "secret":
|
|
result.stderr = "secret-value"
|
|
elif fault == "code-prefix":
|
|
result.stderr = f"{code}\n"
|
|
elif fault == "exit":
|
|
result.returncode = 137
|
|
elif fault == "too-large":
|
|
result.stderr = "x" * 257
|
|
elif fault == "unknown":
|
|
result.stderr = "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_not_reviewed\n"
|
|
assert specialize._script_failure_code(result) == (code if fault is None else "runner_fleet_specialization_script_execution_failed")
|
|
|
|
|
|
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0 or shutil.which("bash") is None,
|
|
reason="real POSIX root Bash parent proof in isolated Linux qualification")
|
|
@pytest.mark.parametrize("deny", [False, True])
|
|
def test_real_bash_parent_identity_and_child_error_propagation_with_stub_only(
|
|
specialize, monkeypatch, tmp_path, deny,
|
|
):
|
|
# Source-identical Bash structure with only its fixed executor redirected to
|
|
# this sandbox's stub. Never call apply, deploy, Docker, native APIs or journals.
|
|
source_root = Path(__file__).resolve().parents[2]
|
|
state = tmp_path / "state"
|
|
state.mkdir(mode=0o700)
|
|
stub = tmp_path / "readonly_stub.py"
|
|
shell = (source_root / "scripts/upgrade_ci_runner_fleet_specialized.sh").read_text()
|
|
shell = shell.replace('/opt/gitea-deploy/runner_fleet_specialize.py', str(stub))
|
|
approved = hashlib.sha256(shell.encode()).hexdigest()
|
|
stub.write_text(
|
|
"import sys\nfrom pathlib import Path\n"
|
|
+ "sys.path.insert(0," + repr(str(source_root / "ops/gitea-deploy-broker")) + ")\n"
|
|
+ "import runner_fleet_specialize as f\n"
|
|
+ "f.SPECIALIZED_STATE_ROOT=Path(" + repr(str(state)) + ")\n"
|
|
+ "f._APPROVED_SCRIPT_SHA256=" + repr("0" * 64 if deny else approved) + "\n"
|
|
+ "if not f._authorized_script_parent():\n"
|
|
+ " print('APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_entrypoint_untrusted',file=sys.stderr)\n"
|
|
+ " sys.exit(1)\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(specialize, "ROOT", tmp_path)
|
|
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_ROOT", state)
|
|
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", state / "specialized.json")
|
|
monkeypatch.setattr(specialize, "_ensure_state_root", lambda: None)
|
|
if deny:
|
|
with pytest.raises(specialize.RunnerFleetSpecializeError,
|
|
match="^runner_fleet_specialization_entrypoint_untrusted$"):
|
|
specialize._run_approved_shell_script(shell.encode())
|
|
else:
|
|
specialize._run_approved_shell_script(shell.encode())
|
|
assert list(state.iterdir()) == []
|
|
|
|
|
|
@pytest.mark.parametrize("controllers,accepted", [
|
|
(["cpu", "memory", "pids"], True),
|
|
(["cpuset", "cpu", "io", "memory", "hugetlb", "pids"], True),
|
|
(["cpuset", "cpu", "io", "memory", "hugetlb", "pids", "foreign"], False),
|
|
])
|
|
@pytest.mark.parametrize("boundary", ["snapshot", "ensure", "journal"])
|
|
def test_observed_hugetlb_inventory_and_legacy_records_remain_finite(
|
|
specialize, monkeypatch, tmp_path, controllers, accepted, boundary,
|
|
):
|
|
if boundary == "snapshot":
|
|
snapshot = {"exists": False, "rootSubtreeControllers": controllers,
|
|
"safeToStartDind": True}
|
|
monkeypatch.setattr(specialize, "inspect_host_cgroup_family", lambda: snapshot)
|
|
read = specialize._cgroup_family_snapshot
|
|
expected = snapshot
|
|
error = "cgroup_inventory_invalid"
|
|
elif boundary == "ensure":
|
|
result = {"created": True, "rootSubtreeBefore": controllers}
|
|
monkeypatch.setattr(specialize, "ensure_host_cgroup_family", lambda **kwargs: result)
|
|
def read():
|
|
return specialize._ensure_cgroup_family_details(allow_existing=False)
|
|
expected = result
|
|
error = "cgroup_family_result_invalid"
|
|
else:
|
|
# The current schema retains absent legacy-DinD provenance and accepts
|
|
# only the finite controller inventories, independently of journal age.
|
|
backup = specialize._SPECIALIZED_BACKUP_ROOT / "specialized-finite-fixture"
|
|
journal = {
|
|
"schemaVersion": 4, "status": "in_progress", "stage": "prepared",
|
|
"sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256,
|
|
"sourceRevision": "a" * 40, "sourceTree": "b" * 40,
|
|
"createdAt": "2026-10-10T00:00:00Z", "updatedAt": "2026-10-10T00:00:00Z",
|
|
"backupDirectory": str(backup), "baseComposeSha256": "c" * 64,
|
|
"baseComposeModelSha256": "d" * 64, "specializedComposeSha256": "e" * 64,
|
|
"legacyTopology": "v1", "originalRegistrations": [], "originalContainers": [],
|
|
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
|
"guardFiles": [], "guardMarkers": {}, "cgroupCreated": False,
|
|
"cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False,
|
|
"cgroupRootSubtree": controllers, "cgroupDindBinding": None,
|
|
"newRegistrationIds": {},
|
|
"imageManifestSha256": None, "buildProbe": None,
|
|
}
|
|
monkeypatch.setattr(specialize, "_read_json_file", lambda *args, **kwargs: journal)
|
|
monkeypatch.setattr(specialize, "_trusted_directory", lambda path: None)
|
|
read = specialize._read_journal
|
|
expected = journal
|
|
error = "journal_invalid"
|
|
if accepted:
|
|
assert read() == expected
|
|
if boundary == "journal":
|
|
journal["schemaVersion"] = 3
|
|
with pytest.raises(specialize.RunnerFleetSpecializeError, match=error):
|
|
read()
|
|
journal["schemaVersion"] = 4
|
|
del journal["cgroupDindBinding"]
|
|
with pytest.raises(specialize.RunnerFleetSpecializeError, match=error):
|
|
read()
|
|
else:
|
|
with pytest.raises(specialize.RunnerFleetSpecializeError, match=error):
|
|
read()
|
|
|
|
|
|
@pytest.mark.parametrize("suffix", ["cgroup_inventory_invalid", "cgroup_inventory_unavailable",
|
|
"identity_adoption_rejected", "legacy_config_missing"])
|
|
def test_new_finite_prejournal_failure_codes_cross_both_boundaries(specialize, monkeypatch, capsys, suffix):
|
|
code = "runner_fleet_specialization_" + suffix
|
|
def fail():
|
|
raise specialize.RunnerFleetSpecializeError(code)
|
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
|
captured = capsys.readouterr()
|
|
assert captured.out == ""
|
|
result = SimpleNamespace(returncode=1, stdout=captured.out, stderr=captured.err)
|
|
assert specialize._script_failure_code(result) == code
|
|
assert specialize.public_specialization_failure(specialize.RunnerFleetSpecializeError(code)) == code
|