Publish curated Unraid-AppOps-Gateway source from 5f1a0fc2fdd07082637995b2c8b4514a1ecbbbb8
This commit is contained in:
1 parent
9787d16330
commit
75269a32eb
8 files changed
+920
-8
No files matched your search
@@ -1,3 +1,3 @@
|
|||||||
# Public source export
|
# Public source export
|
||||||
|
|
||||||
This source snapshot was generated from the private canonical repository at revision `0ea6f267c28d51ac7113e6d943c89857d3a189ab`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
This source snapshot was generated from the private canonical repository at revision `5f1a0fc2fdd07082637995b2c8b4514a1ecbbbb8`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"schemaVersion": 1,
|
"schemaVersion": 1,
|
||||||
"sourceRevision": "0ea6f267c28d51ac7113e6d943c89857d3a189ab",
|
"sourceRevision": "5f1a0fc2fdd07082637995b2c8b4514a1ecbbbb8",
|
||||||
"files": [
|
"files": [
|
||||||
{
|
{
|
||||||
"path": ".dockerignore",
|
"path": ".dockerignore",
|
||||||
@@ -937,6 +937,16 @@
|
|||||||
"sha256": "54b24a90e7238aaed22d0dfbae70d9772006d20cdb9dbd43d781af48892dcd94",
|
"sha256": "54b24a90e7238aaed22d0dfbae70d9772006d20cdb9dbd43d781af48892dcd94",
|
||||||
"bytes": 33759
|
"bytes": 33759
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_ci_acceptance_root_recovery.py",
|
||||||
|
"sha256": "48bac4513e61610331195ccfef5aaf418eac8ba4ec3a4169351d293643900322",
|
||||||
|
"bytes": 14349
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_ci_api_term_root_recovery.py",
|
||||||
|
"sha256": "b59c4f65229a1a74850c512fe6144dec3ad9a2e41f39c28a57bab406c023fd9d",
|
||||||
|
"bytes": 12820
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_ci_job_docker.py",
|
"path": "backend/tests/test_ci_job_docker.py",
|
||||||
"sha256": "7292a3ad3e52e934e6a410fdea8adb6a00dc90fe8de28d8159205ec069951d73",
|
"sha256": "7292a3ad3e52e934e6a410fdea8adb6a00dc90fe8de28d8159205ec069951d73",
|
||||||
@@ -1584,8 +1594,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_acceptance_quiescence.py",
|
"path": "backend/tests/test_runner_acceptance_quiescence.py",
|
||||||
"sha256": "62ce270a8b3e9c9a613cd7bb91f0a7d8b6071bccf914def33f5672084b7a3841",
|
"sha256": "659948130bc1a1067ef849e32ba38078b05e407fe1fd80effb203f9af36cadaa",
|
||||||
"bytes": 42074
|
"bytes": 48198
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_acceptance_surface.py",
|
"path": "backend/tests/test_runner_acceptance_surface.py",
|
||||||
@@ -1649,18 +1659,23 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_resource_envelope.py",
|
"path": "backend/tests/test_runner_resource_envelope.py",
|
||||||
"sha256": "0181c36c9560c5833fb653b50d8739677b45df22b856ff514dd9cba69b732a3d",
|
"sha256": "42333a8bc138319e98e9756f2b3909d0c55dac1efa348cd1d6bd7c918098ae73",
|
||||||
"bytes": 51403
|
"bytes": 53395
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_scheduler_liveness_recovery.py",
|
"path": "backend/tests/test_runner_scheduler_liveness_recovery.py",
|
||||||
"sha256": "40e69e3d0161fb0932a35889ccc7c51ab42bc74c1e0b39c0f5c31a7a76bfe1c3",
|
"sha256": "40e69e3d0161fb0932a35889ccc7c51ab42bc74c1e0b39c0f5c31a7a76bfe1c3",
|
||||||
"bytes": 19338
|
"bytes": 19338
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_runner_specialization_child_boundary.py",
|
||||||
|
"sha256": "820faac9350a3c30613cc3fe928857cff29f3df364563a4ceb916b4b8b5e0c51",
|
||||||
|
"bytes": 8474
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_specialization_controller_surface.py",
|
"path": "backend/tests/test_runner_specialization_controller_surface.py",
|
||||||
"sha256": "a6caa41c68478e1ded3509e44ec319cfb3ca6a060f0129c0d6e090f8929dbf6a",
|
"sha256": "1885820b4f4306c863c104eb1a46409a93358acfccb95db2728386abbb78b369",
|
||||||
"bytes": 12349
|
"bytes": 13370
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_statistics_binding.py",
|
"path": "backend/tests/test_runner_statistics_binding.py",
|
||||||
|
|||||||
@@ -0,0 +1,310 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from test_runner_acceptance_quiescence import module, proven, smart_database # noqa: F401
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def helper():
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
"root_recovery_review_test", ROOT / "scripts/recover_ci_acceptance_once.py")
|
||||||
|
helper = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(helper)
|
||||||
|
return helper
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def partial(helper, module, smart_database, monkeypatch, tmp_path):
|
||||||
|
driver, holder, pg, clients, waits = smart_database
|
||||||
|
baseline = driver.snapshot()
|
||||||
|
registrations = copy.deepcopy(driver.runners[:2])
|
||||||
|
driver.disable_restart(pg["id"])
|
||||||
|
driver.term(pg["id"])
|
||||||
|
for row in driver.runners[:2]:
|
||||||
|
row["disabled"] = True
|
||||||
|
holder["value"] = {
|
||||||
|
"schema": module.SCHEMA, "revision": helper.ORIGIN,
|
||||||
|
"manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing",
|
||||||
|
"baseline": baseline, "registrations": registrations,
|
||||||
|
"registrationProgress": {"9": "disabled", "22": "disabled"},
|
||||||
|
"progress": {pg["id"]: "term_intent"}, "guards": driver.capture_guards(),
|
||||||
|
"guardProgress": {"runner": "installed", "host": "installed"},
|
||||||
|
}
|
||||||
|
|
||||||
|
class Adapter:
|
||||||
|
def __init__(self, projects):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
return getattr(driver, name)
|
||||||
|
|
||||||
|
def maintenance(self):
|
||||||
|
return driver.maintenance()
|
||||||
|
|
||||||
|
def snapshot(self):
|
||||||
|
return driver.snapshot()
|
||||||
|
|
||||||
|
monkeypatch.setattr(module, "ControllerDriver", Adapter)
|
||||||
|
monkeypatch.setattr(helper, "_load", lambda: (module, {}))
|
||||||
|
monkeypatch.setattr(helper, "_installed_proof", lambda: None)
|
||||||
|
monkeypatch.setattr(helper, "_read", lambda *args, **kwargs: b"private reviewed helper")
|
||||||
|
bundle = tmp_path / "bundle"
|
||||||
|
bundle.mkdir(mode=0o700)
|
||||||
|
monkeypatch.setattr(helper, "BUNDLE", bundle)
|
||||||
|
monkeypatch.setattr(helper, "__file__", str(bundle / "recover_ci_acceptance_once.py"))
|
||||||
|
monkeypatch.setattr(helper, "sys", SimpleNamespace(flags=SimpleNamespace(isolated=1),
|
||||||
|
dont_write_bytecode=True))
|
||||||
|
monkeypatch.setattr(helper, "os", SimpleNamespace(name="posix", geteuid=lambda: 0))
|
||||||
|
monkeypatch.setattr(helper, "stat", SimpleNamespace(
|
||||||
|
S_IMODE=lambda mode: 0o700))
|
||||||
|
claims = []
|
||||||
|
monkeypatch.setattr(helper, "_claim", lambda digest: claims.append(digest))
|
||||||
|
return driver, holder, pg, clients, claims
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_preflight_is_readonly_and_preserves_receipt(helper, partial):
|
||||||
|
driver, holder, _, _, claims = partial
|
||||||
|
before = copy.deepcopy(holder["value"])
|
||||||
|
effects = list(driver.effects)
|
||||||
|
result = helper.run()
|
||||||
|
assert result["status"] == "partial" and result["effectsPerformed"] is False
|
||||||
|
assert result["candidateRevision"] == helper.CANDIDATE
|
||||||
|
assert result["receiptOrigin"] == helper.ORIGIN and result["normalDeployment"] is False
|
||||||
|
assert not claims and driver.effects == effects and holder["value"] == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_reviewed_execution_resumes_same_receipt_and_two_clients(helper, module, partial):
|
||||||
|
driver, holder, pg, clients, claims = partial
|
||||||
|
before = holder["value"]
|
||||||
|
identity = (before["revision"], before["manifestDigest"], module._digest(before["baseline"]))
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert result["status"] == "quiesced" and result["invocationClaimed"] is True
|
||||||
|
assert result["retryAllowed"] is False and result["normalDeployment"] is False
|
||||||
|
assert len(claims) == 1
|
||||||
|
assert all(not row["state"]["Running"] for row in [pg, *clients])
|
||||||
|
after = holder["value"]
|
||||||
|
assert identity == (after["revision"], after["manifestDigest"], module._digest(after["baseline"]))
|
||||||
|
assert after["phase"] == "complete"
|
||||||
|
assert all(effect[0] in {"TERM", "restart", "registration"} for effect in driver.effects)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("change", ["missing", "origin", "manifest", "complete", "guarding",
|
||||||
|
"restoring", "guards", "registrations", "cid", "stage"])
|
||||||
|
def test_foreign_or_new_maintenance_never_executes(helper, partial, change):
|
||||||
|
driver, holder, _, _, claims = partial
|
||||||
|
if change == "missing":
|
||||||
|
holder["value"] = None
|
||||||
|
else:
|
||||||
|
receipt = holder["value"]
|
||||||
|
if change == "origin":
|
||||||
|
receipt["revision"] = "a" * 40
|
||||||
|
elif change == "manifest":
|
||||||
|
receipt["manifestDigest"] = "a" * 64
|
||||||
|
elif change in {"complete", "guarding", "restoring"}:
|
||||||
|
receipt["phase"] = change
|
||||||
|
elif change == "guards":
|
||||||
|
receipt["guardProgress"]["host"] = "restored"
|
||||||
|
elif change == "registrations":
|
||||||
|
receipt["registrationProgress"]["22"] = "restored"
|
||||||
|
elif change == "cid":
|
||||||
|
receipt["progress"]["a" * 64] = "term_intent"
|
||||||
|
else:
|
||||||
|
receipt["progress"][next(iter(receipt["progress"]))] = "foreign"
|
||||||
|
effects = list(driver.effects)
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert result["status"] == "blocked" and result["effectsPerformed"] is False
|
||||||
|
assert not claims and driver.effects == effects
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fault", ["queued", "changed-container", "busy-lock", "busy-fleet-lock", "claimed-marker"])
|
||||||
|
def test_admission_races_before_claim_refuse(helper, partial, monkeypatch, fault):
|
||||||
|
from contextlib import contextmanager
|
||||||
|
driver, _, _, _, claims = partial
|
||||||
|
if fault == "queued":
|
||||||
|
driver.pending_jobs = lambda: [{"status": "queued"}]
|
||||||
|
elif fault == "changed-container":
|
||||||
|
driver.state["rows"][0]["imageId"] = "sha256:" + "0" * 64
|
||||||
|
elif fault == "busy-lock":
|
||||||
|
@contextmanager
|
||||||
|
def busy():
|
||||||
|
raise RuntimeError("sensitive unexpected detail")
|
||||||
|
yield
|
||||||
|
driver.maintenance = busy
|
||||||
|
elif fault == "busy-fleet-lock":
|
||||||
|
@contextmanager
|
||||||
|
def busy_fleet(**kwargs):
|
||||||
|
raise RuntimeError("fleet lock busy")
|
||||||
|
yield
|
||||||
|
module = driver.module
|
||||||
|
monkeypatch.setattr(module.fleet, "_migration_lock", busy_fleet)
|
||||||
|
else:
|
||||||
|
def claimed(digest):
|
||||||
|
raise helper.RecoveryRejected("invocation_already_claimed_inspect_only")
|
||||||
|
monkeypatch.setattr(helper, "_claim", claimed)
|
||||||
|
effects = list(driver.effects)
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert result["status"] == "blocked" and not claims and driver.effects == effects
|
||||||
|
assert "sensitive" not in json.dumps(result)
|
||||||
|
|
||||||
|
|
||||||
|
def test_failure_after_claim_is_unknown_never_blind_retry(helper, partial, monkeypatch):
|
||||||
|
driver, _, _, _, claims = partial
|
||||||
|
driver.term = lambda cid: (_ for _ in ()).throw(RuntimeError("token-secret-value"))
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert len(claims) == 1 and result["status"] == "unknown"
|
||||||
|
assert result["effectsPerformed"] is None and result["retryAllowed"] is False
|
||||||
|
assert "token-secret-value" not in json.dumps(result)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("execute,confirmation", [(True, None), (True, "other"), (False, "other")])
|
||||||
|
def test_confirmation_cannot_be_selected(helper, partial, execute, confirmation):
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="explicit_fixed_confirmation_required"):
|
||||||
|
helper.run(execute=execute, confirmation=confirmation)
|
||||||
|
assert not partial[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_fixed_artifact_inventory_and_candidate_hash_match_immutable_code(helper):
|
||||||
|
source = ROOT / "ops/gitea-deploy-broker"
|
||||||
|
tree = ast.parse((source / "status_server.py").read_text())
|
||||||
|
files = next(ast.literal_eval(node.value) for node in tree.body
|
||||||
|
if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name)
|
||||||
|
and target.id == "_CONTROLLER_ARTIFACT_FILES" for target in node.targets))
|
||||||
|
assert helper.FILES == files
|
||||||
|
# The private bundle must contain Git's LF bytes, not a Windows CRLF checkout.
|
||||||
|
canonical = (source / helper.MODULE_NAME).read_bytes().replace(b"\r\n", b"\n")
|
||||||
|
assert hashlib.sha256(canonical).hexdigest() == helper.CANDIDATE_SHA
|
||||||
|
assert helper.ORIGIN != helper.CANDIDATE
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("change", ["module", "artifact", "revision", "tree", "version"])
|
||||||
|
def test_installed_identity_pin_rejects_drift(helper, monkeypatch, change):
|
||||||
|
raws = {name: name.encode() for name in helper.FILES}
|
||||||
|
original_hash = hashlib.sha256(raws[helper.MODULE_NAME]).hexdigest()
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
for name, raw in raws.items():
|
||||||
|
digest.update(name.encode() + b"\0" + raw + b"\0")
|
||||||
|
monkeypatch.setattr(helper, "ORIGINAL_MODULE_SHA", original_hash)
|
||||||
|
monkeypatch.setattr(helper, "ARTIFACT_SHA", digest.hexdigest())
|
||||||
|
release = {"apiVersion": "itworx.controller-artifact/v1", "controllerVersion": "2.1.21",
|
||||||
|
"sourceRepository": "jens/unraid-appops-gateway", "sourceRevision": helper.ORIGIN,
|
||||||
|
"sourceTree": helper.ORIGIN_TREE, "artifactSha256": helper.ARTIFACT_SHA,
|
||||||
|
"builtAt": "2026-10-10T00:00:00Z"}
|
||||||
|
if change in {"module", "artifact"}:
|
||||||
|
raws[helper.MODULE_NAME if change == "module" else helper.FILES[0]] += b"altered"
|
||||||
|
else:
|
||||||
|
release[{"revision": "sourceRevision", "tree": "sourceTree",
|
||||||
|
"version": "controllerVersion"}[change]] = "different"
|
||||||
|
monkeypatch.setattr(helper, "_read", lambda path, **kwargs:
|
||||||
|
json.dumps(release).encode() if path.name == "controller-release.json"
|
||||||
|
else raws[path.name])
|
||||||
|
with pytest.raises(helper.RecoveryRejected):
|
||||||
|
helper._installed_proof()
|
||||||
|
|
||||||
|
|
||||||
|
def test_candidate_hash_refuses_before_any_import(helper, monkeypatch):
|
||||||
|
monkeypatch.setattr(helper, "_installed_proof", lambda: None)
|
||||||
|
monkeypatch.setattr(helper, "_read", lambda *args, **kwargs: b"foreign candidate")
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="candidate_bytes_changed"):
|
||||||
|
helper._load()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", ["root", "isolated", "bytecode"])
|
||||||
|
def test_untrusted_interpreter_refuses(helper, partial, monkeypatch, field):
|
||||||
|
if field == "root":
|
||||||
|
helper.os.geteuid = lambda: 1000
|
||||||
|
elif field == "isolated":
|
||||||
|
helper.sys.flags.isolated = 0
|
||||||
|
else:
|
||||||
|
helper.sys.dont_write_bytecode = False
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="isolated_root_python_required"):
|
||||||
|
helper.run()
|
||||||
|
assert not partial[-1]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root POSIX file safety")
|
||||||
|
@pytest.mark.parametrize("boundary", ["before-link", "after-link"])
|
||||||
|
def test_claim_publication_failure_is_atomic_and_never_replayed(helper, monkeypatch, tmp_path, boundary):
|
||||||
|
bundle = tmp_path / "bundle"
|
||||||
|
bundle.mkdir(mode=0o700)
|
||||||
|
monkeypatch.setattr(helper, "BUNDLE", bundle)
|
||||||
|
monkeypatch.setattr(helper, "MARKER", bundle / "invoked.json")
|
||||||
|
monkeypatch.setattr(helper, "_trusted_parents", lambda path: None)
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def fsync(fd):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
if calls == (1 if boundary == "before-link" else 2):
|
||||||
|
raise OSError("simulated actual durability failure")
|
||||||
|
os.fsync(fd)
|
||||||
|
|
||||||
|
proxy = SimpleNamespace(**{name: getattr(os, name) for name in dir(os)})
|
||||||
|
proxy.fsync = fsync
|
||||||
|
monkeypatch.setattr(helper, "os", proxy)
|
||||||
|
with pytest.raises(OSError):
|
||||||
|
helper._claim("a" * 64)
|
||||||
|
marker = bundle / "invoked.json"
|
||||||
|
assert marker.exists() is (boundary == "after-link")
|
||||||
|
if marker.exists():
|
||||||
|
assert json.loads(helper._read(marker, private=True))["candidateRevision"] == helper.CANDIDATE
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="invocation_already_claimed"):
|
||||||
|
helper._claim("a" * 64)
|
||||||
|
assert not list(bundle.glob(".acceptance-recovery-invocation-*"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root POSIX file safety")
|
||||||
|
def test_private_marker_atomic_once_and_reader_metadata(helper, monkeypatch, tmp_path):
|
||||||
|
bundle = tmp_path / "bundle"
|
||||||
|
bundle.mkdir(mode=0o700)
|
||||||
|
monkeypatch.setattr(helper, "BUNDLE", bundle)
|
||||||
|
monkeypatch.setattr(helper, "MARKER", bundle / "invoked.json")
|
||||||
|
# pytest's sandbox ancestors are not the deployment bundle; test leaf metadata here.
|
||||||
|
monkeypatch.setattr(helper, "_trusted_parents", lambda path: None)
|
||||||
|
helper._claim("a" * 64)
|
||||||
|
marker = bundle / "invoked.json"
|
||||||
|
assert stat.S_IMODE(marker.stat().st_mode) == 0o600
|
||||||
|
assert json.loads(helper._read(marker, private=True))["receiptOrigin"] == helper.ORIGIN
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="invocation_already_claimed"):
|
||||||
|
helper._claim("a" * 64)
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="invocation_already_claimed"):
|
||||||
|
helper._claim("b" * 64)
|
||||||
|
for mode in (0o644, 0o640, 0o664, 0o777):
|
||||||
|
marker.chmod(mode)
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"):
|
||||||
|
helper._read(marker, private=True)
|
||||||
|
marker.chmod(0o600)
|
||||||
|
os.chown(marker, 1000, 0)
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"):
|
||||||
|
helper._read(marker, private=True)
|
||||||
|
os.chown(marker, 0, 0)
|
||||||
|
link = bundle / "extra-link"
|
||||||
|
os.link(marker, link)
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"):
|
||||||
|
helper._read(marker, private=True)
|
||||||
|
link.unlink()
|
||||||
|
proxy = SimpleNamespace(**{name: getattr(os, name) for name in dir(os)})
|
||||||
|
proxy.fstat = lambda fd: SimpleNamespace(**{
|
||||||
|
key: getattr(os.fstat(fd), key) + (1 if key == "st_ino" else 0)
|
||||||
|
for key in ("st_dev", "st_ino", "st_uid", "st_gid", "st_mode", "st_nlink", "st_size",
|
||||||
|
"st_mtime_ns", "st_ctime_ns")})
|
||||||
|
with monkeypatch.context() as scope:
|
||||||
|
scope.setattr(helper, "os", proxy)
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="recovery_file_changed"):
|
||||||
|
helper._read(marker, private=True)
|
||||||
|
marker.unlink()
|
||||||
|
marker.symlink_to(bundle / "absent")
|
||||||
|
with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"):
|
||||||
|
helper._read(marker, private=True)
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import builtins
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from test_runner_acceptance_quiescence import module, proven # noqa: F401
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def helper():
|
||||||
|
spec = importlib.util.spec_from_file_location("api_child_recovery_test",
|
||||||
|
ROOT / "scripts/recover_ci_api_term_once.py")
|
||||||
|
result = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def prepared(helper, module, proven, monkeypatch, tmp_path):
|
||||||
|
from contextlib import nullcontext
|
||||||
|
driver, holder = proven
|
||||||
|
baseline = driver.snapshot()
|
||||||
|
holder["value"] = {
|
||||||
|
"schema": module.SCHEMA, "revision": module.REVIEWED_RECEIPT_PREDECESSOR,
|
||||||
|
"manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing",
|
||||||
|
"baseline": baseline, "registrations": copy.deepcopy(driver.runners[:2]),
|
||||||
|
"registrationProgress": {"9": "disabled", "22": "disabled"},
|
||||||
|
"progress": {row["id"]: "term_intent" for row in baseline["rows"]},
|
||||||
|
"guards": driver.capture_guards(), "guardProgress": {"runner": "installed", "host": "installed"},
|
||||||
|
}
|
||||||
|
for row in driver.state["rows"]:
|
||||||
|
row["restartPolicy"] = {"Name": "no", "MaximumRetryCount": 0}
|
||||||
|
for row in driver.runners[:2]:
|
||||||
|
row["disabled"] = True
|
||||||
|
driver._effect_admission = module.ControllerDriver._effect_admission.__get__(driver)
|
||||||
|
base = SimpleNamespace(ORIGIN=module.REVIEWED_RECEIPT_PREDECESSOR,
|
||||||
|
CANDIDATE="07c75a3aae4d0881f93e4bef2aa0275235c4253a",
|
||||||
|
CANDIDATE_SHA="236af955566bd2713684977008045c7b9f1390b83fff4b160da83c6473bf4246",
|
||||||
|
MARKER=tmp_path / "original-marker.json")
|
||||||
|
def original_scope(module, receipt):
|
||||||
|
assert receipt is not None and receipt["revision"] == base.ORIGIN
|
||||||
|
assert receipt["phase"] == "quiescing"
|
||||||
|
assert receipt["manifestDigest"] == module._digest(module.OBSERVATION)
|
||||||
|
base._scope = original_scope
|
||||||
|
base._load = lambda: (module, {})
|
||||||
|
base._installed_proof = lambda: None
|
||||||
|
base._read = lambda *args, **kwargs: json.dumps({"candidateRevision": base.CANDIDATE,
|
||||||
|
"candidateSha256": base.CANDIDATE_SHA, "receiptOrigin": base.ORIGIN,
|
||||||
|
"receiptDigestBefore": "a" * 64}).encode()
|
||||||
|
monkeypatch.setattr(helper, "_load_base", lambda: base)
|
||||||
|
monkeypatch.setattr(module, "ControllerDriver", lambda projects: driver)
|
||||||
|
bundle = tmp_path / "bundle"
|
||||||
|
bundle.mkdir()
|
||||||
|
monkeypatch.setattr(helper, "BUNDLE", bundle)
|
||||||
|
monkeypatch.setattr(helper, "__file__", str(bundle / "recover_ci_api_term_once.py"))
|
||||||
|
monkeypatch.setattr(helper, "os", SimpleNamespace(name="posix", geteuid=lambda: 0))
|
||||||
|
monkeypatch.setattr(helper, "sys", SimpleNamespace(flags=SimpleNamespace(isolated=1), dont_write_bytecode=True))
|
||||||
|
monkeypatch.setattr(helper, "stat", SimpleNamespace(S_IMODE=lambda mode: 0o700))
|
||||||
|
claims, effects = [], []
|
||||||
|
monkeypatch.setattr(helper, "_claim", lambda *args: claims.append(args))
|
||||||
|
def probe(driver, module, cid, ticket=None):
|
||||||
|
if ticket is None:
|
||||||
|
return {"pid": 11, "parent": 1, "start": 123, "initStart": 10, "namespace": "pid:[111]"}
|
||||||
|
effects.append(cid)
|
||||||
|
monkeypatch.setattr(helper, "_probe", probe)
|
||||||
|
monkeypatch.setattr(module.fleet, "_migration_lock", lambda **kwargs: nullcontext())
|
||||||
|
return driver, holder, base, claims, effects
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_default_never_claims_or_signals(helper, prepared):
|
||||||
|
driver, holder, _, claims, effects = prepared
|
||||||
|
receipt = copy.deepcopy(holder["value"])
|
||||||
|
result = helper.run()
|
||||||
|
assert result["status"] == "ready" and result["effectsPerformed"] is False
|
||||||
|
assert result["existingMaintenanceEffects"] is True and result["normalDeployment"] is False
|
||||||
|
assert not claims and not effects and holder["value"] == receipt
|
||||||
|
assert not driver.effects
|
||||||
|
|
||||||
|
|
||||||
|
def test_three_exact_children_only_and_original_receipt_unchanged(helper, prepared):
|
||||||
|
_, holder, _, claims, effects = prepared
|
||||||
|
before = copy.deepcopy(holder["value"])
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert result["status"] == "term_sent_inspect_exits" and result["resolvedChildren"] == 3
|
||||||
|
assert result["confirmedSignalAcks"] == 3
|
||||||
|
assert len(claims) == 1 and effects == list(helper.API_IDS)
|
||||||
|
assert holder["value"] == before and result["retryAllowed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fault", ["missing", "foreign", "complete", "missing-intent", "foreign-intent",
|
||||||
|
"queue", "busy-fleet", "claimed", "container", "previous-marker"])
|
||||||
|
def test_failed_admission_never_signals(helper, prepared, module, monkeypatch, fault):
|
||||||
|
from contextlib import contextmanager
|
||||||
|
driver, holder, base, claims, effects = prepared
|
||||||
|
if fault == "missing":
|
||||||
|
holder["value"] = None
|
||||||
|
elif fault == "foreign":
|
||||||
|
holder["value"]["revision"] = "f" * 40
|
||||||
|
elif fault == "complete":
|
||||||
|
holder["value"]["phase"] = "complete"
|
||||||
|
elif fault == "missing-intent":
|
||||||
|
holder["value"]["progress"].pop(helper.API_IDS[0])
|
||||||
|
elif fault == "foreign-intent":
|
||||||
|
holder["value"]["progress"][helper.API_IDS[0]] = "restart_intent"
|
||||||
|
elif fault == "queue":
|
||||||
|
driver.pending_jobs = lambda: [{"status": "queued"}]
|
||||||
|
elif fault == "busy-fleet":
|
||||||
|
@contextmanager
|
||||||
|
def busy(**kwargs):
|
||||||
|
raise RuntimeError("busy")
|
||||||
|
yield
|
||||||
|
monkeypatch.setattr(module.fleet, "_migration_lock", busy)
|
||||||
|
elif fault == "claimed":
|
||||||
|
monkeypatch.setattr(helper, "_claim", lambda *args: (_ for _ in ()).throw(RuntimeError("exists")))
|
||||||
|
elif fault == "container":
|
||||||
|
driver.state["rows"][0]["imageId"] = "foreign"
|
||||||
|
else:
|
||||||
|
base._read = lambda *args, **kwargs: b"{}"
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert result["status"] == "blocked" and not effects and not claims
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("exit_attested", [True, False])
|
||||||
|
def test_missing_ack_advances_only_on_exact_owned_exit(helper, prepared, monkeypatch, exit_attested):
|
||||||
|
driver, holder, _, claims, effects = prepared
|
||||||
|
before = copy.deepcopy(holder["value"])
|
||||||
|
normal = helper._probe
|
||||||
|
def probe(driver, module, cid, ticket=None):
|
||||||
|
if ticket is not None and cid == helper.API_IDS[0]:
|
||||||
|
effects.append(cid)
|
||||||
|
if exit_attested:
|
||||||
|
row = next(row for row in driver.state["rows"] if row["id"] == cid)
|
||||||
|
row["state"].update(Running=False, Status="exited", Pid=0, FinishedAt="2026-10-10T00:00:00Z")
|
||||||
|
raise RuntimeError("namespace teardown, token-secret-not-output")
|
||||||
|
return normal(driver, module, cid, ticket)
|
||||||
|
monkeypatch.setattr(helper, "_probe", probe)
|
||||||
|
result = helper.run(execute=True, confirmation=helper.CONFIRMATION)
|
||||||
|
assert len(claims) == 1 and effects.count(helper.API_IDS[0]) == 1
|
||||||
|
assert result["status"] == ("term_sent_inspect_exits" if exit_attested else "unknown")
|
||||||
|
assert len(effects) == (3 if exit_attested else 1)
|
||||||
|
assert "token-secret" not in json.dumps(result) and holder["value"] == before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root durable marker")
|
||||||
|
@pytest.mark.parametrize("fault", [None, "before-link", "after-link"])
|
||||||
|
def test_actual_atomic_marker_once_under_write_failures(helper, monkeypatch, tmp_path, fault):
|
||||||
|
marker = tmp_path / "private" / "once.json"
|
||||||
|
marker.parent.mkdir(mode=0o700)
|
||||||
|
monkeypatch.setattr(helper, "MARKER", marker)
|
||||||
|
base = SimpleNamespace(_trusted_parents=lambda path: None,
|
||||||
|
ORIGIN="0ea6f267c28d51ac7113e6d943c89857d3a189ab",
|
||||||
|
CANDIDATE_SHA="a" * 64,
|
||||||
|
_read=lambda *args, **kwargs: b"reviewed source")
|
||||||
|
calls = 0
|
||||||
|
def fsync(fd):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
if (fault == "before-link" and calls == 1) or (fault == "after-link" and calls == 2):
|
||||||
|
raise OSError("injected actual fsync failure")
|
||||||
|
os.fsync(fd)
|
||||||
|
proxy = SimpleNamespace(**{name: getattr(os, name) for name in dir(os)})
|
||||||
|
proxy.fsync = fsync
|
||||||
|
monkeypatch.setattr(helper, "os", proxy)
|
||||||
|
if fault:
|
||||||
|
with pytest.raises(OSError):
|
||||||
|
helper._claim(base, "b" * 64, {})
|
||||||
|
else:
|
||||||
|
helper._claim(base, "b" * 64, {})
|
||||||
|
assert marker.exists() is (fault != "before-link")
|
||||||
|
assert not list(marker.parent.glob(".api-term-invocation-*"))
|
||||||
|
if marker.exists():
|
||||||
|
payload = json.loads(marker.read_bytes())
|
||||||
|
assert payload["receiptOrigin"] == base.ORIGIN and payload["kind"] == "ci_api_child_term_once"
|
||||||
|
for digest in ("b" * 64, "c" * 64):
|
||||||
|
with pytest.raises(RuntimeError, match="already_invoked"):
|
||||||
|
helper._claim(base, digest, {})
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_confirmation_required(helper, prepared):
|
||||||
|
with pytest.raises(RuntimeError, match="separate_fixed_confirmation_required"):
|
||||||
|
helper.run(execute=True)
|
||||||
|
assert not prepared[-1] and not prepared[-2]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fault", [None, "argv", "parent", "extra-child", "namespace", "exe", "pid-reuse", "esrch", "shell-term"])
|
||||||
|
def test_actual_child_script_pidfd_and_process_races(helper, monkeypatch, capsys, fault):
|
||||||
|
signals, closes = [], []
|
||||||
|
start = 123
|
||||||
|
def procstat(parent, tick):
|
||||||
|
return "11 (python) S " + str(parent) + " " + " ".join(["0"] * 17) + " " + str(tick)
|
||||||
|
class FakePath:
|
||||||
|
def __init__(self, path):
|
||||||
|
self.path = path
|
||||||
|
self.name = path.rsplit("/", 1)[-1]
|
||||||
|
def iterdir(self):
|
||||||
|
return [FakePath("/proc/" + str(pid)) for pid in ([1, 11, 99, 12] if fault == "extra-child" else [1, 11, 99])]
|
||||||
|
def read_bytes(self):
|
||||||
|
args = (["sh", "-c", helper.SHELL_COMMAND] if self.path == "/proc/1/cmdline" else
|
||||||
|
["foreign"] if fault == "argv" else helper.CHILD_ARGV)
|
||||||
|
return ("\0".join(args) + "\0").encode()
|
||||||
|
def read_text(self):
|
||||||
|
if self.path.endswith("/status"):
|
||||||
|
return "SigCgt:\t00014002" if fault == "shell-term" else "SigCgt:\t00010002"
|
||||||
|
if self.path == "/proc/1/stat":
|
||||||
|
return procstat(0, 10)
|
||||||
|
return procstat(0 if self.path == "/proc/99/stat" else 2 if fault == "parent" else 1, start)
|
||||||
|
def pidfd_open(pid, flags):
|
||||||
|
nonlocal start
|
||||||
|
assert pid == 11 and flags == 0
|
||||||
|
if fault == "pid-reuse":
|
||||||
|
start += 1
|
||||||
|
if fault == "esrch":
|
||||||
|
raise ProcessLookupError()
|
||||||
|
return 77
|
||||||
|
def readlink(path):
|
||||||
|
if path.endswith("/exe"):
|
||||||
|
return "/foreign" if fault == "exe" else "/usr/local/bin/python3.12"
|
||||||
|
return "pid:[222]" if fault == "namespace" and "/11/" in path else "pid:[111]"
|
||||||
|
fake_os = SimpleNamespace(geteuid=lambda: 0, getpid=lambda: 99, pidfd_open=pidfd_open,
|
||||||
|
readlink=readlink, close=lambda fd: closes.append(fd))
|
||||||
|
fake_signal = SimpleNamespace(SIGTERM=15, pidfd_send_signal=lambda fd, sig, info, flags:
|
||||||
|
signals.append((fd, sig, info, flags)))
|
||||||
|
ticket = {"pid": 11, "parent": 1, "start": 123, "initStart": 10, "namespace": "pid:[111]"}
|
||||||
|
fake_sys = SimpleNamespace(flags=SimpleNamespace(isolated=1), dont_write_bytecode=True,
|
||||||
|
argv=["-c", json.dumps(ticket)], exit=lambda code: (_ for _ in ()).throw(SystemExit(code)))
|
||||||
|
actual_import = builtins.__import__
|
||||||
|
modules = {"os": fake_os, "signal": fake_signal, "sys": fake_sys,
|
||||||
|
"pathlib": SimpleNamespace(Path=FakePath)}
|
||||||
|
namespace = {"__builtins__": {**vars(builtins), "__import__": lambda name, *args, **kwargs:
|
||||||
|
modules.get(name) or actual_import(name, *args, **kwargs)}}
|
||||||
|
if fault is None:
|
||||||
|
exec(helper.CHILD_SOURCE, namespace)
|
||||||
|
assert signals == [(77, 15, None, 0)] and closes == [77]
|
||||||
|
assert json.loads(capsys.readouterr().out) == {"status": "term_sent"}
|
||||||
|
else:
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
exec(helper.CHILD_SOURCE, namespace)
|
||||||
|
assert not signals
|
||||||
|
assert json.loads(capsys.readouterr().out) == {"status": "rejected"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_original_c185_pin_and_no_stronger_signal_or_host_pid(helper):
|
||||||
|
original = (ROOT / "scripts/recover_ci_acceptance_once.py").read_bytes().replace(b"\r\n", b"\n")
|
||||||
|
assert hashlib.sha256(original).hexdigest() == helper.BASE_SHA
|
||||||
|
assert len(set(helper.API_IDS)) == 3
|
||||||
|
assert "os.kill" not in helper.CHILD_SOURCE and "SIGKILL" not in helper.CHILD_SOURCE
|
||||||
|
assert "pidfd_send_signal(fd,signal.SIGTERM,None,0)" in helper.CHILD_SOURCE
|
||||||
@@ -228,6 +228,124 @@ def test_positive_preserves_resources_and_windows_then_idempotent(module, proven
|
|||||||
module.quiesce(revision="e" * 40, projects={}, driver=driver)
|
module.quiesce(revision="e" * 40, projects={}, driver=driver)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def smart_database(proven):
|
||||||
|
# Synthetic dependency graph over two existing running identities; this
|
||||||
|
# models retained sessions, not a claim that mock-ats uses this production DB.
|
||||||
|
driver, receipt = proven
|
||||||
|
project = "workmesh-runtime-19421"
|
||||||
|
pg = next(row for row in driver.state["rows"]
|
||||||
|
if row["composeLabels"]["com.docker.compose.project"] == project
|
||||||
|
and row["composeLabels"]["com.docker.compose.service"] == "postgres")
|
||||||
|
clients = [row for row in driver.state["rows"]
|
||||||
|
if row["composeLabels"]["com.docker.compose.project"] == project
|
||||||
|
and row["composeLabels"]["com.docker.compose.service"] in {"api", "mock-ats"}]
|
||||||
|
assert len(clients) == 2 and all(row["state"]["Running"] for row in clients)
|
||||||
|
driver.state["rows"] = [pg] + [row for row in driver.state["rows"] if row is not pg]
|
||||||
|
original_term = driver.term
|
||||||
|
pending = False
|
||||||
|
waits = []
|
||||||
|
|
||||||
|
def term(cid):
|
||||||
|
nonlocal pending
|
||||||
|
if cid == pg["id"] and pg["state"]["Running"]:
|
||||||
|
pending = True
|
||||||
|
driver.effects.append(("TERM", cid))
|
||||||
|
else:
|
||||||
|
original_term(cid)
|
||||||
|
if pending and all(not row["state"]["Running"] for row in clients):
|
||||||
|
pg["state"].update(Running=False, Status="exited", Pid=0,
|
||||||
|
FinishedAt="2026-10-10T00:00:00Z")
|
||||||
|
|
||||||
|
def wait(cid):
|
||||||
|
waits.append(sum(effect[0] == "TERM" for effect in driver.effects))
|
||||||
|
return not next(row for row in driver.state["rows"] if row["id"] == cid)["state"]["Running"]
|
||||||
|
|
||||||
|
driver.term, driver.wait_exited = term, wait
|
||||||
|
return driver, receipt, pg, clients, waits
|
||||||
|
|
||||||
|
|
||||||
|
def test_two_clients_receive_term_before_waiting_for_smart_database_shutdown(module, smart_database):
|
||||||
|
driver, receipt, pg, clients, waits = smart_database
|
||||||
|
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||||
|
assert result["status"] == "quiesced"
|
||||||
|
assert all(not row["state"]["Running"] for row in [pg, *clients])
|
||||||
|
assert waits and all(count == 12 for count in waits)
|
||||||
|
assert receipt["value"]["phase"] == "complete"
|
||||||
|
assert sum(effect[0] == "TERM" for effect in driver.effects) == 12
|
||||||
|
|
||||||
|
|
||||||
|
def test_old_partial_receipt_resumes_two_passes_without_rewriting_origin(module, smart_database, monkeypatch):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
driver, holder, pg, clients, _ = smart_database
|
||||||
|
baseline = copy.deepcopy(driver.state)
|
||||||
|
original = {
|
||||||
|
"schema": module.SCHEMA, "revision": module.REVIEWED_RECEIPT_PREDECESSOR,
|
||||||
|
"manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing",
|
||||||
|
"baseline": baseline, "registrations": copy.deepcopy(driver.runners[:2]),
|
||||||
|
"registrationProgress": {"9": "disabled", "22": "disabled"},
|
||||||
|
"progress": {pg["id"]: "term_intent"}, "guards": driver.capture_guards(),
|
||||||
|
"guardProgress": {"runner": "installed", "host": "installed"},
|
||||||
|
}
|
||||||
|
driver.disable_restart(pg["id"])
|
||||||
|
driver.term(pg["id"])
|
||||||
|
assert pg["state"]["Running"] and all(row["state"]["Running"] for row in clients)
|
||||||
|
for runner in driver.runners[:2]:
|
||||||
|
runner["disabled"] = True
|
||||||
|
holder["value"] = original
|
||||||
|
frozen = (original["revision"], original["manifestDigest"], module._digest(baseline))
|
||||||
|
release = {"controllerVersion": "2.1.21", "sourceRepository": module.fleet.SOURCE_REPOSITORY,
|
||||||
|
"sourceRevision": "e" * 40, "sourceTree": "f" * 40}
|
||||||
|
monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release))
|
||||||
|
driver.accepts_reviewed_receipt_predecessor = module.ControllerDriver({}).accepts_reviewed_receipt_predecessor
|
||||||
|
assert module.preflight(revision="e" * 40, projects={}, driver=driver)["status"] == "partial"
|
||||||
|
assert module.quiesce(revision="e" * 40, projects={}, driver=driver)["status"] == "quiesced"
|
||||||
|
final = holder["value"]
|
||||||
|
assert (final["revision"], final["manifestDigest"], module._digest(final["baseline"])) == frozen
|
||||||
|
assert all(not row["state"]["Running"] for row in [pg, *clients])
|
||||||
|
assert module.quiesce(revision="e" * 40, projects={}, driver=driver)["status"] == "already_quiesced"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("change", ["origin", "manifest", "revision", "tree", "repository", "version", "missing-reader"])
|
||||||
|
def test_predecessor_resume_rejects_foreign_receipt_or_current_source(module, proven, monkeypatch, change):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
driver, holder = proven
|
||||||
|
driver.timeout = True
|
||||||
|
module.quiesce(revision=module.REVIEWED_RECEIPT_PREDECESSOR, projects={}, driver=driver)
|
||||||
|
release = {"controllerVersion": "2.1.21", "sourceRepository": module.fleet.SOURCE_REPOSITORY,
|
||||||
|
"sourceRevision": "e" * 40, "sourceTree": "f" * 40}
|
||||||
|
monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release))
|
||||||
|
if change == "origin":
|
||||||
|
holder["value"]["revision"] = "a" * 40
|
||||||
|
elif change == "manifest":
|
||||||
|
holder["value"]["manifestDigest"] = "a" * 64
|
||||||
|
elif change == "revision":
|
||||||
|
release["sourceRevision"] = "a" * 40
|
||||||
|
elif change == "tree":
|
||||||
|
release["sourceTree"] = "invalid"
|
||||||
|
elif change == "repository":
|
||||||
|
release["sourceRepository"] = "jens/foreign"
|
||||||
|
elif change == "version":
|
||||||
|
release["controllerVersion"] = "2.1.22"
|
||||||
|
if change != "missing-reader":
|
||||||
|
driver.accepts_reviewed_receipt_predecessor = module.ControllerDriver({}).accepts_reviewed_receipt_predecessor
|
||||||
|
before = len(driver.effects)
|
||||||
|
with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"):
|
||||||
|
module.quiesce(revision="e" * 40, projects={}, driver=driver)
|
||||||
|
assert len(driver.effects) == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_predecessor_failure_projection_preserves_effect_uncertainty(module, proven):
|
||||||
|
driver, _ = proven
|
||||||
|
driver.timeout = True
|
||||||
|
module.quiesce(revision=module.REVIEWED_RECEIPT_PREDECESSOR, projects={}, driver=driver)
|
||||||
|
result = module.failure_projection("e" * 40)
|
||||||
|
assert result["status"] == "partial" and result["guardHeld"] is True
|
||||||
|
assert result["effectsPerformed"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_mount_order_is_canonical_for_fixed_current_and_resume_baseline(module, proven):
|
def test_mount_order_is_canonical_for_fixed_current_and_resume_baseline(module, proven):
|
||||||
driver, _ = proven
|
driver, _ = proven
|
||||||
baseline = copy.deepcopy(driver.state)
|
baseline = copy.deepcopy(driver.state)
|
||||||
|
|||||||
@@ -906,6 +906,38 @@ def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(m
|
|||||||
assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rollback_preserves_unraid_existing_hugetlb_controller(monkeypatch):
|
||||||
|
envelope = _module()
|
||||||
|
initial = ["cpuset", "cpu", "io", "memory", "hugetlb", "pids"]
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, payload:
|
||||||
|
calls.append((mode, payload)) or {"removed": True, "familyExists": False})
|
||||||
|
envelope.rollback_host_cgroup_family(created=True, initial_controllers=initial)
|
||||||
|
assert calls == [("rollback", {"created": True, "initialControllers": initial})]
|
||||||
|
|
||||||
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123)
|
||||||
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [{
|
||||||
|
"cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3),
|
||||||
|
}])
|
||||||
|
monkeypatch.setattr(envelope, "_probe_read", lambda fd, name:
|
||||||
|
"" if name == "cgroup.procs" else
|
||||||
|
"cpu memory pids" if fd == 123 else " ".join(initial))
|
||||||
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
||||||
|
monkeypatch.setattr(envelope.os, "rmdir", lambda *args, **kwargs: None)
|
||||||
|
writes = []
|
||||||
|
monkeypatch.setattr(envelope, "_probe_write", lambda *args: writes.append(args))
|
||||||
|
result = envelope._probe_rollback_family(1, {"op": "rollback", "created": True,
|
||||||
|
"initialControllers": initial})
|
||||||
|
assert result == {"removed": True, "familyExists": False}
|
||||||
|
assert writes == [(123, "cgroup.subtree_control", "-cpu -memory -pids")]
|
||||||
|
for controllers in (["unknown"], initial + ["unknown"]):
|
||||||
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
||||||
|
envelope.rollback_host_cgroup_family(created=True, initial_controllers=controllers)
|
||||||
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
||||||
|
envelope._probe_rollback_family(1, {"op": "rollback", "created": True,
|
||||||
|
"initialControllers": controllers})
|
||||||
|
|
||||||
|
|
||||||
def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback():
|
def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback():
|
||||||
envelope = _module()
|
envelope = _module()
|
||||||
image_id = "sha256:" + "a" * 64
|
image_id = "sha256:" + "a" * 64
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from test_runner_fleet_specialize import specialize # noqa: F401
|
||||||
|
|
||||||
|
|
||||||
|
def test_child_fixed_failure_is_finite_and_does_not_echo_exception(specialize, monkeypatch, capsys):
|
||||||
|
def fail():
|
||||||
|
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_entrypoint_untrusted")
|
||||||
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
||||||
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
||||||
|
captured = capsys.readouterr()
|
||||||
|
assert captured.out == ""
|
||||||
|
assert captured.err == "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_entrypoint_untrusted\n"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("private", ["token-secret", "runner_fleet_specialization_arbitrary_secret",
|
||||||
|
"runner_fleet_specialization_entrypoint_untrusted\nsecret"])
|
||||||
|
def test_unknown_child_details_are_redacted(specialize, monkeypatch, capsys, private):
|
||||||
|
def fail():
|
||||||
|
raise specialize.RunnerFleetSpecializeError(private)
|
||||||
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
||||||
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
||||||
|
assert capsys.readouterr().err == "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_script_execution_failed\n"
|
||||||
|
assert specialize.public_specialization_failure(RuntimeError(private)) == "runner_fleet_specialization_script_execution_failed"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fault", [None, "stdout", "multiline", "secret", "code-prefix", "exit", "too-large", "unknown"])
|
||||||
|
def test_parent_adopts_only_exact_single_finite_envelope(specialize, fault):
|
||||||
|
code = "runner_fleet_specialization_entrypoint_untrusted"
|
||||||
|
result = SimpleNamespace(returncode=1, stdout="", stderr=f"APPOPS_SPECIALIZATION_FAILURE:{code}\n")
|
||||||
|
if fault == "stdout":
|
||||||
|
result.stdout = "sensitive log"
|
||||||
|
elif fault == "multiline":
|
||||||
|
result.stderr += "secret\n"
|
||||||
|
elif fault == "secret":
|
||||||
|
result.stderr = "secret-value"
|
||||||
|
elif fault == "code-prefix":
|
||||||
|
result.stderr = f"{code}\n"
|
||||||
|
elif fault == "exit":
|
||||||
|
result.returncode = 137
|
||||||
|
elif fault == "too-large":
|
||||||
|
result.stderr = "x" * 257
|
||||||
|
elif fault == "unknown":
|
||||||
|
result.stderr = "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_not_reviewed\n"
|
||||||
|
assert specialize._script_failure_code(result) == (code if fault is None else "runner_fleet_specialization_script_execution_failed")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0 or shutil.which("bash") is None,
|
||||||
|
reason="real POSIX root Bash parent proof in isolated Linux qualification")
|
||||||
|
@pytest.mark.parametrize("deny", [False, True])
|
||||||
|
def test_real_bash_parent_identity_and_child_error_propagation_with_stub_only(
|
||||||
|
specialize, monkeypatch, tmp_path, deny,
|
||||||
|
):
|
||||||
|
# Source-identical Bash structure with only its fixed executor redirected to
|
||||||
|
# this sandbox's stub. Never call apply, deploy, Docker, native APIs or journals.
|
||||||
|
source_root = Path(__file__).resolve().parents[2]
|
||||||
|
state = tmp_path / "state"
|
||||||
|
state.mkdir(mode=0o700)
|
||||||
|
stub = tmp_path / "readonly_stub.py"
|
||||||
|
shell = (source_root / "scripts/upgrade_ci_runner_fleet_specialized.sh").read_text()
|
||||||
|
shell = shell.replace('/opt/gitea-deploy/runner_fleet_specialize.py', str(stub))
|
||||||
|
approved = hashlib.sha256(shell.encode()).hexdigest()
|
||||||
|
stub.write_text(
|
||||||
|
"import sys\nfrom pathlib import Path\n"
|
||||||
|
+ "sys.path.insert(0," + repr(str(source_root / "ops/gitea-deploy-broker")) + ")\n"
|
||||||
|
+ "import runner_fleet_specialize as f\n"
|
||||||
|
+ "f.SPECIALIZED_STATE_ROOT=Path(" + repr(str(state)) + ")\n"
|
||||||
|
+ "f._APPROVED_SCRIPT_SHA256=" + repr("0" * 64 if deny else approved) + "\n"
|
||||||
|
+ "if not f._authorized_script_parent():\n"
|
||||||
|
+ " print('APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_entrypoint_untrusted',file=sys.stderr)\n"
|
||||||
|
+ " sys.exit(1)\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(specialize, "ROOT", tmp_path)
|
||||||
|
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_ROOT", state)
|
||||||
|
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", state / "specialized.json")
|
||||||
|
monkeypatch.setattr(specialize, "_ensure_state_root", lambda: None)
|
||||||
|
if deny:
|
||||||
|
with pytest.raises(specialize.RunnerFleetSpecializeError,
|
||||||
|
match="^runner_fleet_specialization_entrypoint_untrusted$"):
|
||||||
|
specialize._run_approved_shell_script(shell.encode())
|
||||||
|
else:
|
||||||
|
specialize._run_approved_shell_script(shell.encode())
|
||||||
|
assert list(state.iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("controllers,accepted", [
|
||||||
|
(["cpu", "memory", "pids"], True),
|
||||||
|
(["cpuset", "cpu", "io", "memory", "hugetlb", "pids"], True),
|
||||||
|
(["cpuset", "cpu", "io", "memory", "hugetlb", "pids", "foreign"], False),
|
||||||
|
])
|
||||||
|
@pytest.mark.parametrize("boundary", ["snapshot", "ensure", "journal"])
|
||||||
|
def test_observed_hugetlb_inventory_and_legacy_records_remain_finite(
|
||||||
|
specialize, monkeypatch, tmp_path, controllers, accepted, boundary,
|
||||||
|
):
|
||||||
|
if boundary == "snapshot":
|
||||||
|
snapshot = {"exists": False, "rootSubtreeControllers": controllers,
|
||||||
|
"safeToStartDind": True}
|
||||||
|
monkeypatch.setattr(specialize, "inspect_host_cgroup_family", lambda: snapshot)
|
||||||
|
read = specialize._cgroup_family_snapshot
|
||||||
|
expected = snapshot
|
||||||
|
error = "cgroup_inventory_invalid"
|
||||||
|
elif boundary == "ensure":
|
||||||
|
result = {"created": True, "rootSubtreeBefore": controllers}
|
||||||
|
monkeypatch.setattr(specialize, "ensure_host_cgroup_family", lambda **kwargs: result)
|
||||||
|
def read():
|
||||||
|
return specialize._ensure_cgroup_family_details(allow_existing=False)
|
||||||
|
expected = result
|
||||||
|
error = "cgroup_family_result_invalid"
|
||||||
|
else:
|
||||||
|
# Actual schema-v3 parser, including absent legacy-DinD state and its
|
||||||
|
# unchanged provenance, validates both historical and current inventories.
|
||||||
|
backup = specialize._SPECIALIZED_BACKUP_ROOT / "specialized-finite-fixture"
|
||||||
|
journal = {
|
||||||
|
"schemaVersion": 3, "status": "in_progress", "stage": "prepared",
|
||||||
|
"sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256,
|
||||||
|
"sourceRevision": "a" * 40, "sourceTree": "b" * 40,
|
||||||
|
"createdAt": "2026-10-10T00:00:00Z", "updatedAt": "2026-10-10T00:00:00Z",
|
||||||
|
"backupDirectory": str(backup), "baseComposeSha256": "c" * 64,
|
||||||
|
"baseComposeModelSha256": "d" * 64, "specializedComposeSha256": "e" * 64,
|
||||||
|
"legacyTopology": "v1", "originalRegistrations": [], "originalContainers": [],
|
||||||
|
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
||||||
|
"guardFiles": [], "guardMarkers": {}, "cgroupCreated": False,
|
||||||
|
"cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False,
|
||||||
|
"cgroupRootSubtree": controllers, "newRegistrationIds": {},
|
||||||
|
"imageManifestSha256": None, "buildProbe": None,
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(specialize, "_read_json_file", lambda *args, **kwargs: journal)
|
||||||
|
monkeypatch.setattr(specialize, "_trusted_directory", lambda path: None)
|
||||||
|
read = specialize._read_journal
|
||||||
|
expected = journal
|
||||||
|
error = "journal_invalid"
|
||||||
|
if accepted:
|
||||||
|
assert read() == expected
|
||||||
|
else:
|
||||||
|
with pytest.raises(specialize.RunnerFleetSpecializeError, match=error):
|
||||||
|
read()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("suffix", ["cgroup_inventory_invalid", "cgroup_inventory_unavailable",
|
||||||
|
"identity_adoption_rejected", "legacy_config_missing"])
|
||||||
|
def test_new_finite_prejournal_failure_codes_cross_both_boundaries(specialize, monkeypatch, capsys, suffix):
|
||||||
|
code = "runner_fleet_specialization_" + suffix
|
||||||
|
def fail():
|
||||||
|
raise specialize.RunnerFleetSpecializeError(code)
|
||||||
|
monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail)
|
||||||
|
assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1
|
||||||
|
captured = capsys.readouterr()
|
||||||
|
assert captured.out == ""
|
||||||
|
result = SimpleNamespace(returncode=1, stdout=captured.out, stderr=captured.err)
|
||||||
|
assert specialize._script_failure_code(result) == code
|
||||||
|
assert specialize.public_specialization_failure(specialize.RunnerFleetSpecializeError(code)) == code
|
||||||
@@ -87,6 +87,25 @@ def test_pending_protected_validation_blocks_specialization(controller, monkeypa
|
|||||||
assert responses[0][1]["status"] == "preflight_rejected"
|
assert responses[0][1]["status"] == "preflight_rejected"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("private, expected", [
|
||||||
|
("runner_fleet_specialization_entrypoint_untrusted", "runner_fleet_specialization_entrypoint_untrusted"),
|
||||||
|
("private-token-value", "runner_fleet_specialization_script_execution_failed"),
|
||||||
|
("runner_fleet_specialization_entrypoint_untrusted\nprivate-token-value",
|
||||||
|
"runner_fleet_specialization_script_execution_failed"),
|
||||||
|
])
|
||||||
|
def test_specialization_failure_response_contains_only_finite_safe_reason(controller, monkeypatch, private, expected):
|
||||||
|
instance, responses, _ = handler(controller, monkeypatch)
|
||||||
|
monkeypatch.setattr(controller.subprocess, "run", lambda *args, **kwargs:
|
||||||
|
SimpleNamespace(returncode=0, stdout=b"exact source script"))
|
||||||
|
|
||||||
|
def fail(_script):
|
||||||
|
raise controller.RunnerFleetSpecializeError(private)
|
||||||
|
|
||||||
|
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", fail)
|
||||||
|
instance.do_POST()
|
||||||
|
assert responses == [(409, {"status": "upgrade_failed", "revision": "a" * 40, "reason": expected})]
|
||||||
|
|
||||||
|
|
||||||
def test_caller_cannot_choose_script_image_or_runner(controller, monkeypatch):
|
def test_caller_cannot_choose_script_image_or_runner(controller, monkeypatch):
|
||||||
instance, responses, _ = handler(controller, monkeypatch, {"reason": "Specialize CI profiles", "script": "untrusted.sh"})
|
instance, responses, _ = handler(controller, monkeypatch, {"reason": "Specialize CI profiles", "script": "untrusted.sh"})
|
||||||
instance.do_POST()
|
instance.do_POST()
|
||||||
|
|||||||
Reference in new issue
Block a user