Publish curated Unraid-AppOps-Gateway source from b2bb670dd165f2979e421ffc34de833438b2fd85
This commit is contained in:
Public Source Publisher committed
1 parent
5a3941a286
commit
7ee376bdbb
4 files changed
+543
-6
No files matched your search
@@ -1,3 +1,3 @@
|
||||
# Public source export
|
||||
|
||||
This source snapshot was generated from the private canonical repository at revision `1a5159c66078983299a1c2acb8837e4b5a8a607a`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
This source snapshot was generated from the private canonical repository at revision `b2bb670dd165f2979e421ffc34de833438b2fd85`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"sourceRevision": "1a5159c66078983299a1c2acb8837e4b5a8a607a",
|
||||
"sourceRevision": "b2bb670dd165f2979e421ffc34de833438b2fd85",
|
||||
"files": [
|
||||
{
|
||||
"path": ".dockerignore",
|
||||
@@ -1644,14 +1644,19 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_fleet_specialize.py",
|
||||
"sha256": "6771e7ae22ddb0d39e19272d311f9c77b898273c7106eef02ba0c76fc6e48873",
|
||||
"bytes": 119270
|
||||
"sha256": "7cdb734b4596f2bdbd4703166792808a0183985c7165d9d8c51bdb54119f83d9",
|
||||
"bytes": 145449
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_fleet_v2_execution_surface.py",
|
||||
"sha256": "62dcc692085ea9af004c4618e0c770701396be02254daf06bd5b55f56b5119fa",
|
||||
"bytes": 2337
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_guard_shell_syntax.py",
|
||||
"sha256": "878fe58ff4dbb696ce6234b403f08f003a22881dba394cc2de1fbddc307c36ef",
|
||||
"bytes": 3018
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_profiles.py",
|
||||
"sha256": "45a8dba20d1e8d778dc90c20a997630037e2eaa1d462fa62f48671285aa82e0e",
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from contextlib import nullcontext
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -96,6 +99,7 @@ def test_legacy_live_source_does_not_open_promotion_backup(specialize, monkeypat
|
||||
def test_controller_compose_cas_blocks_rollback_before_any_lifecycle_action(
|
||||
specialize, controller_only_source, monkeypatch, separate_controller_hash,
|
||||
):
|
||||
monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance)
|
||||
model, compose, _archive, _archived = controller_only_source
|
||||
compose.write_text(json.dumps(model))
|
||||
original_sha = hashlib.sha256(compose.read_bytes()).hexdigest()
|
||||
@@ -136,6 +140,11 @@ def specialize(monkeypatch):
|
||||
return module
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _fixture_rollback_maintenance():
|
||||
yield lambda: None
|
||||
|
||||
|
||||
def _valid_task_image_manifest(specialize, revision="a" * 40, tree="b" * 40):
|
||||
source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway"
|
||||
return {
|
||||
@@ -1403,8 +1412,16 @@ def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_
|
||||
monkeypatch.setattr(specialize, "RUNNER_GUARD_SOURCE_SHA256", runner_sha)
|
||||
monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", host_sha)
|
||||
monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "")
|
||||
monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None)
|
||||
monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None)
|
||||
backup = tmp_path / "guard-backup"
|
||||
backup.mkdir(mode=0o700)
|
||||
for path, name in ((runner, "gitea-runner-guard.sh"), (host, "host-resource-guard.sh")):
|
||||
target = backup / name
|
||||
target.write_bytes(path.read_bytes())
|
||||
target.chmod(0o600)
|
||||
journal = {
|
||||
"backupDirectory": str(backup),
|
||||
"guardFiles": [
|
||||
{"path": str(runner), "originalSha256": runner_sha, "installedSha256": None},
|
||||
{"path": str(host), "originalSha256": host_sha, "installedSha256": None},
|
||||
@@ -1424,6 +1441,432 @@ def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_
|
||||
assert host.read_text(encoding="utf-8") == "operator-owned guard revision\n"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def guard_transaction(specialize, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance)
|
||||
runner = tmp_path / "runner-guard"
|
||||
host = tmp_path / "host-guard"
|
||||
originals = (
|
||||
(b'#!/bin/bash\nstack_recovery=0\nset_managed_runners_disabled false\nunexpected=0\n'
|
||||
b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n'),
|
||||
(b'#!/bin/bash\n# ITWORX_RUNNER_CAPACITY_V2_BEGIN\nold capacity writer\n'
|
||||
b'# ITWORX_RUNNER_CAPACITY_V2_END\nrunner_healthy=1\n'
|
||||
b' if [ "$sandbox_healthy" -eq 1 ] && \\\n'
|
||||
b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n'
|
||||
b' recover legacy stack\n fi\n'),
|
||||
)
|
||||
backup = tmp_path / "backups"
|
||||
backup.mkdir(mode=0o700)
|
||||
for path, original, name in zip((runner, host), originals, ("gitea-runner-guard.sh", "host-resource-guard.sh"), strict=True):
|
||||
path.write_bytes(original)
|
||||
target = backup / name
|
||||
target.write_bytes(original)
|
||||
target.chmod(0o600)
|
||||
monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner)
|
||||
monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host)
|
||||
for name, original in zip(("RUNNER_GUARD_SOURCE_SHA256", "HOST_GUARD_SOURCE_SHA256"), originals, strict=True):
|
||||
monkeypatch.setattr(specialize, name, hashlib.sha256(original).hexdigest())
|
||||
journal = {"backupDirectory": str(backup), "guardFiles": [
|
||||
{"path": str(path), "originalSha256": hashlib.sha256(original).hexdigest(), "installedSha256": None}
|
||||
for path, original in zip((runner, host), originals, strict=True)
|
||||
]}
|
||||
monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None)
|
||||
monkeypatch.setattr(specialize, "_set_stage", lambda _journal, _stage: None)
|
||||
monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None)
|
||||
return journal, runner, host, originals
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", [0, 1])
|
||||
def test_both_candidate_syntax_checks_precede_any_intent_or_guard_write(specialize, guard_transaction, monkeypatch, invalid):
|
||||
journal, runner, host, originals = guard_transaction
|
||||
checked = []
|
||||
def syntax(encoded):
|
||||
checked.append(encoded)
|
||||
if len(checked) == invalid + 1:
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_guard_syntax_invalid")
|
||||
monkeypatch.setattr(specialize, "_validate_guard_syntax", syntax)
|
||||
monkeypatch.setattr(specialize, "_write_journal", lambda _journal: pytest.fail("invalid candidate published intent"))
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("invalid candidate installed"))
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_syntax_invalid"):
|
||||
specialize._service_guard_scripts(journal)
|
||||
assert (runner.read_bytes(), host.read_bytes()) == originals
|
||||
assert all(row["installedSha256"] is None for row in journal["guardFiles"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("boundary", ["intent_failed", "after_intent", "after_runner", "after_host"])
|
||||
def test_guard_installation_durable_crash_boundaries_restore_only_owned_bytes(specialize, guard_transaction, monkeypatch, boundary):
|
||||
journal, runner, host, originals = guard_transaction
|
||||
persisted = []
|
||||
real_write = specialize._atomic_write_owned
|
||||
writes = []
|
||||
def journal_write(value):
|
||||
assert all(row["installedSha256"] for row in value["guardFiles"])
|
||||
assert (runner.read_bytes(), host.read_bytes()) == originals
|
||||
if boundary == "intent_failed":
|
||||
raise OSError("private canary write failure")
|
||||
persisted.append(copy.deepcopy(value))
|
||||
if boundary == "after_intent":
|
||||
raise OSError("private canary crash after intent")
|
||||
def install(path, encoded, **kwargs):
|
||||
assert persisted and persisted[0]["guardFiles"] == journal["guardFiles"]
|
||||
real_write(path, encoded, **kwargs)
|
||||
writes.append(path)
|
||||
if boundary == "after_runner" or boundary == "after_host" and len(writes) == 2:
|
||||
raise OSError("private canary crash after replace")
|
||||
monkeypatch.setattr(specialize, "_write_journal", journal_write)
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", install)
|
||||
with pytest.raises(OSError):
|
||||
specialize._service_guard_scripts(journal)
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", real_write)
|
||||
specialize._restore_guard_scripts(persisted[0] if persisted else journal)
|
||||
assert (runner.read_bytes(), host.read_bytes()) == originals
|
||||
specialize._restore_guard_scripts(journal)
|
||||
assert (runner.read_bytes(), host.read_bytes()) == originals
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["duplicate", "foreign_path", "original_hash", "planned_hash", "backup", "foreign_edit"])
|
||||
def test_guard_restore_rejects_foreign_journal_backup_or_live_bytes_before_any_write(specialize, guard_transaction, monkeypatch, mutation):
|
||||
journal, runner, host, _originals = guard_transaction
|
||||
specialize._service_guard_scripts(journal)
|
||||
if mutation == "duplicate":
|
||||
journal["guardFiles"][1] = copy.deepcopy(journal["guardFiles"][0])
|
||||
elif mutation == "foreign_path":
|
||||
journal["guardFiles"][1]["path"] = str(host.with_name("foreign"))
|
||||
elif mutation == "original_hash":
|
||||
journal["guardFiles"][0]["originalSha256"] = "f" * 64
|
||||
elif mutation == "planned_hash":
|
||||
journal["guardFiles"][0]["installedSha256"] = "f" * 64
|
||||
elif mutation == "backup":
|
||||
(Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign private canary")
|
||||
else:
|
||||
host.write_bytes(b"foreign private canary")
|
||||
before = runner.read_bytes(), host.read_bytes()
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("foreign state restored"))
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError):
|
||||
specialize._restore_guard_scripts(journal)
|
||||
assert (runner.read_bytes(), host.read_bytes()) == before
|
||||
|
||||
|
||||
def test_guard_syntax_checker_never_exposes_bash_output(specialize, monkeypatch):
|
||||
def parse(args, **kwargs):
|
||||
assert args == ["bash", "-n"] and kwargs["input"] == b"candidate"
|
||||
assert kwargs["timeout"] == 15 and kwargs["capture_output"] is True
|
||||
return SimpleNamespace(returncode=2, stdout=b"private canary", stderr=b"private canary")
|
||||
monkeypatch.setattr(specialize.subprocess, "run", parse)
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError, match="^runner_fleet_specialization_guard_syntax_invalid$"):
|
||||
specialize._validate_guard_syntax(b"candidate")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", [None, "foreign", "begin_only", "end_only", "repeated"])
|
||||
def test_host_guard_without_capacity_stanza_requires_exact_approved_bytes(specialize, guard_transaction, monkeypatch, mutation):
|
||||
_journal, _runner, _host, originals = guard_transaction
|
||||
text = originals[1].decode()
|
||||
start = text.index("# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n")
|
||||
end = text.index("# ITWORX_RUNNER_CAPACITY_V2_END\n") + len("# ITWORX_RUNNER_CAPACITY_V2_END\n")
|
||||
approved = text[:start] + text[end:]
|
||||
monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", hashlib.sha256(approved.encode()).hexdigest())
|
||||
if mutation == "foreign":
|
||||
approved += "# operator edit\n"
|
||||
elif mutation == "begin_only":
|
||||
approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n"
|
||||
elif mutation == "end_only":
|
||||
approved += "# ITWORX_RUNNER_CAPACITY_V2_END\n"
|
||||
elif mutation == "repeated":
|
||||
approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n# ITWORX_RUNNER_CAPACITY_V2_END\n" * 2
|
||||
if mutation is None:
|
||||
result = specialize._patch_host_guard(approved, "fixed --state")
|
||||
assert 'specialization_state="$(fixed --state' in result
|
||||
assert '"$specialization_state" != "legacy"' in result
|
||||
else:
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError, match="capacity_block_invalid"):
|
||||
specialize._patch_host_guard(approved, "fixed --state")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["path", "backup", "foreign_edit"])
|
||||
def test_guard_rollback_refusal_preserves_marker_journal_before_lifecycle_effects(specialize, guard_transaction, monkeypatch, mutation):
|
||||
journal, runner, host, _originals = guard_transaction
|
||||
specialize._service_guard_scripts(journal)
|
||||
live_compose = runner.with_name("compose")
|
||||
live_compose.write_bytes(b"fixed compose")
|
||||
monkeypatch.setattr(specialize, "COMPOSE_PATH", live_compose)
|
||||
journal["baseComposeSha256"] = hashlib.sha256(live_compose.read_bytes()).hexdigest()
|
||||
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
|
||||
monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: pytest.fail("refusal reached lifecycle effects"))
|
||||
marker, durable = runner.with_name("marker"), runner.with_name("journal")
|
||||
marker.write_bytes(b"owned marker")
|
||||
durable.write_bytes(b"owned journal")
|
||||
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker)
|
||||
monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable)
|
||||
if mutation == "path":
|
||||
journal["guardFiles"][0]["path"] = str(runner.with_name("foreign"))
|
||||
elif mutation == "backup":
|
||||
(Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign canary")
|
||||
elif mutation == "foreign_edit":
|
||||
host.write_bytes(b"foreign canary")
|
||||
assert specialize._rollback_before_activation(journal) is False
|
||||
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["cleanup", "restore_cas", None])
|
||||
def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, guard_transaction, monkeypatch, failure):
|
||||
journal, runner, host, originals = guard_transaction
|
||||
specialize._service_guard_scripts(journal)
|
||||
fenced = runner.read_bytes(), host.read_bytes()
|
||||
compose, marker, durable = runner.with_name("compose"), runner.with_name("marker"), runner.with_name("journal")
|
||||
compose.write_bytes(b"fixed compose")
|
||||
marker.write_bytes(b"owned marker")
|
||||
durable.write_bytes(b"owned journal")
|
||||
monkeypatch.setattr(specialize, "COMPOSE_PATH", compose)
|
||||
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker)
|
||||
monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable)
|
||||
journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(),
|
||||
originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
||||
originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupRootSubtree=[])
|
||||
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
|
||||
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose")
|
||||
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
|
||||
monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: [])
|
||||
monkeypatch.setattr(specialize, "_inspect", lambda _name: None)
|
||||
events = []
|
||||
def cleanup(_journal):
|
||||
assert (runner.read_bytes(), host.read_bytes()) == fenced
|
||||
events.append("cleanup")
|
||||
if failure == "cleanup":
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_command_failed")
|
||||
monkeypatch.setattr(specialize, "_cleanup_new_containers", cleanup)
|
||||
def cgroup(**_kwargs):
|
||||
assert (runner.read_bytes(), host.read_bytes()) == fenced
|
||||
events.append("cgroup")
|
||||
return {"familyExists": False}
|
||||
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup)
|
||||
write = specialize._atomic_write_owned
|
||||
def restore(*args, **kwargs):
|
||||
assert events == ["cleanup", "cgroup"]
|
||||
assert marker.exists() and durable.exists()
|
||||
if failure == "restore_cas":
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed")
|
||||
write(*args, **kwargs)
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", restore)
|
||||
assert specialize._rollback_before_activation(journal) is (failure is None)
|
||||
if failure:
|
||||
assert (runner.read_bytes(), host.read_bytes()) == fenced
|
||||
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
|
||||
else:
|
||||
assert (runner.read_bytes(), host.read_bytes()) == originals
|
||||
assert not marker.exists() and not durable.exists()
|
||||
|
||||
|
||||
def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch):
|
||||
marker, journal = tmp_path / "marker", tmp_path / "journal"
|
||||
marker.write_bytes(b"owned marker")
|
||||
journal.write_bytes(b"owned journal")
|
||||
@contextmanager
|
||||
def busy():
|
||||
raise RuntimeError("private canary lock refusal")
|
||||
yield
|
||||
monkeypatch.setattr(specialize, "_rollback_maintenance", busy)
|
||||
monkeypatch.setattr(specialize, "_rollback_before_activation_locked", lambda *_args: pytest.fail("busy barrier reached effects"))
|
||||
assert specialize._rollback_before_activation({}) is False
|
||||
assert marker.read_bytes() == b"owned marker" and journal.read_bytes() == b"owned journal"
|
||||
|
||||
|
||||
def test_rollback_barrier_drift_refuses_before_guards_or_cleanup(specialize, guard_transaction, monkeypatch):
|
||||
journal, _runner, _host, _originals = guard_transaction
|
||||
@contextmanager
|
||||
def changed():
|
||||
def refuse():
|
||||
raise RuntimeError("private canary changed inode")
|
||||
yield refuse
|
||||
monkeypatch.setattr(specialize, "_rollback_maintenance", changed)
|
||||
monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: pytest.fail("drift restored guard"))
|
||||
monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda *_args: pytest.fail("drift changed fleet"))
|
||||
assert specialize._rollback_before_activation(journal) is False
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="real common flock contract requires POSIX")
|
||||
def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_finishes(specialize, tmp_path, monkeypatch):
|
||||
if os.geteuid() != 0:
|
||||
pytest.skip("protected maintenance lock requires root")
|
||||
import runner_acceptance_quiescence as quiescence
|
||||
lock = tmp_path / "maintenance.lock"
|
||||
lock.touch(mode=0o600)
|
||||
monkeypatch.setattr(quiescence, "MAINTENANCE_LOCK", lock)
|
||||
monkeypatch.setattr(quiescence, "_LOCK_PROBE", quiescence._LOCK_PROBE.replace(
|
||||
"/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock", str(lock)))
|
||||
monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=dict))
|
||||
probe = (
|
||||
"import fcntl,os,sys\nfd=os.open(sys.argv[1],os.O_RDWR|os.O_NOFOLLOW)\n"
|
||||
"try: fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)\n"
|
||||
"except BlockingIOError: sys.exit(0)\n"
|
||||
"sys.exit(1)\n"
|
||||
)
|
||||
events = []
|
||||
def held(phase):
|
||||
result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False)
|
||||
assert result.returncode == 0 and not result.stdout and not result.stderr
|
||||
events.append(phase)
|
||||
compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal"
|
||||
compose.write_bytes(b"captured compose")
|
||||
marker.write_bytes(b"owned marker")
|
||||
durable.write_bytes(b"owned journal")
|
||||
monkeypatch.setattr(specialize, "COMPOSE_PATH", compose)
|
||||
monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker)
|
||||
monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable)
|
||||
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
|
||||
monkeypatch.setattr(specialize, "_guard_script_plan", lambda _journal: [])
|
||||
monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: held("guards-restored"))
|
||||
monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: held("cleanup"))
|
||||
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"captured compose")
|
||||
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
|
||||
monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: [])
|
||||
monkeypatch.setattr(specialize, "_inspect", lambda _name: None)
|
||||
def cgroup(**_kwargs):
|
||||
held("cgroup")
|
||||
assert marker.exists() and durable.exists()
|
||||
return {"familyExists": False}
|
||||
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup)
|
||||
journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}],
|
||||
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
||||
"originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupRootSubtree": []}
|
||||
assert specialize._rollback_before_activation(journal) is True
|
||||
assert events == ["cleanup", "cgroup", "guards-restored"]
|
||||
assert not marker.exists() and not durable.exists()
|
||||
# The independent participant can acquire only after the wrapper returns.
|
||||
result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False)
|
||||
assert result.returncode == 1 and not result.stdout and not result.stderr
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def owned_orphan(specialize, guard_transaction, tmp_path, monkeypatch):
|
||||
import runner_acceptance_quiescence as quiescence
|
||||
journal, runner, host, originals = guard_transaction
|
||||
corrected = specialize._guard_script_plan(journal)[0][3]
|
||||
closing = f'\nfi\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode()
|
||||
orphan = corrected.replace(closing, f'\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode(), 1)
|
||||
runner.write_bytes(orphan)
|
||||
monkeypatch.setattr(specialize, "_OWNED_MALFORMED_RUNNER_GUARD_SHA256", hashlib.sha256(orphan).hexdigest())
|
||||
receipt = {
|
||||
"schema": quiescence.SCHEMA, "revision": "0ea6f267c28d51ac7113e6d943c89857d3a189ab",
|
||||
"manifestDigest": quiescence._digest(quiescence.OBSERVATION), "phase": "complete",
|
||||
"progress": {row["id"]: "quiesced" for row in quiescence._fixed_rows()},
|
||||
"registrationProgress": {"9": "restored", "22": "restored"},
|
||||
"guardProgress": {"runner": "restored", "host": "restored"},
|
||||
"guards": {name: {
|
||||
"original": base64.b64encode(original).decode(), "originalSha256": hashlib.sha256(original).hexdigest(),
|
||||
"wrappedSha256": hashlib.sha256(quiescence.ControllerDriver._wrapped_guard(original)).hexdigest(),
|
||||
"mode": path.stat().st_mode & 0o777,
|
||||
} for name, path, original in zip(("runner", "host"), (runner, host), originals, strict=True)},
|
||||
"baseline": {"rows": [{"state": {"Running": False}} for _row in quiescence._fixed_rows()]},
|
||||
}
|
||||
events = []
|
||||
for name in ("SPECIALIZED_STATE_PATH", "MIGRATION_JOURNAL_PATH"):
|
||||
monkeypatch.setattr(specialize, name, tmp_path / name)
|
||||
monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: False)
|
||||
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
|
||||
monkeypatch.setattr(specialize, "_read_fleet_base_compose", lambda: (b"source", None))
|
||||
monkeypatch.setattr(specialize, "_validate_base_composition", lambda _source: "v1")
|
||||
monkeypatch.setattr(specialize, "_runner_rows", lambda: [{"id": 9}])
|
||||
monkeypatch.setattr(specialize, "_validate_legacy_registrations", lambda *_args: [{"id": 9}])
|
||||
monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: ("v1", [{"id": "fixed"}]))
|
||||
monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=lambda: {"fixed": "registry"}))
|
||||
monkeypatch.setattr(quiescence, "_read_receipt", lambda: copy.deepcopy(receipt))
|
||||
monkeypatch.setattr(quiescence, "_idle", lambda *_args: events.append("idle"))
|
||||
def attest(snapshot, *, baseline, progress):
|
||||
assert baseline == receipt["baseline"] and progress == receipt["progress"]
|
||||
assert snapshot["rows"] == baseline["rows"]
|
||||
events.append("attested")
|
||||
monkeypatch.setattr(quiescence, "attest_snapshot", attest)
|
||||
class Driver:
|
||||
_wrapped_guard = staticmethod(quiescence.ControllerDriver._wrapped_guard)
|
||||
def __init__(self, projects):
|
||||
assert projects == {"fixed": "registry"}
|
||||
@contextmanager
|
||||
def maintenance(self):
|
||||
with (tmp_path / "fixture-lock").open("wb") as lock:
|
||||
self._maintenance_fd = lock.fileno()
|
||||
events.append("locked")
|
||||
yield
|
||||
events.append("unlocked")
|
||||
def assert_barrier(self):
|
||||
events.append("barrier")
|
||||
def snapshot(self):
|
||||
self._guard_binding()
|
||||
events.append("snapshot")
|
||||
return copy.deepcopy(receipt["baseline"])
|
||||
monkeypatch.setattr(quiescence, "ControllerDriver", Driver)
|
||||
return quiescence, receipt, runner, host, originals, orphan, events, Driver
|
||||
|
||||
|
||||
def test_owned_orphan_recovery_uses_full_repeated_admission_and_cas_then_is_idempotent(specialize, owned_orphan, monkeypatch):
|
||||
_q, receipt, runner, host, originals, orphan, events, _driver = owned_orphan
|
||||
before = copy.deepcopy(receipt)
|
||||
write = specialize._atomic_write_owned
|
||||
def restore(path, data, **kwargs):
|
||||
assert events.count("attested") == 2 and events.count("idle") == 3
|
||||
assert path == runner and data == originals[0]
|
||||
assert kwargs["expected_sha256"] == hashlib.sha256(orphan).hexdigest()
|
||||
events.append("restore")
|
||||
write(path, data, **kwargs)
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", restore)
|
||||
specialize._recover_owned_orphan_guard()
|
||||
assert (runner.read_bytes(), host.read_bytes()) == originals and receipt == before
|
||||
assert events.index("locked") < events.index("restore") < events.index("unlocked")
|
||||
specialize._recover_owned_orphan_guard()
|
||||
assert events.count("restore") == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["revision", "manifest", "phase", "progress", "registration", "guard_progress",
|
||||
"original", "wrapped", "foreign_runner", "foreign_host", "marker", "journal",
|
||||
"storage", "family", "idle", "lock", "origin", "changed_receipt", "container", "cas"])
|
||||
def test_orphan_recovery_refuses_unowned_or_changed_state_without_write(specialize, owned_orphan, monkeypatch, mutation):
|
||||
q, receipt, runner, host, _originals, _orphan, _events, driver = owned_orphan
|
||||
if mutation in {"revision", "manifest", "phase"}:
|
||||
receipt[{"revision": "revision", "manifest": "manifestDigest", "phase": "phase"}[mutation]] = "foreign"
|
||||
elif mutation == "progress":
|
||||
receipt["progress"].pop(next(iter(receipt["progress"])))
|
||||
elif mutation == "registration":
|
||||
receipt["registrationProgress"]["9"] = "disabled"
|
||||
elif mutation == "guard_progress":
|
||||
receipt["guardProgress"]["host"] = "installed"
|
||||
elif mutation in {"original", "wrapped"}:
|
||||
receipt["guards"]["runner"]["originalSha256" if mutation == "original" else "wrappedSha256"] = "f" * 64
|
||||
elif mutation in {"foreign_runner", "foreign_host"}:
|
||||
(runner if mutation == "foreign_runner" else host).write_bytes(b"private foreign canary")
|
||||
elif mutation in {"marker", "journal"}:
|
||||
(specialize.SPECIALIZED_STATE_PATH if mutation == "marker" else specialize.MIGRATION_JOURNAL_PATH).touch()
|
||||
elif mutation == "storage":
|
||||
monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: True)
|
||||
elif mutation == "family":
|
||||
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": True})
|
||||
elif mutation == "idle":
|
||||
monkeypatch.setattr(q, "_idle", lambda *_args: (_ for _ in ()).throw(q.QuiescenceError("global_ci_work_present")))
|
||||
elif mutation == "lock":
|
||||
monkeypatch.setattr(driver, "assert_barrier", lambda _self: (_ for _ in ()).throw(q.QuiescenceError("dispatch_maintenance_busy")))
|
||||
elif mutation == "origin":
|
||||
monkeypatch.setattr(q, "attest_snapshot", lambda *_args, **_kwargs: (_ for _ in ()).throw(q.QuiescenceError("origin_changed")))
|
||||
elif mutation == "changed_receipt":
|
||||
reads = []
|
||||
def changed():
|
||||
reads.append(True)
|
||||
result = copy.deepcopy(receipt)
|
||||
if len(reads) > 1:
|
||||
result["revision"] = "foreign"
|
||||
return result
|
||||
monkeypatch.setattr(q, "_read_receipt", changed)
|
||||
elif mutation == "container":
|
||||
reads = []
|
||||
monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: reads.append(True) or ("v1", [{"id": str(len(reads))}]))
|
||||
else:
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed")))
|
||||
before = runner.read_bytes(), host.read_bytes()
|
||||
if mutation != "cas":
|
||||
monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("unowned orphan restored"))
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError):
|
||||
specialize._recover_owned_orphan_guard()
|
||||
assert (runner.read_bytes(), host.read_bytes()) == before
|
||||
|
||||
|
||||
def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(specialize, monkeypatch):
|
||||
script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes()
|
||||
digest = hashlib.sha256(script).hexdigest()
|
||||
@@ -2180,6 +2623,7 @@ def test_specialization_stage_fixture_quiesces_old_before_registering_and_never_
|
||||
def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup(
|
||||
specialize, tmp_path, monkeypatch, original_running
|
||||
):
|
||||
monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance)
|
||||
image_id = specialize._DIND_IMAGE_ID
|
||||
base_model = _base_v1_compose_model(specialize)
|
||||
specialized_model = {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Validate generated guard programs with Bash's actual parser, without effects."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
CONTROLLER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
|
||||
RUNNER = '''#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
readonly DEPLOY_RUNNER="deployment-runner"
|
||||
stack_recovery=0
|
||||
if [ "$stack_recovery" -eq 0 ]; then
|
||||
stack_recovery=1
|
||||
fi
|
||||
unexpected=0
|
||||
for name in validation-runner; do
|
||||
[ "$name" = "$DEPLOY_RUNNER" ] && continue
|
||||
unexpected=1
|
||||
done
|
||||
'''
|
||||
HOST = '''#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# ITWORX_RUNNER_CAPACITY_V2_BEGIN
|
||||
capacity_active=true
|
||||
# ITWORX_RUNNER_CAPACITY_V2_END
|
||||
runner_healthy=1
|
||||
if [ "$runner_healthy" -eq 1 ]; then
|
||||
if [ "$sandbox_healthy" -eq 1 ] && \\
|
||||
timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then
|
||||
:
|
||||
fi
|
||||
fi
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fleet(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"runner_guard_shell_contract", CONTROLLER / "runner_fleet_specialize.py",
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="Real Bash guard validation requires POSIX")
|
||||
@pytest.mark.parametrize("kind", ["runner", "host", "host_legacy"])
|
||||
def test_complete_generated_guard_passes_real_bash_parser(fleet, kind, monkeypatch):
|
||||
bash = shutil.which("bash")
|
||||
assert bash is not None, "The deployment qualification requires Bash"
|
||||
state_command = "timeout 15 /fixed/controller --state"
|
||||
if kind == "runner":
|
||||
original = RUNNER
|
||||
generated = fleet._patch_runner_guard(
|
||||
original, state_command,
|
||||
'timeout 10 /fixed/controller --guard-allows-container "$name"',
|
||||
)
|
||||
else:
|
||||
original = HOST
|
||||
if kind == "host_legacy":
|
||||
original = original.replace(
|
||||
"# ITWORX_RUNNER_CAPACITY_V2_BEGIN\ncapacity_active=true\n"
|
||||
"# ITWORX_RUNNER_CAPACITY_V2_END\n", "",
|
||||
)
|
||||
# This isolated fixture represents a single approved legacy source.
|
||||
# Live acceptance separately binds the actual unchanged guard hash.
|
||||
monkeypatch.setattr(
|
||||
fleet, "HOST_GUARD_SOURCE_SHA256",
|
||||
hashlib.sha256(original.encode("utf-8")).hexdigest(),
|
||||
)
|
||||
generated = fleet._patch_host_guard(original, state_command)
|
||||
for program in (original, generated):
|
||||
# Parse stdin only. No shell program, controller, Docker or guard runs.
|
||||
result = subprocess.run(
|
||||
[bash, "-n"], input=program.encode("utf-8"),
|
||||
capture_output=True, timeout=5, check=False,
|
||||
)
|
||||
assert result.returncode == 0, "Generated guard failed real Bash syntax validation"
|
||||
Reference in new issue
Block a user