Publish curated Unraid-AppOps-Gateway source from 87aa7dda6889fd951d202754d76daaaa1e98e26b
This commit is contained in:
1 parent
07bb0bbb90
commit
ad3a9736cc
5 files changed
+891
-6
No files matched your search
@@ -1,3 +1,3 @@
|
|||||||
# Public source export
|
# Public source export
|
||||||
|
|
||||||
This source snapshot was generated from the private canonical repository at revision `5572962b172902d61bdd220ceb8fb69426f17b5f`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
This source snapshot was generated from the private canonical repository at revision `87aa7dda6889fd951d202754d76daaaa1e98e26b`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"schemaVersion": 1,
|
"schemaVersion": 1,
|
||||||
"sourceRevision": "5572962b172902d61bdd220ceb8fb69426f17b5f",
|
"sourceRevision": "87aa7dda6889fd951d202754d76daaaa1e98e26b",
|
||||||
"files": [
|
"files": [
|
||||||
{
|
{
|
||||||
"path": ".dockerignore",
|
"path": ".dockerignore",
|
||||||
@@ -1627,6 +1627,16 @@
|
|||||||
"sha256": "ec3a53cebab1fae3a68817798d72b2831f35eda4c46fbb3d07ad84fc5eb55ab3",
|
"sha256": "ec3a53cebab1fae3a68817798d72b2831f35eda4c46fbb3d07ad84fc5eb55ab3",
|
||||||
"bytes": 2830
|
"bytes": 2830
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_runner_dind_health_cutover_recovery.py",
|
||||||
|
"sha256": "ae087c30252ba9c864d8ddd3a3063feb0d2870fb549f747eaba956d95b3304eb",
|
||||||
|
"bytes": 13128
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_runner_dind_rollback_tail_recovery.py",
|
||||||
|
"sha256": "62538fc576fc8cae56fd2c939892bcbc1360800f3276ccdc78a493697c87380e",
|
||||||
|
"bytes": 14095
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_failed_cutover_recovery.py",
|
"path": "backend/tests/test_runner_failed_cutover_recovery.py",
|
||||||
"sha256": "18ca28b3e218fc4b4d8069b7b51fb2195b95ee96386bdc5fa075ed610f168528",
|
"sha256": "18ca28b3e218fc4b4d8069b7b51fb2195b95ee96386bdc5fa075ed610f168528",
|
||||||
@@ -1649,8 +1659,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_fleet_specialize.py",
|
"path": "backend/tests/test_runner_fleet_specialize.py",
|
||||||
"sha256": "a79cf1f123284ef501fbe03363d0cc1b046df519442af1e63702e6cf8201ccb9",
|
"sha256": "8d75435eb4f7b4adb52ffaff7bbb234941c382c68a4e71bdbbc3abc177aa279a",
|
||||||
"bytes": 167920
|
"bytes": 186243
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_runner_fleet_v2_execution_surface.py",
|
"path": "backend/tests/test_runner_fleet_v2_execution_surface.py",
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
BROKER = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"
|
||||||
|
|
||||||
|
|
||||||
|
def load(name, path):
|
||||||
|
spec = importlib.util.spec_from_file_location(name, path)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def recovery(monkeypatch):
|
||||||
|
monkeypatch.syspath_prepend(str(BROKER))
|
||||||
|
return load("dind_health_recovery_test", BROKER / "runner_dind_health_cutover_recovery.py")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def owned(recovery, monkeypatch, tmp_path):
|
||||||
|
shared = load("dind_recovery_fixture", Path(__file__).with_name("test_runner_failed_cutover_recovery.py"))
|
||||||
|
monkeypatch.setattr(recovery, "PROBE_ID", "a" * 64, raising=False)
|
||||||
|
values = shared.owned.__wrapped__(recovery, monkeypatch, tmp_path)
|
||||||
|
fleet, _, q, driver, marker, journal, _, rows, dind, events, pending = values
|
||||||
|
journal.update(stage="registrations_created", cgroupCreated=True, cgroupEnsurePending=False,
|
||||||
|
newRegistrationIds=dict(recovery.REGISTRATIONS), backupDirectory=recovery.BACKUP)
|
||||||
|
for key, profile in fleet.RUNNER_PROFILES.items():
|
||||||
|
rows.append({"id": recovery.REGISTRATIONS[key], "name": profile.container_name,
|
||||||
|
"displayName": profile.display_name, "labels": sorted({profile.runner_label, *profile.legacy_labels}),
|
||||||
|
"disabled": True, "busy": False})
|
||||||
|
monkeypatch.setattr(fleet, "_validate_legacy_registrations", lambda current, _: [
|
||||||
|
{k: v for k, v in row.items() if k != "busy"} for row in current if row["id"] in (9, 22)])
|
||||||
|
manifest = fleet.TASK_IMAGE_MANIFEST_PATH.read_bytes()
|
||||||
|
sha = hashlib.sha256(manifest).hexdigest()
|
||||||
|
monkeypatch.setattr(recovery, "MANIFEST_SHA", sha)
|
||||||
|
marker["imageManifestSha256"] = journal["imageManifestSha256"] = sha
|
||||||
|
monkeypatch.setattr(recovery, "MARKER_SHA", hashlib.sha256(json.dumps(marker).encode()).hexdigest())
|
||||||
|
monkeypatch.setattr(recovery, "JOURNAL_SHA", hashlib.sha256(json.dumps(journal).encode()).hexdigest())
|
||||||
|
health = ["CMD-SHELL", "docker -H tcp://127.0.0.1:2375 info >/dev/null 2>&1"]
|
||||||
|
dind["Id"] = recovery.DIND_ID
|
||||||
|
dind["HostConfig"]["RestartPolicy"] = {"Name": "unless-stopped", "MaximumRetryCount": 0}
|
||||||
|
dind["Config"]["Healthcheck"] = {"Test": health}
|
||||||
|
dind["State"]["Health"]["Status"] = "unhealthy"
|
||||||
|
model = {"services": {"runner-dind": {"healthcheck": {"test": health}}}}
|
||||||
|
monkeypatch.setattr(fleet, "_captured_compose_mapping", lambda *a, **kw: copy.deepcopy(model))
|
||||||
|
expected = copy.deepcopy(dind)
|
||||||
|
expected["HostConfig"]["RestartPolicy"]["Name"] = "no"
|
||||||
|
def validate(item, _model):
|
||||||
|
return all(item.get(key) == expected[key] for key in ("Id", "Image", "Config", "HostConfig", "Mounts", "NetworkSettings"))
|
||||||
|
monkeypatch.setattr(fleet, "_validate_dind_outer", validate)
|
||||||
|
monkeypatch.setattr(fleet, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "absent")
|
||||||
|
monkeypatch.setattr(recovery, "_workspace_proof", lambda _: {"owned": "six-empty"})
|
||||||
|
monkeypatch.setattr(recovery, "_profile_state_proof", lambda _: {"private": "hash-only"})
|
||||||
|
monkeypatch.setattr(recovery, "_probe_absent", lambda: True)
|
||||||
|
monkeypatch.setattr(recovery, "__file__", str(recovery.BUNDLE / "runner_dind_health_cutover_recovery.py"))
|
||||||
|
def native(args, **kw):
|
||||||
|
assert args[3].startswith("DOCKER_HOST=unix://")
|
||||||
|
if args[6:] == ["info", "--format", "{{json .}}"]:
|
||||||
|
return json.dumps({"ID": "fixed-inner-engine", "ServerVersion": "28.5.2"})
|
||||||
|
assert args[6:] == ["ps", "-a", "--no-trunc", "-q"]
|
||||||
|
return "private-secret-canary" if pending else ""
|
||||||
|
monkeypatch.setattr(driver, "_run", lambda self, *a, **kw: native(*a, **kw))
|
||||||
|
def effect(args, **kw):
|
||||||
|
assert args == ["docker", "update", "--restart=no", recovery.DIND_ID]
|
||||||
|
assert kw == {"timeout": 20, "max_output": 16 * 1024}
|
||||||
|
events.append("exact-policy-update")
|
||||||
|
dind["HostConfig"]["RestartPolicy"]["Name"] = "no"
|
||||||
|
monkeypatch.setattr(fleet, "_run", effect)
|
||||||
|
monkeypatch.setattr(recovery, "_load", lambda: (fleet, fleet, q, {}))
|
||||||
|
return values
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_double_admission_never_claims_or_updates(recovery, owned, monkeypatch):
|
||||||
|
fleet, *rest = owned
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("read-only claim"))
|
||||||
|
monkeypatch.setattr(fleet, "_run", lambda *a, **kw: pytest.fail("read-only effect"))
|
||||||
|
assert recovery.run()["status"] == "ready"
|
||||||
|
assert rest[-2].count("guard-cas-checked") == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_precedes_exact_update_and_existing_locked_rollback(recovery, owned, monkeypatch):
|
||||||
|
fleet, _, _, _, _, _, _, _, dind, events, _ = owned
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: events.append("claim"))
|
||||||
|
def rollback(journal, barrier):
|
||||||
|
barrier()
|
||||||
|
assert dind["HostConfig"]["RestartPolicy"]["Name"] == "no"
|
||||||
|
events.append("existing-rollback")
|
||||||
|
return True
|
||||||
|
monkeypatch.setattr(fleet, "_rollback_before_activation_locked", rollback)
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "rolled_back" and result["invocationConsumed"] is True
|
||||||
|
assert events.index("claim") < events.index("exact-policy-update") < events.index("existing-rollback") < events.index("maintenance-exit")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("change", ["cid", "image", "mount", "config", "health", "policy", "retry", "queue", "busy",
|
||||||
|
"registration", "guard", "claim", "marker", "journal", "backup", "manifest", "workspace", "profile"])
|
||||||
|
def test_changed_incident_refuses_before_claim_or_effect(recovery, owned, monkeypatch, change):
|
||||||
|
fleet, _, _, _, marker, journal, _, rows, dind, _, pending = owned
|
||||||
|
if change in {"cid", "image"}:
|
||||||
|
dind["Id" if change == "cid" else "Image"] = "f" * 64
|
||||||
|
elif change == "mount":
|
||||||
|
dind["Mounts"].append({"Source": "foreign"})
|
||||||
|
elif change == "config":
|
||||||
|
dind["Config"]["foreign"] = True
|
||||||
|
elif change == "health":
|
||||||
|
dind["Config"]["Healthcheck"]["Test"] = ["NONE"]
|
||||||
|
elif change == "policy":
|
||||||
|
dind["HostConfig"]["RestartPolicy"]["Name"] = "always"
|
||||||
|
elif change == "retry":
|
||||||
|
dind["HostConfig"]["RestartPolicy"]["MaximumRetryCount"] = 1
|
||||||
|
elif change == "queue":
|
||||||
|
pending.append({"id": 1})
|
||||||
|
elif change == "busy":
|
||||||
|
rows[-1]["busy"] = True
|
||||||
|
elif change == "registration":
|
||||||
|
rows[-1]["id"] = 999
|
||||||
|
elif change == "guard":
|
||||||
|
monkeypatch.setattr(fleet, "_guard_script_plan", lambda _: (_ for _ in ()).throw(RuntimeError("private-secret-canary")))
|
||||||
|
elif change == "claim":
|
||||||
|
recovery.CLAIM.write_text("consumed")
|
||||||
|
elif change == "marker":
|
||||||
|
marker["updatedAt"] = "foreign"
|
||||||
|
elif change == "journal":
|
||||||
|
journal["stage"] = "receipt_pending"
|
||||||
|
elif change == "backup":
|
||||||
|
journal["backupDirectory"] += "-foreign"
|
||||||
|
monkeypatch.setattr(recovery, "JOURNAL_SHA", hashlib.sha256(json.dumps(journal).encode()).hexdigest())
|
||||||
|
elif change == "manifest":
|
||||||
|
fleet.TASK_IMAGE_MANIFEST_PATH.write_text("{}")
|
||||||
|
else:
|
||||||
|
monkeypatch.setattr(recovery, "_workspace_proof" if change == "workspace" else "_profile_state_proof",
|
||||||
|
lambda _: (_ for _ in ()).throw(recovery.RecoveryRejected("profile_state_changed")))
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("refusal must not claim"))
|
||||||
|
monkeypatch.setattr(fleet, "_run", lambda *a, **kw: pytest.fail("refusal must not update"))
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "blocked" and result["effectsPerformed"] is False
|
||||||
|
assert "private-secret-canary" not in str(result)
|
||||||
|
|
||||||
|
|
||||||
|
def test_postclaim_policy_failure_is_unknown_and_claim_not_replayed(recovery, owned, monkeypatch):
|
||||||
|
fleet = owned[0]
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: recovery.CLAIM.write_text("consumed"))
|
||||||
|
monkeypatch.setattr(fleet, "_run", lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("private-secret-canary")))
|
||||||
|
first = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert first["status"] == "unknown" and first["invocationConsumed"] is True and first["effectsPerformed"] is None
|
||||||
|
second = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert second["status"] == "blocked" and second["reason"] == "invocation_consumed_inspect_only"
|
||||||
|
assert "private-secret-canary" not in str(first) + str(second)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root metadata")
|
||||||
|
@pytest.mark.parametrize("change", [None, "extra_file", "config", "id", "labels", "ephemeral", "lock_content", "lock_mode", "lock_symlink"])
|
||||||
|
def test_native_profile_state_hashes_secret_without_exposing_it(recovery, monkeypatch, tmp_path, change):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
root = tmp_path / "state"
|
||||||
|
root.mkdir(mode=0o700)
|
||||||
|
config, data = root / "config", root / "data"
|
||||||
|
config.mkdir(mode=0o700)
|
||||||
|
data.mkdir(mode=0o700)
|
||||||
|
for key in ("checks", "build"):
|
||||||
|
file = config / f"{key}.yaml"
|
||||||
|
file.write_text("rendered-config")
|
||||||
|
file.chmod(0o600)
|
||||||
|
directory = data / key
|
||||||
|
directory.mkdir(mode=0o700)
|
||||||
|
file = directory / ".runner"
|
||||||
|
file.write_text(json.dumps({"WARNING": "private metadata", "ephemeral": False,
|
||||||
|
"id": recovery.REGISTRATIONS[key], "uuid": "private-uuid",
|
||||||
|
"name": key, "token": "private-canary", "address": "https://gitea.example.com",
|
||||||
|
"labels": [key + ":docker://fixed"]}))
|
||||||
|
file.chmod(0o644)
|
||||||
|
(directory / ".runner.lock").touch(mode=0o600)
|
||||||
|
compose = root / "compose"
|
||||||
|
compose.write_bytes(b"captured")
|
||||||
|
compose.chmod(0o600)
|
||||||
|
fleet = SimpleNamespace(RUNNER_PROFILES={key: SimpleNamespace(display_name=key, runner_label=key, legacy_labels=()) for key in ("checks", "build")},
|
||||||
|
_render_runner_config=lambda *a: b"rendered-config",
|
||||||
|
_task_image_reference=lambda *a: "fixed", RUNNER_CONFIG_ROOT=config,
|
||||||
|
_SPECIALIZED_DATA_ROOT=data, SPECIALIZED_COMPOSE_PATH=compose,
|
||||||
|
_captured_compose_model=lambda *a, **kw: b"captured", _read_journal=lambda: {})
|
||||||
|
monkeypatch.setattr(recovery, "_parents", lambda _: None)
|
||||||
|
if change == "extra_file":
|
||||||
|
(data / "build" / "foreign").write_text("private-canary")
|
||||||
|
elif change == "config":
|
||||||
|
(config / "build.yaml").write_text("foreign")
|
||||||
|
elif change in {"id", "labels", "ephemeral"}:
|
||||||
|
path = data / "build" / ".runner"
|
||||||
|
metadata = json.loads(path.read_text())
|
||||||
|
metadata[change] = {"id": 999, "labels": ["foreign"], "ephemeral": True}[change]
|
||||||
|
path.write_text(json.dumps(metadata))
|
||||||
|
elif change == "lock_content":
|
||||||
|
(data / "build" / ".runner.lock").write_text("foreign")
|
||||||
|
elif change == "lock_mode":
|
||||||
|
(data / "build" / ".runner.lock").chmod(0o644)
|
||||||
|
elif change == "lock_symlink":
|
||||||
|
path = data / "build" / ".runner.lock"
|
||||||
|
path.unlink()
|
||||||
|
path.symlink_to(compose)
|
||||||
|
if change is not None:
|
||||||
|
with pytest.raises(recovery.RecoveryRejected):
|
||||||
|
recovery._profile_state_proof(fleet)
|
||||||
|
else:
|
||||||
|
proof = recovery._profile_state_proof(fleet)
|
||||||
|
assert "secret-canary" not in str(proof) and "private-canary" not in str(proof)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("change", ["cid", "registration", "journal", "private_work", "workspace"])
|
||||||
|
def test_postclaim_drift_never_reaches_policy_update(recovery, owned, monkeypatch, change):
|
||||||
|
fleet, _, _, _, _, journal, _, rows, dind, _, pending = owned
|
||||||
|
def claim(_):
|
||||||
|
if change == "cid":
|
||||||
|
dind["Id"] = "f" * 64
|
||||||
|
elif change == "registration":
|
||||||
|
rows[-1]["disabled"] = False
|
||||||
|
elif change == "journal":
|
||||||
|
journal["foreign"] = True
|
||||||
|
elif change == "private_work":
|
||||||
|
pending.append({"id": 1})
|
||||||
|
else:
|
||||||
|
monkeypatch.setattr(recovery, "_workspace_proof", lambda _: {"changed": True})
|
||||||
|
monkeypatch.setattr(recovery, "_claim", claim)
|
||||||
|
monkeypatch.setattr(fleet, "_run", lambda *a, **kw: pytest.fail("changed admission must not update"))
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "unknown" and result["effectsPerformed"] is None
|
||||||
|
assert result["invocationConsumed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_inherited_health_counter_changes_only_do_not_change_process_identity(recovery, owned):
|
||||||
|
fleet, _, q, _, _, _, _, _, dind, *_ = owned
|
||||||
|
before = recovery._dind_identity(fleet, dind, q)
|
||||||
|
dind["State"]["Health"].update(FailingStreak=99, Log=[{"Output": "secret-canary"}])
|
||||||
|
assert recovery._dind_identity(fleet, dind, q) == before
|
||||||
|
dind["State"]["Pid"] += 1
|
||||||
|
assert recovery._dind_identity(fleet, dind, q) != before
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
BROKER = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"
|
||||||
|
|
||||||
|
|
||||||
|
def load(name, path):
|
||||||
|
spec = importlib.util.spec_from_file_location(name, path)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def recovery(monkeypatch):
|
||||||
|
monkeypatch.syspath_prepend(str(BROKER))
|
||||||
|
return load("dind_tail_test", BROKER / "runner_dind_rollback_tail_recovery.py")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def owned(recovery, monkeypatch, tmp_path):
|
||||||
|
shared = load("tail_shared", Path(__file__).with_name("test_runner_failed_cutover_recovery.py"))
|
||||||
|
monkeypatch.setattr(recovery, "PROBE_ID", "a" * 64, raising=False)
|
||||||
|
values = shared.owned.__wrapped__(recovery, monkeypatch, tmp_path)
|
||||||
|
fleet, _, q, driver, marker, journal, _, rows, dind, events, pending = values
|
||||||
|
journal.update(stage="registrations_created", cgroupCreated=True, cgroupEnsurePending=False,
|
||||||
|
newRegistrationIds=dict(recovery.REGISTRATIONS), backupDirectory=recovery.BACKUP,
|
||||||
|
cgroupRootSubtree=list(recovery.CONTROLLERS), originalContainers=[])
|
||||||
|
for row in rows:
|
||||||
|
row.update(displayName=None, status="online")
|
||||||
|
rows.extend([{"id": rid, "name": name, "labels": [label], "disabled": False, "busy": False,
|
||||||
|
"displayName": None, "status": "online"}
|
||||||
|
for rid, name, label in ((13, "itworx-unraid-deploy", "unraid-deploy"),
|
||||||
|
(14, "itworx-windows11-native", "windows-native"))])
|
||||||
|
fleet.TASK_IMAGE_MANIFEST_PATH.unlink()
|
||||||
|
fleet.TASK_IMAGE_ROOT.rmdir()
|
||||||
|
monkeypatch.setattr(fleet, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "absent")
|
||||||
|
dind["Id"] = recovery.RUNTIME_IDS[fleet.DIND_NAME]
|
||||||
|
legacy = {name: {**copy.deepcopy(dind), "Id": cid} for name, cid in recovery.RUNTIME_IDS.items()
|
||||||
|
if name != fleet.DIND_NAME}
|
||||||
|
monkeypatch.setattr(fleet, "_validate_legacy_containers", lambda _: (copy.deepcopy(legacy), []))
|
||||||
|
monkeypatch.setattr(fleet, "_inspect", lambda name: copy.deepcopy(dind) if name == fleet.DIND_NAME else copy.deepcopy(legacy.get(name)))
|
||||||
|
family = {"exists": True, "controllers": list(recovery.CONTROLLERS),
|
||||||
|
"rootSubtreeControllers": list(recovery.CONTROLLERS), "familySubtreeControllers": list(recovery.CONTROLLERS),
|
||||||
|
"groupCount": 1, "hasProcesses": False, "familyCpuMax": "max 100000", "familyMemoryMax": "max"}
|
||||||
|
monkeypatch.setattr(fleet, "_cgroup_family_snapshot", lambda: copy.deepcopy(family))
|
||||||
|
monkeypatch.setattr(recovery, "_probe_absent", lambda: True)
|
||||||
|
monkeypatch.setattr(recovery, "_prior_receipt", lambda: b"pinned")
|
||||||
|
monkeypatch.setattr(recovery, "MARKER_SHA", hashlib.sha256(json.dumps(marker).encode()).hexdigest())
|
||||||
|
monkeypatch.setattr(recovery, "JOURNAL_SHA", hashlib.sha256(json.dumps(journal).encode()).hexdigest())
|
||||||
|
monkeypatch.setattr(recovery, "__file__", str(recovery.BUNDLE / "runner_dind_rollback_tail_recovery.py"))
|
||||||
|
def legacy_private_inventory(self, args, **kwargs):
|
||||||
|
assert args == ["docker", "exec", "-e", "DOCKER_HOST=tcp://127.0.0.1:2375",
|
||||||
|
fleet.DIND_NAME, "docker", "ps", "-a", "--no-trunc", "-q"]
|
||||||
|
assert kwargs == {"timeout": 20, "max_output": 128 * 1024}
|
||||||
|
return ""
|
||||||
|
monkeypatch.setattr(driver, "_run", legacy_private_inventory)
|
||||||
|
monkeypatch.setattr(recovery, "_load", lambda: (fleet, fleet, q, {}))
|
||||||
|
def ensure(*, allow_existing):
|
||||||
|
assert allow_existing is True
|
||||||
|
events.append("ensure")
|
||||||
|
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
|
||||||
|
return False
|
||||||
|
monkeypatch.setattr(fleet, "_ensure_cgroup_family", ensure)
|
||||||
|
return values, family, legacy
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_discloses_probe_and_never_claims_or_ensures(recovery, owned, monkeypatch):
|
||||||
|
values, _, _ = owned
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("claim"))
|
||||||
|
monkeypatch.setattr(values[0], "_ensure_cgroup_family", lambda **kw: pytest.fail("ensure"))
|
||||||
|
result = recovery.run()
|
||||||
|
assert result["status"] == "ready"
|
||||||
|
assert result["diagnosticProbeLifecycle"] is True
|
||||||
|
assert result["recoveryEffectsPerformed"] is False
|
||||||
|
assert values[9].count("guard-cas-checked") == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_ensure_native_rollback_all_under_both_locks(recovery, owned, monkeypatch):
|
||||||
|
values, family, _ = owned
|
||||||
|
fleet, _, _, _, _, _, _, _, _, events, _ = values
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: events.append("claim"))
|
||||||
|
def tail(journal, barrier):
|
||||||
|
barrier()
|
||||||
|
assert events.index("claim") < events.index("ensure")
|
||||||
|
assert "maintenance-exit" not in events and "migration-exit" not in events
|
||||||
|
assert family["familyCpuMax"] == "800000 100000"
|
||||||
|
events.append("native-tail")
|
||||||
|
return True
|
||||||
|
monkeypatch.setattr(fleet, "_rollback_before_activation_locked", tail)
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "rolled_back" and result["invocationConsumed"] is True
|
||||||
|
assert events.index("native-tail") < events.index("maintenance-exit") < events.index("migration-exit")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mutation", ["cid", "unhealthy", "busy", "extra-registration", "specialized-file", "guards", "private-work",
|
||||||
|
"family-process", "family-child", "family-cpu", "family-memory", "family-subtree", "journal", "claim"])
|
||||||
|
def test_mutations_rejected_before_claim_and_effect(recovery, owned, monkeypatch, mutation):
|
||||||
|
values, family, legacy = owned
|
||||||
|
fleet, _, _, _, _, journal, _, rows, dind, events, pending = values
|
||||||
|
if mutation == "cid":
|
||||||
|
dind["Id"] = "f" * 64
|
||||||
|
elif mutation == "unhealthy":
|
||||||
|
dind["State"]["Health"]["Status"] = "unhealthy"
|
||||||
|
elif mutation == "busy":
|
||||||
|
rows[0]["busy"] = True
|
||||||
|
elif mutation == "extra-registration":
|
||||||
|
rows.append({"id": 23, "busy": False})
|
||||||
|
elif mutation == "specialized-file":
|
||||||
|
fleet.RUNNER_CONFIG_ROOT.write_text("private-canary")
|
||||||
|
elif mutation == "guards":
|
||||||
|
monkeypatch.setattr(fleet, "_guard_state_unchanged", lambda _: False)
|
||||||
|
elif mutation == "private-work":
|
||||||
|
monkeypatch.setattr(values[3], "_run", lambda *a, **kw: "private-canary")
|
||||||
|
elif mutation == "family-process":
|
||||||
|
family["hasProcesses"] = True
|
||||||
|
elif mutation == "family-child":
|
||||||
|
family["groupCount"] = 2
|
||||||
|
elif mutation == "family-cpu":
|
||||||
|
family["familyCpuMax"] = "800000 100000"
|
||||||
|
elif mutation == "family-memory":
|
||||||
|
family["familyMemoryMax"] = "1"
|
||||||
|
elif mutation == "family-subtree":
|
||||||
|
family["familySubtreeControllers"].pop()
|
||||||
|
elif mutation == "journal":
|
||||||
|
journal["backupDirectory"] += "foreign"
|
||||||
|
elif mutation == "claim":
|
||||||
|
recovery.CLAIM.write_text("consumed")
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("claim"))
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "blocked"
|
||||||
|
assert result["recoveryEffectsPerformed"] is False
|
||||||
|
assert "ensure" not in events and "private-canary" not in json.dumps(result)
|
||||||
|
|
||||||
|
|
||||||
|
def test_admission_drift_refuses_before_claim(recovery, owned, monkeypatch):
|
||||||
|
original = recovery._admit
|
||||||
|
calls = []
|
||||||
|
def drift(*args, **kwargs):
|
||||||
|
journal, proof = original(*args, **kwargs)
|
||||||
|
calls.append(1)
|
||||||
|
return journal, {**proof, "runtimeDigest": str(len(calls))}
|
||||||
|
monkeypatch.setattr(recovery, "_admit", drift)
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("claim"))
|
||||||
|
assert recovery.run(execute=True, confirmation=recovery.CONFIRMATION)["reason"] == "admission_changed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ensure_failure_consumes_and_reports_unknown_redacted(recovery, owned, monkeypatch):
|
||||||
|
monkeypatch.setattr(recovery, "_claim", lambda _: None)
|
||||||
|
def fail(**kwargs):
|
||||||
|
raise RuntimeError("token=private-canary")
|
||||||
|
monkeypatch.setattr(owned[0][0], "_ensure_cgroup_family", fail)
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "unknown" and result["invocationConsumed"] is True
|
||||||
|
assert result["recoveryEffectsPerformed"] is None and "private-canary" not in json.dumps(result)
|
||||||
|
|
||||||
|
|
||||||
|
def test_prior_receipt_closed_schema_and_raw_hash(recovery, monkeypatch):
|
||||||
|
raw = json.dumps({"schema": "appops.failed-cutover-recovery/v1", "phase": "consumed",
|
||||||
|
"incidentJobId": recovery.INCIDENT_JOB, "incidentBackupDirectory": recovery.BACKUP,
|
||||||
|
"sourceRevision": recovery.ORIGIN, "sourceTree": recovery.TREE, "artifactSha256": recovery.ARTIFACT_SHA,
|
||||||
|
"candidateSha256": recovery.FLEET_SHA, "proof": recovery.PRIOR_PROOF}).encode()
|
||||||
|
monkeypatch.setattr(recovery, "_read", lambda *a, **kw: raw)
|
||||||
|
monkeypatch.setattr(recovery, "PRIOR_SHA", hashlib.sha256(raw).hexdigest())
|
||||||
|
assert recovery._prior_receipt() == raw
|
||||||
|
for changed in ({"extra": "private-canary"}, {"incidentJobId": "other"}, {"proof": {}}):
|
||||||
|
altered = json.dumps({**json.loads(raw), **changed}).encode()
|
||||||
|
monkeypatch.setattr(recovery, "_read", lambda *a, **kw: altered)
|
||||||
|
monkeypatch.setattr(recovery, "PRIOR_SHA", hashlib.sha256(altered).hexdigest())
|
||||||
|
with pytest.raises(recovery.RecoveryRejected, match="prior_receipt_changed"):
|
||||||
|
recovery._prior_receipt()
|
||||||
|
|
||||||
|
|
||||||
|
def test_postclaim_runtime_change_refuses_before_ensure(recovery, owned, monkeypatch):
|
||||||
|
values, _, _ = owned
|
||||||
|
def consume(_):
|
||||||
|
values[8]["Id"] = "f" * 64
|
||||||
|
monkeypatch.setattr(recovery, "_claim", consume)
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "unknown" and result["invocationConsumed"] is True
|
||||||
|
assert "ensure" not in values[9]
|
||||||
|
|
||||||
|
|
||||||
|
def test_once_claim_cannot_be_replayed(recovery, owned, monkeypatch):
|
||||||
|
values, _, _ = owned
|
||||||
|
def consume(_):
|
||||||
|
recovery.CLAIM.write_text("consumed")
|
||||||
|
monkeypatch.setattr(recovery, "_claim", consume)
|
||||||
|
monkeypatch.setattr(values[0], "_rollback_before_activation_locked", lambda journal, barrier: False)
|
||||||
|
assert recovery.run(execute=True, confirmation=recovery.CONFIRMATION)["status"] == "unknown"
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "blocked" and result["reason"] == "invocation_consumed_inspect_only"
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_write_uncertainty_is_unknown(recovery, owned, monkeypatch):
|
||||||
|
def uncertain(_):
|
||||||
|
raise OSError("secret-canary")
|
||||||
|
monkeypatch.setattr(recovery, "_claim", uncertain)
|
||||||
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
||||||
|
assert result["status"] == "unknown" and result["invocationConsumed"] is None
|
||||||
|
assert result["recoveryEffectsPerformed"] is None and "secret-canary" not in json.dumps(result)
|
||||||
|
assert "ensure" not in owned[0][9]
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_runtime_ids_are_full_immutable_ids(recovery):
|
||||||
|
assert all(len(cid) == 64 and set(cid) <= set("0123456789abcdef") for cid in recovery.RUNTIME_IDS.values())
|
||||||
|
assert recovery.CANDIDATE_SHA == recovery.FLEET_SHA
|
||||||
|
assert recovery.CLAIM != recovery.PRIOR_CLAIM
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(sys.platform == "win32", reason="native POSIX ownership required")
|
||||||
|
def test_native_existing_migration_lock_rejects_missing_and_retains_lock(recovery):
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from types import SimpleNamespace
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
pytest.skip("native root ownership required")
|
||||||
|
parent = Path("/data") if Path("/data").is_dir() else Path("/root")
|
||||||
|
recovery._parents(parent / "native-test-root")
|
||||||
|
with tempfile.TemporaryDirectory(prefix="appops-tail-lock-test-", dir=parent) as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
root.chmod(0o700)
|
||||||
|
fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=root / "journal.json")
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
with recovery._migration_existing_lock(fleet):
|
||||||
|
pytest.fail("missing lock admitted")
|
||||||
|
lock = root / "specialization.lock"
|
||||||
|
lock.write_bytes(b"")
|
||||||
|
lock.chmod(0o600)
|
||||||
|
with recovery._migration_existing_lock(fleet) as barrier:
|
||||||
|
barrier()
|
||||||
|
with pytest.raises(recovery.RecoveryRejected, match="migration_lock_busy"):
|
||||||
|
with recovery._migration_existing_lock(fleet):
|
||||||
|
pytest.fail("second lock acquired")
|
||||||
|
assert lock.exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(sys.platform == "win32", reason="native POSIX ownership required")
|
||||||
|
def test_native_untrusted_parent_refused_before_missing_lock(recovery):
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from types import SimpleNamespace
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
pytest.skip("native root ownership required")
|
||||||
|
parent = Path("/data") if Path("/data").is_dir() else Path("/root")
|
||||||
|
recovery._parents(parent / "native-test-root")
|
||||||
|
with tempfile.TemporaryDirectory(prefix="appops-tail-untrusted-test-", dir=parent) as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
root.chmod(0o777)
|
||||||
|
fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=root / "journal.json")
|
||||||
|
with pytest.raises(recovery.RecoveryRejected, match="root_path_untrusted"):
|
||||||
|
with recovery._migration_existing_lock(fleet):
|
||||||
|
pytest.fail("untrusted parent admitted")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_tail_uses_captured_legacy_tcp_not_specialized_unix(recovery, owned, monkeypatch):
|
||||||
|
values, _, _ = owned
|
||||||
|
fleet, _, _, driver, *_ = values
|
||||||
|
seen = []
|
||||||
|
def inventory(self, args, **kwargs):
|
||||||
|
assert args[3] == "DOCKER_HOST=tcp://127.0.0.1:2375"
|
||||||
|
assert "unix://" not in " ".join(args)
|
||||||
|
seen.append(args[3])
|
||||||
|
return ""
|
||||||
|
monkeypatch.setattr(driver, "_run", inventory)
|
||||||
|
monkeypatch.setattr(fleet, "_DIND_DOCKER_HOST", "unix:///run/user/1000/docker.sock")
|
||||||
|
assert recovery.run()["status"] == "ready"
|
||||||
|
assert seen == ["DOCKER_HOST=tcp://127.0.0.1:2375"] * 2
|
||||||
@@ -1181,6 +1181,66 @@ def test_specialized_compose_validation_accepts_real_compose_normalization(
|
|||||||
assert specialize._validate_composition(b"normalized candidate")
|
assert specialize._validate_composition(b"normalized candidate")
|
||||||
|
|
||||||
|
|
||||||
|
def test_specialized_dind_healthcheck_replaces_captured_tcp_endpoint(specialize, monkeypatch):
|
||||||
|
base = _base_v1_compose_model(specialize)
|
||||||
|
base["services"]["runner-dind"]["healthcheck"] = {
|
||||||
|
"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"],
|
||||||
|
"interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s",
|
||||||
|
}
|
||||||
|
rendered = json.loads(specialize._render_specialized_compose(base))
|
||||||
|
check = rendered["services"]["runner-dind"]["healthcheck"]
|
||||||
|
assert check == {
|
||||||
|
"test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"],
|
||||||
|
"interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s",
|
||||||
|
}
|
||||||
|
assert base["services"]["runner-dind"]["healthcheck"]["test"][1].startswith("DOCKER_HOST=tcp:")
|
||||||
|
monkeypatch.setattr(specialize, "_compose_model", lambda _raw: rendered)
|
||||||
|
assert specialize._validate_composition(b"reviewed model")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("check", [None, {"disable": True}, {"test": ["NONE"]},
|
||||||
|
{"disable": True, "test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"]},
|
||||||
|
{"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]},
|
||||||
|
{"test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]},
|
||||||
|
{"test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]},
|
||||||
|
])
|
||||||
|
def test_specialized_compose_rejects_healthcheck_endpoint_drift(specialize, monkeypatch, check):
|
||||||
|
model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize)))
|
||||||
|
model["services"]["runner-dind"]["healthcheck"] = check
|
||||||
|
monkeypatch.setattr(specialize, "_compose_model", lambda _raw: model)
|
||||||
|
with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_compose_invalid"):
|
||||||
|
specialize._validate_composition(b"foreign healthcheck")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("check", [None, {"Test": ["NONE"]},
|
||||||
|
{"Test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]},
|
||||||
|
{"Test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]},
|
||||||
|
{"Test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]},
|
||||||
|
])
|
||||||
|
def test_specialized_dind_runtime_rejects_healthcheck_drift(specialize, check):
|
||||||
|
base = _base_v1_compose_model(specialize)
|
||||||
|
model = json.loads(specialize._render_specialized_compose(base))
|
||||||
|
service = model["services"]["runner-dind"]
|
||||||
|
item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID,
|
||||||
|
running=True, restart_policy="no")
|
||||||
|
item["Config"].update(User="0", Cmd=service["command"],
|
||||||
|
Env=[f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"],
|
||||||
|
Healthcheck={"Test": service["healthcheck"]["test"]})
|
||||||
|
item["HostConfig"].update(CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={})
|
||||||
|
item["Mounts"] = [{"Type": "bind", "Source": mount["source"],
|
||||||
|
"Destination": mount["target"], "RW": not mount["read_only"]}
|
||||||
|
for mount in service["volumes"]]
|
||||||
|
assert specialize._validate_dind_outer(item, model)
|
||||||
|
changed = copy.deepcopy(item)
|
||||||
|
changed["Config"]["Healthcheck"] = check
|
||||||
|
assert specialize._validate_dind_outer(changed, model) is False
|
||||||
|
changed_model = copy.deepcopy(model)
|
||||||
|
changed_model["services"]["runner-dind"]["healthcheck"] = {
|
||||||
|
"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"],
|
||||||
|
}
|
||||||
|
assert specialize._validate_dind_outer(item, changed_model) is False
|
||||||
|
|
||||||
|
|
||||||
def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize):
|
def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize):
|
||||||
base = _base_v1_compose_model(specialize)
|
base = _base_v1_compose_model(specialize)
|
||||||
item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID,
|
item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID,
|
||||||
@@ -1208,6 +1268,7 @@ def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces(
|
|||||||
image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no")
|
image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no")
|
||||||
item["Config"].update(User="0", Cmd=service["command"])
|
item["Config"].update(User="0", Cmd=service["command"])
|
||||||
item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"]
|
item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"]
|
||||||
|
item["Config"]["Healthcheck"] = {"Test": service["healthcheck"]["test"]}
|
||||||
item["HostConfig"].update(
|
item["HostConfig"].update(
|
||||||
CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={}
|
CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={}
|
||||||
)
|
)
|
||||||
@@ -1358,6 +1419,123 @@ def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(speciali
|
|||||||
specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow")
|
specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow")
|
||||||
|
|
||||||
|
|
||||||
|
def _resource_guard_source(kind):
|
||||||
|
prefix = '''#!/bin/bash
|
||||||
|
set -eu
|
||||||
|
RUNNER="legacy-primary"
|
||||||
|
SECONDARY_RUNNER="legacy-secondary"
|
||||||
|
RUNNER_DIND="private-dind"
|
||||||
|
DEPLOY_RUNNER="deploy-runner"
|
||||||
|
RUNNER_PAUSE_FILE="/nonexistent-test-pause"
|
||||||
|
RUNNER_MARKER="/nonexistent-test-marker"
|
||||||
|
controller() { printf "controller:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = --state ]; then printf "%s" "$GUARD_STATE"; fi; }
|
||||||
|
timeout() { shift; "$@"; }
|
||||||
|
docker() { printf "docker:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = inspect ]; then printf legacy; fi; }
|
||||||
|
enforce_limits() { printf "enforce:%s\\n" "$*" >> "$EVENTS"; }
|
||||||
|
'''
|
||||||
|
if kind == "runner":
|
||||||
|
return prefix + '''# The sandbox is the aggregate resource envelope for all jobs and services.
|
||||||
|
if [ -n "$(timeout 10 docker inspect -f '{{.Id}}' "$RUNNER_DIND" 2>/dev/null || true)" ]; then
|
||||||
|
dind_limits="$(timeout 10 docker inspect -f '{{.HostConfig.NanoCpus}}' "$RUNNER_DIND")"
|
||||||
|
if [ "$dind_limits" != "expected" ]; then
|
||||||
|
timeout 20 docker update --restart unless-stopped "$RUNNER_DIND"
|
||||||
|
fi
|
||||||
|
if [ "$dind_limits" = "legacy" ]; then
|
||||||
|
timeout 20 docker update --cpuset-cpus 0-5 "$RUNNER_DIND"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
enforce_limits "$DEPLOY_RUNNER"
|
||||||
|
stack_recovery=0
|
||||||
|
:
|
||||||
|
unexpected=0
|
||||||
|
for name in "$DEPLOY_RUNNER"; do
|
||||||
|
[ "$name" = "$DEPLOY_RUNNER" ] && continue
|
||||||
|
done
|
||||||
|
'''
|
||||||
|
return prefix + '''# ITWORX_RUNNER_CAPACITY_V2_BEGIN
|
||||||
|
:
|
||||||
|
# ITWORX_RUNNER_CAPACITY_V2_END
|
||||||
|
if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then
|
||||||
|
enforce_limits "$RUNNER" no
|
||||||
|
enforce_limits "$SECONDARY_RUNNER" no
|
||||||
|
enforce_limits "$RUNNER_DIND" no
|
||||||
|
else
|
||||||
|
enforce_limits "$RUNNER" unless-stopped
|
||||||
|
enforce_limits "$SECONDARY_RUNNER" unless-stopped
|
||||||
|
enforce_limits "$RUNNER_DIND" unless-stopped
|
||||||
|
fi
|
||||||
|
# The deploy runner is deliberately tiny and is required for deployment and
|
||||||
|
enforce_limits "$DEPLOY_RUNNER"
|
||||||
|
enforce_limits tdarr
|
||||||
|
enforce_limits controller
|
||||||
|
enforce_limits containment-pressure
|
||||||
|
runner_healthy=1
|
||||||
|
if [ "$runner_healthy" -eq 0 ]; then
|
||||||
|
if [ "$sandbox_healthy" -eq 1 ] && \\
|
||||||
|
timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("kind", ["runner", "host"])
|
||||||
|
def test_resource_writer_is_gated_before_any_legacy_mutation(specialize, kind):
|
||||||
|
source = _resource_guard_source(kind)
|
||||||
|
if kind == "runner":
|
||||||
|
patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow")
|
||||||
|
writer = "# The sandbox is the aggregate resource envelope"
|
||||||
|
else:
|
||||||
|
patched = specialize._patch_host_guard(source, "controller --state")
|
||||||
|
writer = 'if [ -e "$RUNNER_PAUSE_FILE" ]'
|
||||||
|
assert patched.index('specialization_state="$(controller --state') < patched.index(writer)
|
||||||
|
assert 'if [ "$specialization_state" = "legacy" ]; then\n ' + writer in patched
|
||||||
|
assert 'enforce_limits "$DEPLOY_RUNNER"' in patched
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "posix", reason="requires actual Bash stub execution")
|
||||||
|
@pytest.mark.parametrize("kind", ["runner", "host"])
|
||||||
|
@pytest.mark.parametrize("state", ["legacy", "blocked", "active"])
|
||||||
|
def test_guard_bash_skips_only_legacy_resource_writer_in_cutover(specialize, tmp_path, kind, state):
|
||||||
|
source = _resource_guard_source(kind)
|
||||||
|
if kind == "runner":
|
||||||
|
patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow")
|
||||||
|
else:
|
||||||
|
patched = specialize._patch_host_guard(source, "controller --state")
|
||||||
|
events = tmp_path / "events"
|
||||||
|
result = subprocess.run(["bash", "-c", patched], capture_output=True, check=False, timeout=5,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "EVENTS": str(events), "GUARD_STATE": state})
|
||||||
|
assert result.returncode == 0
|
||||||
|
rows = events.read_text().splitlines()
|
||||||
|
assert rows[0] == "controller:--state"
|
||||||
|
legacy_writes = [row for row in rows if row.startswith("docker:update")
|
||||||
|
or row.startswith("enforce:legacy-") or row.startswith("enforce:private-dind")]
|
||||||
|
assert bool(legacy_writes) is (state == "legacy")
|
||||||
|
assert "enforce:deploy-runner" in rows
|
||||||
|
if kind == "host":
|
||||||
|
assert {"enforce:tdarr", "enforce:controller", "enforce:containment-pressure"}.issubset(rows)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("kind", ["runner", "host"])
|
||||||
|
def test_resource_writer_shape_drift_is_rejected(specialize, kind):
|
||||||
|
source = _resource_guard_source(kind)
|
||||||
|
anchor = ("# The sandbox is the aggregate resource envelope for all jobs and services.\n"
|
||||||
|
if kind == "runner" else 'if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then\n')
|
||||||
|
patch = (lambda text: specialize._patch_runner_guard(text, "controller --state", "controller --allow")) if kind == "runner" else (
|
||||||
|
lambda text: specialize._patch_host_guard(text, "controller --state"))
|
||||||
|
with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_shape_invalid"):
|
||||||
|
patch(source.replace(anchor, anchor + anchor, 1))
|
||||||
|
|
||||||
|
|
||||||
|
def test_historical_orphan_transform_does_not_acquire_new_resource_gate(specialize):
|
||||||
|
source = _resource_guard_source("runner")
|
||||||
|
historical = specialize._patch_runner_guard_historical(source, "controller --state", "controller --allow")
|
||||||
|
patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow")
|
||||||
|
writer = historical.index("# The sandbox is the aggregate resource envelope")
|
||||||
|
assert 'specialization_state="$(controller --state' not in historical[:writer]
|
||||||
|
assert 'specialization_state="$(controller --state' in patched[:patched.index("# The sandbox is the aggregate resource envelope")]
|
||||||
|
|
||||||
|
|
||||||
def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch):
|
def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch):
|
||||||
script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes()
|
script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes()
|
||||||
monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest())
|
monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest())
|
||||||
@@ -1621,7 +1799,9 @@ def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, g
|
|||||||
monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable)
|
monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable)
|
||||||
journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(),
|
journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(),
|
||||||
originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
||||||
originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupRootSubtree=[])
|
originalContainers=[], originalRegistrations=[], cgroupCreated=True,
|
||||||
|
cgroupFamilyInitiallyPresent=False, cgroupRootSubtree=[])
|
||||||
|
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
|
||||||
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
|
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
|
||||||
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose")
|
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose")
|
||||||
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
|
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
|
||||||
@@ -1656,6 +1836,174 @@ def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, g
|
|||||||
assert not marker.exists() and not durable.exists()
|
assert not marker.exists() and not durable.exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def rollback_empty_family(specialize, tmp_path, monkeypatch):
|
||||||
|
compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal"
|
||||||
|
compose.write_bytes(b"fixed compose")
|
||||||
|
marker.write_bytes(b"owned marker")
|
||||||
|
durable.write_bytes(b"owned journal")
|
||||||
|
for name, value in (("COMPOSE_PATH", compose), ("SPECIALIZED_STATE_PATH", marker),
|
||||||
|
("MIGRATION_JOURNAL_PATH", durable)):
|
||||||
|
monkeypatch.setattr(specialize, name, value)
|
||||||
|
journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(),
|
||||||
|
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
||||||
|
"originalContainers": [], "originalRegistrations": [], "guardFiles": [],
|
||||||
|
"cgroupCreated": True, "cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False,
|
||||||
|
"cgroupRootSubtree": ["cpu", "cpuset", "hugetlb", "io", "memory", "pids"]}
|
||||||
|
family = {"exists": True, "groupCount": 1, "hasProcesses": False,
|
||||||
|
"familyCpuMax": "max 100000", "familyMemoryMax": "max",
|
||||||
|
"familySubtreeControllers": list(journal["cgroupRootSubtree"]),
|
||||||
|
"rootSubtreeControllers": list(journal["cgroupRootSubtree"])}
|
||||||
|
rebound = {"created": False, "rootSubtreeBefore": list(journal["cgroupRootSubtree"])}
|
||||||
|
events = []
|
||||||
|
@contextmanager
|
||||||
|
def maintenance():
|
||||||
|
yield lambda: events.append("barrier")
|
||||||
|
monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance)
|
||||||
|
monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True)
|
||||||
|
monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: None)
|
||||||
|
monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose")
|
||||||
|
monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {})
|
||||||
|
monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: [])
|
||||||
|
monkeypatch.setattr(specialize, "_inspect", lambda _name: None)
|
||||||
|
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: events.append("snapshot") or family)
|
||||||
|
def ensure(**kwargs):
|
||||||
|
assert kwargs == {"allow_existing": True}
|
||||||
|
events.append("ensure")
|
||||||
|
return rebound
|
||||||
|
def rollback(**kwargs):
|
||||||
|
assert kwargs == {"created": True, "initial_controllers": tuple(journal["cgroupRootSubtree"])}
|
||||||
|
events.append("rollback")
|
||||||
|
return {"removed": True, "familyExists": False}
|
||||||
|
monkeypatch.setattr(specialize, "_ensure_cgroup_family_details", ensure)
|
||||||
|
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", rollback)
|
||||||
|
return journal, family, rebound, events, marker, durable
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("pending", [False, True])
|
||||||
|
@pytest.mark.parametrize("cpu", ["max", "max 100000"])
|
||||||
|
def test_rollback_rebinds_only_owned_empty_default_family_before_strict_removal(
|
||||||
|
specialize, rollback_empty_family, pending, cpu,
|
||||||
|
):
|
||||||
|
journal, family, _rebound, events, marker, durable = rollback_empty_family
|
||||||
|
journal.update(cgroupCreated=not pending, cgroupEnsurePending=pending)
|
||||||
|
family["familyCpuMax"] = cpu
|
||||||
|
original = copy.deepcopy(journal)
|
||||||
|
assert specialize._rollback_before_activation(journal) is True
|
||||||
|
assert journal == original
|
||||||
|
snapshot = events.index("snapshot")
|
||||||
|
assert events[snapshot:snapshot + 6] == ["snapshot", "barrier", "ensure", "barrier", "barrier", "rollback"]
|
||||||
|
assert not marker.exists() and not durable.exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("absent", [False, True])
|
||||||
|
def test_rollback_never_ensures_absent_or_already_bounded_family(specialize, rollback_empty_family, absent):
|
||||||
|
journal, family, _rebound, events, _marker, _durable = rollback_empty_family
|
||||||
|
if absent:
|
||||||
|
family.clear()
|
||||||
|
family["exists"] = False
|
||||||
|
else:
|
||||||
|
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
|
||||||
|
assert specialize._rollback_before_activation(journal) is True
|
||||||
|
assert "ensure" not in events and "rollback" in events
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mutation", ["initially_present", "initially_unknown", "foreign_cpu", "foreign_memory",
|
||||||
|
"partial_default", "processes", "unknown_processes", "children", "bool_count", "newly_created", "root_changed"])
|
||||||
|
def test_empty_family_rebind_refusal_retains_marker_and_journal(specialize, rollback_empty_family, mutation):
|
||||||
|
journal, family, rebound, events, marker, durable = rollback_empty_family
|
||||||
|
if mutation == "initially_present":
|
||||||
|
journal["cgroupFamilyInitiallyPresent"] = True
|
||||||
|
elif mutation == "initially_unknown":
|
||||||
|
journal.pop("cgroupFamilyInitiallyPresent")
|
||||||
|
elif mutation == "foreign_cpu":
|
||||||
|
family["familyCpuMax"] = "400000 100000"
|
||||||
|
elif mutation == "foreign_memory":
|
||||||
|
family["familyMemoryMax"] = str(8 * 1024**3)
|
||||||
|
elif mutation == "partial_default":
|
||||||
|
family["familyCpuMax"] = "800000 100000"
|
||||||
|
elif mutation == "processes":
|
||||||
|
family["hasProcesses"] = True
|
||||||
|
elif mutation == "unknown_processes":
|
||||||
|
family.pop("hasProcesses")
|
||||||
|
elif mutation == "children":
|
||||||
|
family["groupCount"] = 2
|
||||||
|
elif mutation == "bool_count":
|
||||||
|
family["groupCount"] = True
|
||||||
|
elif mutation == "newly_created":
|
||||||
|
rebound["created"] = True
|
||||||
|
else:
|
||||||
|
rebound["rootSubtreeBefore"] = ["cpu", "memory", "pids"]
|
||||||
|
assert specialize._rollback_before_activation(journal) is False
|
||||||
|
assert "rollback" not in events
|
||||||
|
assert ("ensure" in events) is (mutation in {"newly_created", "root_changed"})
|
||||||
|
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_barrier_loss_prevents_cgroup_removal_and_keeps_durable_state(
|
||||||
|
specialize, rollback_empty_family, monkeypatch,
|
||||||
|
):
|
||||||
|
journal, _family, _rebound, events, marker, durable = rollback_empty_family
|
||||||
|
@contextmanager
|
||||||
|
def maintenance():
|
||||||
|
def barrier():
|
||||||
|
if events and events[-1] == "ensure":
|
||||||
|
raise RuntimeError("maintenance identity changed")
|
||||||
|
events.append("barrier")
|
||||||
|
yield barrier
|
||||||
|
monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance)
|
||||||
|
assert specialize._rollback_before_activation(journal) is False
|
||||||
|
assert "ensure" in events and "rollback" not in events
|
||||||
|
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bounded", [False, True])
|
||||||
|
def test_family_rollback_accepts_only_required_controllers_for_minimal_initial_root(
|
||||||
|
specialize, rollback_empty_family, bounded,
|
||||||
|
):
|
||||||
|
journal, family, rebound, events, _marker, _durable = rollback_empty_family
|
||||||
|
initial = ["cpu", "memory", "pids"]
|
||||||
|
journal["cgroupRootSubtree"] = list(initial)
|
||||||
|
family["rootSubtreeControllers"] = list(initial)
|
||||||
|
family["familySubtreeControllers"] = list(initial)
|
||||||
|
rebound["rootSubtreeBefore"] = list(initial)
|
||||||
|
if bounded:
|
||||||
|
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
|
||||||
|
assert specialize._rollback_before_activation(journal) is True
|
||||||
|
assert ("ensure" in events) is not bounded
|
||||||
|
assert "rollback" in events
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bounded", [False, True])
|
||||||
|
@pytest.mark.parametrize("mutation", ["missing", "null", "string", "duplicate", "unknown", "nonstring",
|
||||||
|
"required_missing", "unowned_extra"])
|
||||||
|
def test_family_subtree_drift_refuses_before_ensure_or_removal(
|
||||||
|
specialize, rollback_empty_family, bounded, mutation,
|
||||||
|
):
|
||||||
|
journal, family, _rebound, events, marker, durable = rollback_empty_family
|
||||||
|
if bounded:
|
||||||
|
family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3))
|
||||||
|
if mutation == "missing":
|
||||||
|
family.pop("familySubtreeControllers")
|
||||||
|
elif mutation == "null":
|
||||||
|
family["familySubtreeControllers"] = None
|
||||||
|
elif mutation == "string":
|
||||||
|
family["familySubtreeControllers"] = "cpu memory pids"
|
||||||
|
elif mutation == "duplicate":
|
||||||
|
family["familySubtreeControllers"].append("cpu")
|
||||||
|
elif mutation == "unknown":
|
||||||
|
family["familySubtreeControllers"].append("foreign-controller")
|
||||||
|
elif mutation == "nonstring":
|
||||||
|
family["familySubtreeControllers"].append(["cpu"])
|
||||||
|
elif mutation == "required_missing":
|
||||||
|
family["familySubtreeControllers"].remove("pids")
|
||||||
|
else:
|
||||||
|
journal["cgroupRootSubtree"] = ["cpu", "memory", "pids"]
|
||||||
|
assert specialize._rollback_before_activation(journal) is False
|
||||||
|
assert "ensure" not in events and "rollback" not in events
|
||||||
|
assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal"
|
||||||
|
|
||||||
|
|
||||||
def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch):
|
def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch):
|
||||||
marker, journal = tmp_path / "marker", tmp_path / "journal"
|
marker, journal = tmp_path / "marker", tmp_path / "journal"
|
||||||
marker.write_bytes(b"owned marker")
|
marker.write_bytes(b"owned marker")
|
||||||
@@ -1727,7 +2075,9 @@ def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_fini
|
|||||||
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup)
|
monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup)
|
||||||
journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}],
|
journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}],
|
||||||
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
"originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None},
|
||||||
"originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupRootSubtree": []}
|
"originalContainers": [], "originalRegistrations": [], "cgroupCreated": True,
|
||||||
|
"cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": []}
|
||||||
|
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
|
||||||
assert specialize._rollback_before_activation(journal) is True
|
assert specialize._rollback_before_activation(journal) is True
|
||||||
assert events == ["cleanup", "cgroup", "guards-restored"]
|
assert events == ["cleanup", "cgroup", "guards-restored"]
|
||||||
assert not marker.exists() and not durable.exists()
|
assert not marker.exists() and not durable.exists()
|
||||||
@@ -3059,6 +3409,7 @@ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup(
|
|||||||
"guardFiles": [],
|
"guardFiles": [],
|
||||||
"cgroupCreated": True,
|
"cgroupCreated": True,
|
||||||
"cgroupEnsurePending": False,
|
"cgroupEnsurePending": False,
|
||||||
|
"cgroupFamilyInitiallyPresent": False,
|
||||||
"cgroupRootSubtree": ["cpu", "memory", "pids"],
|
"cgroupRootSubtree": ["cpu", "memory", "pids"],
|
||||||
"backupDirectory": str(tmp_path / "specialized-test"),
|
"backupDirectory": str(tmp_path / "specialized-test"),
|
||||||
}
|
}
|
||||||
@@ -3118,6 +3469,7 @@ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup(
|
|||||||
monkeypatch.setattr(specialize, "_run", run)
|
monkeypatch.setattr(specialize, "_run", run)
|
||||||
monkeypatch.setattr(specialize, "_run_compose", run_compose)
|
monkeypatch.setattr(specialize, "_run_compose", run_compose)
|
||||||
monkeypatch.setattr(specialize, "_runner_rows", lambda: [])
|
monkeypatch.setattr(specialize, "_runner_rows", lambda: [])
|
||||||
|
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False})
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
specialize,
|
specialize,
|
||||||
"rollback_host_cgroup_family",
|
"rollback_host_cgroup_family",
|
||||||
|
|||||||
Reference in new issue
Block a user