Publish curated Unraid-AppOps-Gateway source from c61e0e1e292914893b40354f904f23dd9fd4a958
This commit is contained in:
Public Source Publisher committed
1 parent
b47ce43436
commit
9b94d8d0f5
18 files changed
+1651
-75
No files matched your search
@@ -0,0 +1,931 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
BROKER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def module(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(BROKER))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"acceptance_quiescence_test", BROKER / "runner_acceptance_quiescence.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module.fleet, "_migration_lock", lambda **kwargs: nullcontext())
|
||||
return module
|
||||
|
||||
|
||||
class Driver:
|
||||
def __init__(self, module):
|
||||
self.module = module
|
||||
self.state = {
|
||||
"daemon": {
|
||||
"legacyAttested": True,
|
||||
"engineId": "fixed-private-engine",
|
||||
"outerId": "a" * 64,
|
||||
"dockerRootDir": module.ROOTLESS_STORAGE,
|
||||
"rootless": True,
|
||||
},
|
||||
"productExclusionVerified": True,
|
||||
"dispatchBarrierVerified": True,
|
||||
"resources": {
|
||||
"attachmentsVerified": True,
|
||||
"ownershipVerified": True,
|
||||
"networks": ["fixed-network"],
|
||||
"volumes": ["preserved-data"],
|
||||
},
|
||||
"rows": copy.deepcopy(module._fixed_rows()),
|
||||
"origins": {},
|
||||
}
|
||||
for row in self.state["rows"]:
|
||||
row["ownerLabelDigest"] = "b" * 64 if row["ownerLabelPresent"] else None
|
||||
row["labelDigest"] = module._digest(row["composeLabels"])
|
||||
for project, claim in module.ORIGINS.items():
|
||||
if claim is None:
|
||||
self.state["origins"][project] = None
|
||||
continue
|
||||
attempt = claim["attempt"] if claim["attempt"] is not None else 0
|
||||
self.state["origins"][project] = {
|
||||
"repository": "jens/workmesh",
|
||||
"run": claim["run"],
|
||||
"head": claim["head"],
|
||||
"status": "completed",
|
||||
"conclusion": "failure",
|
||||
"attempt": attempt,
|
||||
"task": claim["task"],
|
||||
"taskHead": claim["head"],
|
||||
"taskAttempt": attempt,
|
||||
"jobsTerminal": True,
|
||||
}
|
||||
self.runners = [
|
||||
{
|
||||
"id": 9,
|
||||
"name": "legacy-linux-1",
|
||||
"labels": ["linux", "docker"],
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"name": "legacy-linux-2",
|
||||
"labels": ["linux", "docker"],
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
},
|
||||
{
|
||||
"id": 700,
|
||||
"name": "windows-preserved",
|
||||
"labels": ["windows"],
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
},
|
||||
]
|
||||
self.effects = []
|
||||
self.pending = []
|
||||
self.timeout = False
|
||||
self.after_term = None
|
||||
|
||||
def maintenance(self):
|
||||
return nullcontext()
|
||||
|
||||
def capture_guards(self):
|
||||
return {"runner": {"synthetic": True}, "host": {"synthetic": True}}
|
||||
|
||||
def install_guards(self, receipt, persist):
|
||||
for name in ("runner", "host"):
|
||||
if receipt["guardProgress"].get(name) in {
|
||||
"installed",
|
||||
"restore_intent",
|
||||
"restored",
|
||||
}:
|
||||
continue
|
||||
receipt["guardProgress"][name] = "install_intent"
|
||||
persist()
|
||||
receipt["guardProgress"][name] = "installed"
|
||||
persist()
|
||||
|
||||
def restore_guards(self, receipt, persist):
|
||||
for name in ("runner", "host"):
|
||||
if receipt["guardProgress"].get(name) == "restored":
|
||||
continue
|
||||
receipt["guardProgress"][name] = "restore_intent"
|
||||
persist()
|
||||
receipt["guardProgress"][name] = "restored"
|
||||
persist()
|
||||
|
||||
def snapshot(self):
|
||||
return copy.deepcopy(self.state)
|
||||
|
||||
def registrations(self):
|
||||
return copy.deepcopy(self.runners)
|
||||
|
||||
def attest_registrations(self, rows):
|
||||
assert {row["id"] for row in rows if "linux" in row["labels"]} == {9, 22}
|
||||
|
||||
def pending_jobs(self):
|
||||
return copy.deepcopy(self.pending)
|
||||
|
||||
def pause(self):
|
||||
pass
|
||||
|
||||
def set_registration_disabled(self, original, value):
|
||||
self.effects.append(("registration", original["id"], value))
|
||||
next(row for row in self.runners if row["id"] == original["id"])["disabled"] = (
|
||||
value
|
||||
)
|
||||
|
||||
def disable_restart(self, cid):
|
||||
self.effects.append(("restart", cid))
|
||||
next(row for row in self.state["rows"] if row["id"] == cid)["restartPolicy"] = {
|
||||
"Name": "no",
|
||||
"MaximumRetryCount": 0,
|
||||
}
|
||||
|
||||
def term(self, cid):
|
||||
row = next(row for row in self.state["rows"] if row["id"] == cid)
|
||||
if row["state"]["Running"]:
|
||||
self.effects.append(("TERM", cid))
|
||||
if not self.timeout:
|
||||
row["state"].update(
|
||||
{
|
||||
"Running": False,
|
||||
"Status": "exited",
|
||||
"Pid": 0,
|
||||
"FinishedAt": "2026-10-09T20:00:00Z",
|
||||
}
|
||||
)
|
||||
if self.after_term:
|
||||
self.after_term(self)
|
||||
|
||||
def wait_exited(self, cid):
|
||||
return not self.timeout
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def proven(module, monkeypatch):
|
||||
receipt = {"value": None}
|
||||
monkeypatch.setattr(
|
||||
module, "_read_receipt", lambda: copy.deepcopy(receipt["value"])
|
||||
)
|
||||
|
||||
def save(value, expected):
|
||||
current = receipt["value"]
|
||||
actual = module._digest(current) if current is not None else None
|
||||
module._require(actual == expected, "maintenance_receipt_cas_failed")
|
||||
receipt["value"] = copy.deepcopy(value)
|
||||
return module._digest(value)
|
||||
|
||||
monkeypatch.setattr(module, "_save_receipt", save)
|
||||
return Driver(module), receipt
|
||||
|
||||
|
||||
def test_missing_backup_origin_never_becomes_authority(module, monkeypatch):
|
||||
monkeypatch.setitem(module.ORIGINS, "workmesh-backup-exercise-39a1a242e4b9", None)
|
||||
monkeypatch.setattr(module, "_read_receipt", lambda: None)
|
||||
driver = Driver(module)
|
||||
result = module.preflight(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "blocked"
|
||||
assert result["effectsPerformed"] is False
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
def test_positive_preserves_resources_and_windows_then_idempotent(module, proven):
|
||||
driver, receipt = proven
|
||||
data_before = copy.deepcopy(driver.state["resources"])
|
||||
ids_before = {row["id"] for row in driver.state["rows"]}
|
||||
assert (
|
||||
module.preflight(revision="d" * 40, projects={}, driver=driver)["status"]
|
||||
== "ready"
|
||||
)
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "quiesced"
|
||||
assert result["preservedData"] is True
|
||||
assert driver.state["resources"] == data_before
|
||||
assert {row["id"] for row in driver.state["rows"]} == ids_before
|
||||
assert all(not row["state"]["Running"] for row in driver.state["rows"])
|
||||
assert sum(effect[0] == "TERM" for effect in driver.effects) == 12
|
||||
assert driver.runners[2]["disabled"] is False
|
||||
assert all(
|
||||
effect[0] in {"registration", "restart", "TERM"} for effect in driver.effects
|
||||
)
|
||||
assert receipt["value"]["phase"] == "complete"
|
||||
count = len(driver.effects)
|
||||
assert (
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"]
|
||||
== "already_quiesced"
|
||||
)
|
||||
assert len(driver.effects) == count
|
||||
with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"):
|
||||
module.quiesce(revision="e" * 40, projects={}, driver=driver)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"])
|
||||
def test_initial_new_work_refuses_all_effects(module, proven, kind):
|
||||
driver, _ = proven
|
||||
driver.pending = [{"id": 99, "status": kind}]
|
||||
with pytest.raises(module.QuiescenceError, match="global_ci_work_present"):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"])
|
||||
def test_new_work_during_execution_blocks_next_container(module, proven, kind):
|
||||
driver, receipt = proven
|
||||
driver.after_term = lambda self: setattr(
|
||||
self, "pending", [{"id": 99, "status": kind}]
|
||||
)
|
||||
with pytest.raises(module.QuiescenceError, match="global_ci_work_present"):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert sum(e[0] == "restart" for e in driver.effects) == 1
|
||||
assert receipt["value"]["phase"] != "complete"
|
||||
assert all(row["disabled"] for row in driver.runners[:2])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
lambda s: s["daemon"].update({"rootless": False}),
|
||||
lambda s: s["daemon"].update({"dockerRootDir": "/srv/example/appdata/production"}),
|
||||
lambda s: s["rows"][0].update({"id": "f" * 64}),
|
||||
lambda s: s["rows"][0].update({"imageId": "sha256:" + "f" * 64}),
|
||||
lambda s: s["rows"][0]["composeLabels"].update(
|
||||
{"com.docker.compose.config-hash": "f" * 64}
|
||||
),
|
||||
lambda s: s["rows"][0]["composeLabels"].update(
|
||||
{"com.docker.compose.project.working_dir": "/production"}
|
||||
),
|
||||
lambda s: s["rows"][0]["mounts"].append(
|
||||
{"Type": "bind", "Source": "/production", "RW": True}
|
||||
),
|
||||
lambda s: s["rows"][0]["networks"].update({"foreign": "f" * 64}),
|
||||
lambda s: s["resources"].update({"attachmentsVerified": False}),
|
||||
lambda s: s["resources"].update({"ownershipVerified": False}),
|
||||
lambda s: s.update({"productExclusionVerified": False}),
|
||||
lambda s: s.update({"dispatchBarrierVerified": False}),
|
||||
lambda s: s["rows"].append(copy.deepcopy(s["rows"][0])),
|
||||
lambda s: s["rows"][0].update({"ownerLabelDigest": "e" * 64}),
|
||||
],
|
||||
)
|
||||
def test_identity_or_scope_drift_has_no_effect(module, proven, change):
|
||||
driver, _ = proven
|
||||
change(driver.state)
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("attempt", None),
|
||||
("attempt", 2),
|
||||
("taskAttempt", 3),
|
||||
("task", 999),
|
||||
("head", "e" * 40),
|
||||
("jobsTerminal", False),
|
||||
],
|
||||
)
|
||||
def test_origin_drift_refuses_effects(module, proven, field, value):
|
||||
driver, _ = proven
|
||||
driver.state["origins"]["workmesh-product-21300-7bace5f2edf0d259"][field] = value
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
def test_legacy_attempt_zero_is_actual_sentinel_and_missing_is_not_assumed(
|
||||
module, proven
|
||||
):
|
||||
driver, _ = proven
|
||||
module.ORIGINS["workmesh-runtime-19421"]["attempt"] = 0
|
||||
driver.state["origins"]["workmesh-runtime-19421"].update(attempt=0, taskAttempt=0)
|
||||
module.attest_snapshot(driver.snapshot())
|
||||
driver.state["origins"]["workmesh-runtime-19421"]["attempt"] = None
|
||||
with pytest.raises(module.QuiescenceError, match="attempt_unavailable"):
|
||||
module.attest_snapshot(driver.snapshot())
|
||||
|
||||
|
||||
def test_timeout_retains_owned_guard_and_resumes_only_own_transition(module, proven):
|
||||
driver, receipt = proven
|
||||
driver.timeout = True
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "partial" and result["guardHeld"] is True
|
||||
assert receipt["value"]["progress"][driver.state["rows"][0]["id"]] == "term_intent"
|
||||
driver.timeout = False
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "quiesced"
|
||||
|
||||
|
||||
def test_registration_drift_after_partial_never_restores_foreign_identity(
|
||||
module, proven
|
||||
):
|
||||
driver, _ = proven
|
||||
driver.timeout = True
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
before = len(driver.effects)
|
||||
driver.runners[0]["labels"].append("foreign-project")
|
||||
with pytest.raises(module.QuiescenceError, match="legacy_registration_changed"):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert len(driver.effects) == before
|
||||
|
||||
|
||||
def test_no_arbitrary_or_destructive_docker_effect(module, monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Result:
|
||||
returncode = 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_run", lambda args, **kwargs: calls.append(args) or Result()
|
||||
)
|
||||
driver = module.ControllerDriver({})
|
||||
monkeypatch.setattr(driver, "assert_barrier", lambda: None)
|
||||
monkeypatch.setattr(driver, "_guard_binding", lambda: {})
|
||||
monkeypatch.setattr(driver, "_effect_admission", lambda cid, args: None)
|
||||
cid = module._fixed_rows()[0]["id"]
|
||||
driver.disable_restart(cid)
|
||||
assert calls == [
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-e",
|
||||
"DOCKER_HOST=tcp://127.0.0.1:2375",
|
||||
"gitea-runner-dind",
|
||||
"docker",
|
||||
"update",
|
||||
"--restart=no",
|
||||
cid,
|
||||
]
|
||||
]
|
||||
with pytest.raises(module.QuiescenceError, match="container_target_invalid"):
|
||||
driver.disable_restart("user-selected")
|
||||
|
||||
|
||||
def test_public_preflight_never_serializes_private_canary(module):
|
||||
class BadDriver:
|
||||
def maintenance(self):
|
||||
return nullcontext()
|
||||
|
||||
def pending_jobs(self):
|
||||
raise ValueError("Bearer SECRET-CANARY")
|
||||
|
||||
def proof_gaps(self):
|
||||
return ["terminal_origin_unavailable"]
|
||||
|
||||
result = module.preflight(revision="d" * 40, projects={}, driver=BadDriver())
|
||||
assert "SECRET-CANARY" not in str(result)
|
||||
assert result["effectsPerformed"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("boundary", range(1, 93))
|
||||
def test_every_durable_write_boundary_resumes_without_adoption(
|
||||
module, proven, monkeypatch, boundary
|
||||
):
|
||||
driver, receipt = proven
|
||||
save = module._save_receipt
|
||||
count = 0
|
||||
|
||||
def crash(value, expected):
|
||||
nonlocal count
|
||||
count += 1
|
||||
result = save(value, expected)
|
||||
if count == boundary:
|
||||
raise OSError("simulated power loss after durable commit")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(module, "_save_receipt", crash)
|
||||
try:
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
except OSError:
|
||||
pass
|
||||
monkeypatch.setattr(module, "_save_receipt", save)
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] in {"quiesced", "already_quiesced"}
|
||||
assert receipt["value"]["phase"] == "complete"
|
||||
assert all(not row["state"]["Running"] for row in driver.state["rows"])
|
||||
assert sum(e[0] == "TERM" for e in driver.effects) == 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effect", ["set_registration_disabled", "disable_restart", "term"]
|
||||
)
|
||||
def test_effect_commit_before_receipt_completion_is_owned_resume(
|
||||
module, proven, monkeypatch, effect
|
||||
):
|
||||
driver, _ = proven
|
||||
original = getattr(driver, effect)
|
||||
crashed = False
|
||||
|
||||
def wrapper(*args):
|
||||
nonlocal crashed
|
||||
original(*args)
|
||||
if not crashed:
|
||||
crashed = True
|
||||
raise OSError("simulated loss after external effect")
|
||||
|
||||
monkeypatch.setattr(driver, effect, wrapper)
|
||||
with pytest.raises(OSError):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
monkeypatch.setattr(driver, effect, original)
|
||||
assert (
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"]
|
||||
== "quiesced"
|
||||
)
|
||||
|
||||
|
||||
def test_current_fixed_origins_include_reviewed_backup_and_attempts(module):
|
||||
assert module.ControllerDriver({}).proof_gaps() == []
|
||||
assert module.ORIGINS["workmesh-backup-exercise-39a1a242e4b9"]["run"] == 20184
|
||||
assert all(row["attempt"] == 1 for row in module.ORIGINS.values())
|
||||
|
||||
|
||||
def test_malformed_complete_receipt_cannot_reenable_maintenance(module, proven):
|
||||
driver, receipt = proven
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert module.maintenance_fenced() is False
|
||||
receipt["value"]["progress"].pop(next(iter(receipt["value"]["progress"])))
|
||||
assert module.maintenance_fenced() is True
|
||||
|
||||
|
||||
def test_failure_projection_never_misreports_partial_effects(module, proven):
|
||||
driver, _ = proven
|
||||
assert module.failure_projection("d" * 40)["effectsPerformed"] is False
|
||||
driver.timeout = True
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
result = module.failure_projection("d" * 40)
|
||||
assert result["status"] == "partial"
|
||||
assert result["effectsPerformed"] is True and result["guardHeld"] is True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_origins(module, monkeypatch):
|
||||
dates = {
|
||||
19421: ("2026-09-29T00:53:46Z", "2026-09-29T00:57:44Z", 768),
|
||||
19617: ("2026-09-29T07:36:50Z", "2026-09-29T07:41:39Z", 834),
|
||||
20184: ("2026-09-29T22:34:11Z", "2026-09-29T22:35:35Z", 941),
|
||||
21300: ("2026-10-01T20:53:43Z", "2026-10-01T20:54:47Z", 1128),
|
||||
}
|
||||
tasks, runs, jobs = [], {}, {}
|
||||
for claim in module.ORIGINS.values():
|
||||
start, end, number = dates[claim["run"]]
|
||||
conclusion = "cancelled" if claim["run"] == 20184 else "failure"
|
||||
tasks.append(
|
||||
{
|
||||
"id": claim["task"],
|
||||
"run_number": number,
|
||||
"head_sha": claim["head"],
|
||||
"workflow_id": "runtime-acceptance.yml",
|
||||
"status": conclusion,
|
||||
"created_at": start,
|
||||
"run_started_at": start,
|
||||
"updated_at": end,
|
||||
"url": f"http://127.0.0.1:3000/Jens/WorkMesh/actions/runs/{claim['run']}",
|
||||
}
|
||||
)
|
||||
runs[claim["run"]] = {
|
||||
"id": claim["run"],
|
||||
"run_number": number,
|
||||
"head_sha": claim["head"],
|
||||
"status": "completed",
|
||||
"conclusion": conclusion,
|
||||
"run_attempt": 1,
|
||||
"run_started_at": None,
|
||||
"updated_at": end,
|
||||
}
|
||||
jobs[claim["run"]] = [
|
||||
{
|
||||
"id": claim["job"],
|
||||
"run_id": claim["run"],
|
||||
"head_sha": claim["head"],
|
||||
"run_attempt": 1,
|
||||
"runner_id": claim["runner"],
|
||||
"started_at": start,
|
||||
"status": "completed",
|
||||
"conclusion": conclusion,
|
||||
}
|
||||
]
|
||||
# Pagination includes unrelated metadata but never returns it publicly.
|
||||
tasks.extend({"id": i} for i in range(50000, 50055))
|
||||
calls = []
|
||||
|
||||
def api(path, **kwargs):
|
||||
calls.append(path)
|
||||
if "/actions/tasks?" in path:
|
||||
page = int(path.rsplit("=", 1)[-1])
|
||||
return {
|
||||
"total_count": len(tasks),
|
||||
"workflow_runs": copy.deepcopy(tasks[(page - 1) * 50 : page * 50]),
|
||||
}
|
||||
run = int(path.split("/runs/")[1].split("/")[0])
|
||||
return (
|
||||
{"total_count": len(jobs[run]), "jobs": copy.deepcopy(jobs[run])}
|
||||
if "/jobs?" in path
|
||||
else copy.deepcopy(runs[run])
|
||||
)
|
||||
|
||||
monkeypatch.setattr(module.fleet, "_api_json", api)
|
||||
return tasks, runs, jobs, calls
|
||||
|
||||
|
||||
def test_native_documented_task_binding_current_attempt_and_null_run_start(
|
||||
module, native_origins
|
||||
):
|
||||
result = module.ControllerDriver({})._origins()
|
||||
module._attest_origins(result)
|
||||
assert all(row["attempt"] == 1 and row["jobsTerminal"] for row in result.values())
|
||||
assert len([path for path in native_origins[3] if "/tasks?" in path]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
"task-url",
|
||||
"task-head",
|
||||
"task-workflow",
|
||||
"task-time",
|
||||
"foreign-job",
|
||||
"duplicate-job-start",
|
||||
"later-attempt",
|
||||
"active-job",
|
||||
"missing-job",
|
||||
"missing-attempt",
|
||||
"terminal-task-changed",
|
||||
"duplicate-task",
|
||||
],
|
||||
)
|
||||
def test_native_origin_drift_refuses_authority(module, native_origins, change):
|
||||
tasks, runs, jobs, _ = native_origins
|
||||
claim = module.ORIGINS[next(iter(module.ORIGINS))]
|
||||
task = next(row for row in tasks if row["id"] == claim["task"])
|
||||
job = jobs[claim["run"]][0]
|
||||
if change == "task-url":
|
||||
task["url"] = task["url"].replace("127.0.0.1", "foreign.invalid")
|
||||
elif change == "task-head":
|
||||
task["head_sha"] = "f" * 40
|
||||
elif change == "task-workflow":
|
||||
task["workflow_id"] = "other.yml"
|
||||
elif change == "task-time":
|
||||
task["created_at"] = "2026-10-02T00:00:00Z"
|
||||
elif change == "foreign-job":
|
||||
job["runner_id"] = 800
|
||||
elif change == "duplicate-job-start":
|
||||
jobs[claim["run"]].append(copy.deepcopy(job))
|
||||
elif change == "later-attempt":
|
||||
runs[claim["run"]]["run_attempt"] = 2
|
||||
elif change == "active-job":
|
||||
job.update(status="in_progress", conclusion=None)
|
||||
elif change == "missing-job":
|
||||
jobs[claim["run"]] = []
|
||||
elif change == "missing-attempt":
|
||||
runs[claim["run"]].pop("run_attempt")
|
||||
elif change == "terminal-task-changed":
|
||||
task["status"] = "success"
|
||||
elif change == "duplicate-task":
|
||||
tasks[-1] = copy.deepcopy(tasks[0])
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module._attest_origins(module.ControllerDriver({})._origins())
|
||||
|
||||
|
||||
def test_exact_inspect_inventory_cannot_omit_foreign_volume_attachment(
|
||||
module, monkeypatch
|
||||
):
|
||||
driver = module.ControllerDriver({})
|
||||
ids = ["a" * 64, "b" * 64]
|
||||
monkeypatch.setattr(driver, "_docker", lambda *args: [{"Id": ids[0]}])
|
||||
with pytest.raises(module.QuiescenceError, match="legacy_inventory_unavailable"):
|
||||
driver._snapshot_from_ids({}, ids)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def product(module, monkeypatch):
|
||||
source = "/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh"
|
||||
names = [
|
||||
"workmesh-postgres-1",
|
||||
"workmesh-api-1",
|
||||
"workmesh-n8n-1",
|
||||
"workmesh-workmesh-1",
|
||||
]
|
||||
target = {
|
||||
"slug": "workmesh",
|
||||
"branch": "main",
|
||||
"exact_source": True,
|
||||
"source_dir": source,
|
||||
"env_source": "/srv/example/appdata/workmesh/.env",
|
||||
"compose_files": ["compose.yaml"],
|
||||
"compose_project": "workmesh",
|
||||
"containers": names,
|
||||
}
|
||||
rows = {
|
||||
name: {
|
||||
"Id": str(i + 1) * 64,
|
||||
"Name": "/" + name,
|
||||
"Image": "sha256:" + str(i + 1) * 64,
|
||||
"Created": "2026-09-01T00:00:00Z",
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Config": {
|
||||
"Labels": {
|
||||
"com.docker.compose.project": "workmesh",
|
||||
"com.docker.compose.service": name.removeprefix(
|
||||
"workmesh-"
|
||||
).removesuffix("-1"),
|
||||
"com.docker.compose.project.working_dir": source,
|
||||
"com.docker.compose.project.config_files": source + "/compose.yaml",
|
||||
}
|
||||
},
|
||||
"Mounts": [
|
||||
{
|
||||
"Type": "volume",
|
||||
"Name": "production_data",
|
||||
"Source": "/var/lib/docker/volumes/production_data/_data",
|
||||
"Destination": "/data",
|
||||
"RW": True,
|
||||
}
|
||||
],
|
||||
"NetworkSettings": {
|
||||
"Networks": {"workmesh_default": {"NetworkID": "e" * 64}}
|
||||
},
|
||||
}
|
||||
for i, name in enumerate(names)
|
||||
}
|
||||
info = {"ID": "host-engine", "DockerRootDir": "/var/lib/docker"}
|
||||
inventory = [row["Id"] for row in rows.values()]
|
||||
|
||||
def run(argv, **kw):
|
||||
import json
|
||||
|
||||
if argv[1] == "info":
|
||||
return json.dumps(info)
|
||||
if argv[1] == "inspect":
|
||||
return json.dumps([rows[argv[2]]])
|
||||
assert argv[1:3] == ["ps", "-a"]
|
||||
return "\n".join(inventory)
|
||||
|
||||
monkeypatch.setattr(module.fleet, "_run", run)
|
||||
return target, rows, info, inventory
|
||||
|
||||
|
||||
def test_manual_product_observation_protects_exact_healthy_target_without_fake_receipt(
|
||||
module, product
|
||||
):
|
||||
target, _, _, _ = product
|
||||
proof = module.ControllerDriver({"jens/workmesh": target})._product_binding(
|
||||
{"engineId": "private"}, []
|
||||
)
|
||||
assert (
|
||||
len(proof["rows"]) == 4
|
||||
and proof["receiptStatus"] == "legacy-physical-observation"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
["source", "engine", "root", "cid", "health", "mount", "network", "extra-product"],
|
||||
)
|
||||
def test_product_disjointness_fails_on_changed_identity_or_overlap(
|
||||
module, product, change
|
||||
):
|
||||
target, rows, info, inventory = product
|
||||
row = next(iter(rows.values()))
|
||||
private = [
|
||||
{
|
||||
"id": "a" * 64,
|
||||
"mounts": [{"Source": "/private-data"}],
|
||||
"networks": {"private": "b" * 64},
|
||||
}
|
||||
]
|
||||
if change == "source":
|
||||
row["Config"]["Labels"]["com.docker.compose.project.config_files"] = (
|
||||
"/foreign/compose.yaml"
|
||||
)
|
||||
elif change == "engine":
|
||||
info["ID"] = "private"
|
||||
elif change == "root":
|
||||
info["DockerRootDir"] = module.ROOTLESS_STORAGE
|
||||
elif change == "cid":
|
||||
row["Id"] = "a" * 64
|
||||
elif change == "health":
|
||||
row["State"]["Health"]["Status"] = "unhealthy"
|
||||
elif change == "mount":
|
||||
row["Mounts"][0]["Source"] = "/private-data"
|
||||
elif change == "network":
|
||||
row["NetworkSettings"]["Networks"]["workmesh_default"]["NetworkID"] = "b" * 64
|
||||
elif change == "extra-product":
|
||||
inventory.append("f" * 64)
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module.ControllerDriver({"jens/workmesh": target})._product_binding(
|
||||
{"engineId": "private"}, private
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def real_guards(module, proven, monkeypatch, tmp_path):
|
||||
import hashlib
|
||||
|
||||
driver = module.ControllerDriver({})
|
||||
monkeypatch.setattr(driver, "assert_barrier", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
driver, "registrations", lambda: copy.deepcopy(proven[0].runners)
|
||||
)
|
||||
monkeypatch.setattr(driver, "attest_registrations", lambda rows: None)
|
||||
monkeypatch.setattr(driver, "pending_jobs", lambda: [])
|
||||
sources = {}
|
||||
for name in ("runner", "host"):
|
||||
stanza = (
|
||||
"if ! flock -n 8; then\nexit 0\nfi"
|
||||
if name == "runner"
|
||||
else "flock -n 8 || exit 0"
|
||||
)
|
||||
raw = (
|
||||
'#!/bin/bash\nreadonly MAINTENANCE_LOCK="/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock"\nexec 8>"$MAINTENANCE_LOCK"\n'
|
||||
+ stanza
|
||||
+ "\n# private SECRET-CANARY\n"
|
||||
).encode()
|
||||
path = tmp_path / name
|
||||
path.write_bytes(raw)
|
||||
sources[name] = (path, hashlib.sha256(raw).hexdigest())
|
||||
monkeypatch.setattr(module, "GUARDS", sources)
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_read_source_file", lambda path, **kw: path.read_bytes()
|
||||
)
|
||||
|
||||
def write(path, raw, *, mode, expected_sha256):
|
||||
module._require(
|
||||
hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256,
|
||||
"guard_cas_failed",
|
||||
)
|
||||
path.write_bytes(raw)
|
||||
|
||||
monkeypatch.setattr(module.fleet, "_atomic_write_owned", write)
|
||||
# Only the filesystem-independent binding is exercised on Windows here.
|
||||
monkeypatch.setattr(
|
||||
module.os, "fstat", lambda fd: type("Info", (), {"st_dev": 1, "st_ino": 2})()
|
||||
)
|
||||
return driver, sources, proven[1]
|
||||
|
||||
|
||||
def test_actual_guard_wrapper_owned_cas_and_exact_restore(module, real_guards):
|
||||
driver, sources, store = real_guards
|
||||
captured = driver.capture_guards()
|
||||
value = {
|
||||
"guards": captured,
|
||||
"guardProgress": {},
|
||||
"registrations": driver.registrations(),
|
||||
"registrationProgress": {},
|
||||
"progress": {row["id"]: "quiesced" for row in module._fixed_rows()},
|
||||
}
|
||||
store["value"] = copy.deepcopy(value)
|
||||
|
||||
def persist():
|
||||
store["value"] = copy.deepcopy(value)
|
||||
|
||||
driver.install_guards(value, persist)
|
||||
assert all(
|
||||
module.GUARD_MARKER.encode() in path.read_bytes()
|
||||
for path, _ in sources.values()
|
||||
)
|
||||
value["registrationProgress"] = {"9": "restored", "22": "restored"}
|
||||
persist()
|
||||
driver.restore_guards(value, persist)
|
||||
assert value["guardProgress"] == {"runner": "restored", "host": "restored"}
|
||||
assert all(
|
||||
module.hashlib.sha256(path.read_bytes()).hexdigest() == digest
|
||||
for path, digest in sources.values()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stage", [None, "install_intent", "installed", "restore_intent", "restored"]
|
||||
)
|
||||
def test_foreign_guard_changes_are_never_adopted_or_overwritten(
|
||||
module, real_guards, stage
|
||||
):
|
||||
driver, sources, store = real_guards
|
||||
captured = driver.capture_guards()
|
||||
store["value"] = {
|
||||
"guards": captured,
|
||||
"guardProgress": {"runner": stage} if stage else {},
|
||||
}
|
||||
path = sources["runner"][0]
|
||||
path.write_bytes(path.read_bytes() + b"foreign-change")
|
||||
before = path.read_bytes()
|
||||
with pytest.raises(module.QuiescenceError, match="dispatch_guard_changed"):
|
||||
driver._guard_binding()
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"failure", ["short-write", "temp-fsync", "before-link", "after-link"]
|
||||
)
|
||||
def test_initial_receipt_atomic_publication_faults(
|
||||
module, monkeypatch, tmp_path, failure
|
||||
):
|
||||
import json
|
||||
|
||||
monkeypatch.setattr(module, "STATE_PATH", tmp_path / "receipt.json")
|
||||
monkeypatch.setattr(module.fleet, "_trusted_directory", lambda path: None)
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_trusted_private_file", lambda path, **kw: path.read_bytes()
|
||||
)
|
||||
receipt = {
|
||||
"schema": module.SCHEMA,
|
||||
"revision": "d" * 40,
|
||||
"manifestDigest": module._digest(module.OBSERVATION),
|
||||
"phase": "guarding",
|
||||
"baseline": {},
|
||||
"registrations": [],
|
||||
"registrationProgress": {},
|
||||
"progress": {},
|
||||
"guards": {"runner": {}, "host": {}},
|
||||
"guardProgress": {},
|
||||
}
|
||||
real_open, real_link, real_fsync = module.os.fdopen, module.os.link, module.os.fsync
|
||||
if failure == "short-write":
|
||||
|
||||
class Short:
|
||||
def __init__(self, fd, mode):
|
||||
self.stream = real_open(fd, mode)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stream.close()
|
||||
|
||||
def write(self, raw):
|
||||
return self.stream.write(raw[:1])
|
||||
|
||||
monkeypatch.setattr(module.os, "fdopen", Short)
|
||||
elif failure == "temp-fsync":
|
||||
monkeypatch.setattr(
|
||||
module.os, "fsync", lambda fd: (_ for _ in ()).throw(OSError("fault"))
|
||||
)
|
||||
else:
|
||||
|
||||
def link(src, dst, **kw):
|
||||
if failure == "after-link":
|
||||
real_link(src, dst, **kw)
|
||||
raise OSError("fault")
|
||||
|
||||
monkeypatch.setattr(module.os, "link", link)
|
||||
with pytest.raises((OSError, module.QuiescenceError)):
|
||||
module._save_receipt(receipt, None)
|
||||
if failure == "after-link":
|
||||
assert json.loads(module.STATE_PATH.read_bytes()) == receipt
|
||||
assert module._read_receipt() == receipt
|
||||
else:
|
||||
assert not module.STATE_PATH.exists()
|
||||
assert list(tmp_path.glob(".acceptance-receipt-*")) == []
|
||||
|
||||
|
||||
def test_deadline_expires_before_native_read_or_effect(module, monkeypatch):
|
||||
driver = module.ControllerDriver({})
|
||||
driver._deadline = module.time.monotonic() - 1
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_run", lambda *a, **kw: pytest.fail("no effect after deadline")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module.fleet,
|
||||
"_api_json",
|
||||
lambda *a, **kw: pytest.fail("no read after deadline"),
|
||||
)
|
||||
with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"):
|
||||
driver._run(["docker", "info"])
|
||||
with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"):
|
||||
driver._api_json("/admin/actions/runners")
|
||||
|
||||
|
||||
def test_readonly_poll_reports_real_partial_and_complete_without_resuming(
|
||||
module, proven
|
||||
):
|
||||
driver, receipt = proven
|
||||
driver.timeout = True
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert module.operation_status()["status"] == "partial"
|
||||
assert module.operation_status(active=True)["status"] == "in_progress"
|
||||
before = len(driver.effects)
|
||||
assert module.operation_status()["phase"] == "quiescing"
|
||||
assert len(driver.effects) == before
|
||||
driver.timeout = False
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert module.operation_status()["status"] == "quiesced"
|
||||
assert module.operation_status()["completedContainers"] == 18
|
||||
|
||||
|
||||
def test_external_migration_lock_is_nonblocking_and_readonly_busy(module, proven, monkeypatch):
|
||||
def busy(**kwargs):
|
||||
assert kwargs == {"blocking": False}
|
||||
raise module.fleet.RunnerFleetSpecializeError("runner_fleet_specialization_recovery_busy")
|
||||
monkeypatch.setattr(module.fleet, "_migration_lock", busy)
|
||||
driver, _ = proven
|
||||
result = module.preflight(revision="d"*40, projects={}, driver=driver)
|
||||
assert result["status"] == "in_progress" and result["effectsPerformed"] is False
|
||||
assert driver.effects == []
|
||||
Reference in new issue
Block a user