932 lines
33 KiB
Python
932 lines
33 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import importlib.util
|
|
import sys
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
BROKER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
|
|
|
|
|
|
@pytest.fixture
|
|
def module(monkeypatch):
|
|
monkeypatch.syspath_prepend(str(BROKER))
|
|
spec = importlib.util.spec_from_file_location(
|
|
"acceptance_quiescence_test", BROKER / "runner_acceptance_quiescence.py"
|
|
)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
monkeypatch.setattr(module.fleet, "_migration_lock", lambda **kwargs: nullcontext())
|
|
return module
|
|
|
|
|
|
class Driver:
|
|
def __init__(self, module):
|
|
self.module = module
|
|
self.state = {
|
|
"daemon": {
|
|
"legacyAttested": True,
|
|
"engineId": "fixed-private-engine",
|
|
"outerId": "a" * 64,
|
|
"dockerRootDir": module.ROOTLESS_STORAGE,
|
|
"rootless": True,
|
|
},
|
|
"productExclusionVerified": True,
|
|
"dispatchBarrierVerified": True,
|
|
"resources": {
|
|
"attachmentsVerified": True,
|
|
"ownershipVerified": True,
|
|
"networks": ["fixed-network"],
|
|
"volumes": ["preserved-data"],
|
|
},
|
|
"rows": copy.deepcopy(module._fixed_rows()),
|
|
"origins": {},
|
|
}
|
|
for row in self.state["rows"]:
|
|
row["ownerLabelDigest"] = "b" * 64 if row["ownerLabelPresent"] else None
|
|
row["labelDigest"] = module._digest(row["composeLabels"])
|
|
for project, claim in module.ORIGINS.items():
|
|
if claim is None:
|
|
self.state["origins"][project] = None
|
|
continue
|
|
attempt = claim["attempt"] if claim["attempt"] is not None else 0
|
|
self.state["origins"][project] = {
|
|
"repository": "jens/workmesh",
|
|
"run": claim["run"],
|
|
"head": claim["head"],
|
|
"status": "completed",
|
|
"conclusion": "failure",
|
|
"attempt": attempt,
|
|
"task": claim["task"],
|
|
"taskHead": claim["head"],
|
|
"taskAttempt": attempt,
|
|
"jobsTerminal": True,
|
|
}
|
|
self.runners = [
|
|
{
|
|
"id": 9,
|
|
"name": "legacy-linux-1",
|
|
"labels": ["linux", "docker"],
|
|
"busy": False,
|
|
"disabled": False,
|
|
},
|
|
{
|
|
"id": 22,
|
|
"name": "legacy-linux-2",
|
|
"labels": ["linux", "docker"],
|
|
"busy": False,
|
|
"disabled": False,
|
|
},
|
|
{
|
|
"id": 700,
|
|
"name": "windows-preserved",
|
|
"labels": ["windows"],
|
|
"busy": False,
|
|
"disabled": False,
|
|
},
|
|
]
|
|
self.effects = []
|
|
self.pending = []
|
|
self.timeout = False
|
|
self.after_term = None
|
|
|
|
def maintenance(self):
|
|
return nullcontext()
|
|
|
|
def capture_guards(self):
|
|
return {"runner": {"synthetic": True}, "host": {"synthetic": True}}
|
|
|
|
def install_guards(self, receipt, persist):
|
|
for name in ("runner", "host"):
|
|
if receipt["guardProgress"].get(name) in {
|
|
"installed",
|
|
"restore_intent",
|
|
"restored",
|
|
}:
|
|
continue
|
|
receipt["guardProgress"][name] = "install_intent"
|
|
persist()
|
|
receipt["guardProgress"][name] = "installed"
|
|
persist()
|
|
|
|
def restore_guards(self, receipt, persist):
|
|
for name in ("runner", "host"):
|
|
if receipt["guardProgress"].get(name) == "restored":
|
|
continue
|
|
receipt["guardProgress"][name] = "restore_intent"
|
|
persist()
|
|
receipt["guardProgress"][name] = "restored"
|
|
persist()
|
|
|
|
def snapshot(self):
|
|
return copy.deepcopy(self.state)
|
|
|
|
def registrations(self):
|
|
return copy.deepcopy(self.runners)
|
|
|
|
def attest_registrations(self, rows):
|
|
assert {row["id"] for row in rows if "linux" in row["labels"]} == {9, 22}
|
|
|
|
def pending_jobs(self):
|
|
return copy.deepcopy(self.pending)
|
|
|
|
def pause(self):
|
|
pass
|
|
|
|
def set_registration_disabled(self, original, value):
|
|
self.effects.append(("registration", original["id"], value))
|
|
next(row for row in self.runners if row["id"] == original["id"])["disabled"] = (
|
|
value
|
|
)
|
|
|
|
def disable_restart(self, cid):
|
|
self.effects.append(("restart", cid))
|
|
next(row for row in self.state["rows"] if row["id"] == cid)["restartPolicy"] = {
|
|
"Name": "no",
|
|
"MaximumRetryCount": 0,
|
|
}
|
|
|
|
def term(self, cid):
|
|
row = next(row for row in self.state["rows"] if row["id"] == cid)
|
|
if row["state"]["Running"]:
|
|
self.effects.append(("TERM", cid))
|
|
if not self.timeout:
|
|
row["state"].update(
|
|
{
|
|
"Running": False,
|
|
"Status": "exited",
|
|
"Pid": 0,
|
|
"FinishedAt": "2026-10-09T20:00:00Z",
|
|
}
|
|
)
|
|
if self.after_term:
|
|
self.after_term(self)
|
|
|
|
def wait_exited(self, cid):
|
|
return not self.timeout
|
|
|
|
|
|
@pytest.fixture
|
|
def proven(module, monkeypatch):
|
|
receipt = {"value": None}
|
|
monkeypatch.setattr(
|
|
module, "_read_receipt", lambda: copy.deepcopy(receipt["value"])
|
|
)
|
|
|
|
def save(value, expected):
|
|
current = receipt["value"]
|
|
actual = module._digest(current) if current is not None else None
|
|
module._require(actual == expected, "maintenance_receipt_cas_failed")
|
|
receipt["value"] = copy.deepcopy(value)
|
|
return module._digest(value)
|
|
|
|
monkeypatch.setattr(module, "_save_receipt", save)
|
|
return Driver(module), receipt
|
|
|
|
|
|
def test_missing_backup_origin_never_becomes_authority(module, monkeypatch):
|
|
monkeypatch.setitem(module.ORIGINS, "workmesh-backup-exercise-39a1a242e4b9", None)
|
|
monkeypatch.setattr(module, "_read_receipt", lambda: None)
|
|
driver = Driver(module)
|
|
result = module.preflight(revision="d" * 40, projects={}, driver=driver)
|
|
assert result["status"] == "blocked"
|
|
assert result["effectsPerformed"] is False
|
|
assert driver.effects == []
|
|
|
|
|
|
def test_positive_preserves_resources_and_windows_then_idempotent(module, proven):
|
|
driver, receipt = proven
|
|
data_before = copy.deepcopy(driver.state["resources"])
|
|
ids_before = {row["id"] for row in driver.state["rows"]}
|
|
assert (
|
|
module.preflight(revision="d" * 40, projects={}, driver=driver)["status"]
|
|
== "ready"
|
|
)
|
|
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert result["status"] == "quiesced"
|
|
assert result["preservedData"] is True
|
|
assert driver.state["resources"] == data_before
|
|
assert {row["id"] for row in driver.state["rows"]} == ids_before
|
|
assert all(not row["state"]["Running"] for row in driver.state["rows"])
|
|
assert sum(effect[0] == "TERM" for effect in driver.effects) == 12
|
|
assert driver.runners[2]["disabled"] is False
|
|
assert all(
|
|
effect[0] in {"registration", "restart", "TERM"} for effect in driver.effects
|
|
)
|
|
assert receipt["value"]["phase"] == "complete"
|
|
count = len(driver.effects)
|
|
assert (
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"]
|
|
== "already_quiesced"
|
|
)
|
|
assert len(driver.effects) == count
|
|
with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"):
|
|
module.quiesce(revision="e" * 40, projects={}, driver=driver)
|
|
|
|
|
|
@pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"])
|
|
def test_initial_new_work_refuses_all_effects(module, proven, kind):
|
|
driver, _ = proven
|
|
driver.pending = [{"id": 99, "status": kind}]
|
|
with pytest.raises(module.QuiescenceError, match="global_ci_work_present"):
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert driver.effects == []
|
|
|
|
|
|
@pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"])
|
|
def test_new_work_during_execution_blocks_next_container(module, proven, kind):
|
|
driver, receipt = proven
|
|
driver.after_term = lambda self: setattr(
|
|
self, "pending", [{"id": 99, "status": kind}]
|
|
)
|
|
with pytest.raises(module.QuiescenceError, match="global_ci_work_present"):
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert sum(e[0] == "restart" for e in driver.effects) == 1
|
|
assert receipt["value"]["phase"] != "complete"
|
|
assert all(row["disabled"] for row in driver.runners[:2])
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
[
|
|
lambda s: s["daemon"].update({"rootless": False}),
|
|
lambda s: s["daemon"].update({"dockerRootDir": "/srv/example/appdata/production"}),
|
|
lambda s: s["rows"][0].update({"id": "f" * 64}),
|
|
lambda s: s["rows"][0].update({"imageId": "sha256:" + "f" * 64}),
|
|
lambda s: s["rows"][0]["composeLabels"].update(
|
|
{"com.docker.compose.config-hash": "f" * 64}
|
|
),
|
|
lambda s: s["rows"][0]["composeLabels"].update(
|
|
{"com.docker.compose.project.working_dir": "/production"}
|
|
),
|
|
lambda s: s["rows"][0]["mounts"].append(
|
|
{"Type": "bind", "Source": "/production", "RW": True}
|
|
),
|
|
lambda s: s["rows"][0]["networks"].update({"foreign": "f" * 64}),
|
|
lambda s: s["resources"].update({"attachmentsVerified": False}),
|
|
lambda s: s["resources"].update({"ownershipVerified": False}),
|
|
lambda s: s.update({"productExclusionVerified": False}),
|
|
lambda s: s.update({"dispatchBarrierVerified": False}),
|
|
lambda s: s["rows"].append(copy.deepcopy(s["rows"][0])),
|
|
lambda s: s["rows"][0].update({"ownerLabelDigest": "e" * 64}),
|
|
],
|
|
)
|
|
def test_identity_or_scope_drift_has_no_effect(module, proven, change):
|
|
driver, _ = proven
|
|
change(driver.state)
|
|
with pytest.raises(module.QuiescenceError):
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert driver.effects == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field,value",
|
|
[
|
|
("attempt", None),
|
|
("attempt", 2),
|
|
("taskAttempt", 3),
|
|
("task", 999),
|
|
("head", "e" * 40),
|
|
("jobsTerminal", False),
|
|
],
|
|
)
|
|
def test_origin_drift_refuses_effects(module, proven, field, value):
|
|
driver, _ = proven
|
|
driver.state["origins"]["workmesh-product-21300-7bace5f2edf0d259"][field] = value
|
|
with pytest.raises(module.QuiescenceError):
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert driver.effects == []
|
|
|
|
|
|
def test_legacy_attempt_zero_is_actual_sentinel_and_missing_is_not_assumed(
|
|
module, proven
|
|
):
|
|
driver, _ = proven
|
|
module.ORIGINS["workmesh-runtime-19421"]["attempt"] = 0
|
|
driver.state["origins"]["workmesh-runtime-19421"].update(attempt=0, taskAttempt=0)
|
|
module.attest_snapshot(driver.snapshot())
|
|
driver.state["origins"]["workmesh-runtime-19421"]["attempt"] = None
|
|
with pytest.raises(module.QuiescenceError, match="attempt_unavailable"):
|
|
module.attest_snapshot(driver.snapshot())
|
|
|
|
|
|
def test_timeout_retains_owned_guard_and_resumes_only_own_transition(module, proven):
|
|
driver, receipt = proven
|
|
driver.timeout = True
|
|
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert result["status"] == "partial" and result["guardHeld"] is True
|
|
assert receipt["value"]["progress"][driver.state["rows"][0]["id"]] == "term_intent"
|
|
driver.timeout = False
|
|
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert result["status"] == "quiesced"
|
|
|
|
|
|
def test_registration_drift_after_partial_never_restores_foreign_identity(
|
|
module, proven
|
|
):
|
|
driver, _ = proven
|
|
driver.timeout = True
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
before = len(driver.effects)
|
|
driver.runners[0]["labels"].append("foreign-project")
|
|
with pytest.raises(module.QuiescenceError, match="legacy_registration_changed"):
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert len(driver.effects) == before
|
|
|
|
|
|
def test_no_arbitrary_or_destructive_docker_effect(module, monkeypatch):
|
|
calls = []
|
|
|
|
class Result:
|
|
returncode = 0
|
|
|
|
monkeypatch.setattr(
|
|
module.fleet, "_run", lambda args, **kwargs: calls.append(args) or Result()
|
|
)
|
|
driver = module.ControllerDriver({})
|
|
monkeypatch.setattr(driver, "assert_barrier", lambda: None)
|
|
monkeypatch.setattr(driver, "_guard_binding", lambda: {})
|
|
monkeypatch.setattr(driver, "_effect_admission", lambda cid, args: None)
|
|
cid = module._fixed_rows()[0]["id"]
|
|
driver.disable_restart(cid)
|
|
assert calls == [
|
|
[
|
|
"docker",
|
|
"exec",
|
|
"-e",
|
|
"DOCKER_HOST=tcp://127.0.0.1:2375",
|
|
"gitea-runner-dind",
|
|
"docker",
|
|
"update",
|
|
"--restart=no",
|
|
cid,
|
|
]
|
|
]
|
|
with pytest.raises(module.QuiescenceError, match="container_target_invalid"):
|
|
driver.disable_restart("user-selected")
|
|
|
|
|
|
def test_public_preflight_never_serializes_private_canary(module):
|
|
class BadDriver:
|
|
def maintenance(self):
|
|
return nullcontext()
|
|
|
|
def pending_jobs(self):
|
|
raise ValueError("Bearer SECRET-CANARY")
|
|
|
|
def proof_gaps(self):
|
|
return ["terminal_origin_unavailable"]
|
|
|
|
result = module.preflight(revision="d" * 40, projects={}, driver=BadDriver())
|
|
assert "SECRET-CANARY" not in str(result)
|
|
assert result["effectsPerformed"] is False
|
|
|
|
|
|
@pytest.mark.parametrize("boundary", range(1, 93))
|
|
def test_every_durable_write_boundary_resumes_without_adoption(
|
|
module, proven, monkeypatch, boundary
|
|
):
|
|
driver, receipt = proven
|
|
save = module._save_receipt
|
|
count = 0
|
|
|
|
def crash(value, expected):
|
|
nonlocal count
|
|
count += 1
|
|
result = save(value, expected)
|
|
if count == boundary:
|
|
raise OSError("simulated power loss after durable commit")
|
|
return result
|
|
|
|
monkeypatch.setattr(module, "_save_receipt", crash)
|
|
try:
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
except OSError:
|
|
pass
|
|
monkeypatch.setattr(module, "_save_receipt", save)
|
|
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert result["status"] in {"quiesced", "already_quiesced"}
|
|
assert receipt["value"]["phase"] == "complete"
|
|
assert all(not row["state"]["Running"] for row in driver.state["rows"])
|
|
assert sum(e[0] == "TERM" for e in driver.effects) == 12
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"effect", ["set_registration_disabled", "disable_restart", "term"]
|
|
)
|
|
def test_effect_commit_before_receipt_completion_is_owned_resume(
|
|
module, proven, monkeypatch, effect
|
|
):
|
|
driver, _ = proven
|
|
original = getattr(driver, effect)
|
|
crashed = False
|
|
|
|
def wrapper(*args):
|
|
nonlocal crashed
|
|
original(*args)
|
|
if not crashed:
|
|
crashed = True
|
|
raise OSError("simulated loss after external effect")
|
|
|
|
monkeypatch.setattr(driver, effect, wrapper)
|
|
with pytest.raises(OSError):
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
monkeypatch.setattr(driver, effect, original)
|
|
assert (
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"]
|
|
== "quiesced"
|
|
)
|
|
|
|
|
|
def test_current_fixed_origins_include_reviewed_backup_and_attempts(module):
|
|
assert module.ControllerDriver({}).proof_gaps() == []
|
|
assert module.ORIGINS["workmesh-backup-exercise-39a1a242e4b9"]["run"] == 20184
|
|
assert all(row["attempt"] == 1 for row in module.ORIGINS.values())
|
|
|
|
|
|
def test_malformed_complete_receipt_cannot_reenable_maintenance(module, proven):
|
|
driver, receipt = proven
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert module.maintenance_fenced() is False
|
|
receipt["value"]["progress"].pop(next(iter(receipt["value"]["progress"])))
|
|
assert module.maintenance_fenced() is True
|
|
|
|
|
|
def test_failure_projection_never_misreports_partial_effects(module, proven):
|
|
driver, _ = proven
|
|
assert module.failure_projection("d" * 40)["effectsPerformed"] is False
|
|
driver.timeout = True
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
result = module.failure_projection("d" * 40)
|
|
assert result["status"] == "partial"
|
|
assert result["effectsPerformed"] is True and result["guardHeld"] is True
|
|
|
|
|
|
@pytest.fixture
|
|
def native_origins(module, monkeypatch):
|
|
dates = {
|
|
19421: ("2026-09-29T00:53:46Z", "2026-09-29T00:57:44Z", 768),
|
|
19617: ("2026-09-29T07:36:50Z", "2026-09-29T07:41:39Z", 834),
|
|
20184: ("2026-09-29T22:34:11Z", "2026-09-29T22:35:35Z", 941),
|
|
21300: ("2026-10-01T20:53:43Z", "2026-10-01T20:54:47Z", 1128),
|
|
}
|
|
tasks, runs, jobs = [], {}, {}
|
|
for claim in module.ORIGINS.values():
|
|
start, end, number = dates[claim["run"]]
|
|
conclusion = "cancelled" if claim["run"] == 20184 else "failure"
|
|
tasks.append(
|
|
{
|
|
"id": claim["task"],
|
|
"run_number": number,
|
|
"head_sha": claim["head"],
|
|
"workflow_id": "runtime-acceptance.yml",
|
|
"status": conclusion,
|
|
"created_at": start,
|
|
"run_started_at": start,
|
|
"updated_at": end,
|
|
"url": f"http://127.0.0.1:3000/Jens/WorkMesh/actions/runs/{claim['run']}",
|
|
}
|
|
)
|
|
runs[claim["run"]] = {
|
|
"id": claim["run"],
|
|
"run_number": number,
|
|
"head_sha": claim["head"],
|
|
"status": "completed",
|
|
"conclusion": conclusion,
|
|
"run_attempt": 1,
|
|
"run_started_at": None,
|
|
"updated_at": end,
|
|
}
|
|
jobs[claim["run"]] = [
|
|
{
|
|
"id": claim["job"],
|
|
"run_id": claim["run"],
|
|
"head_sha": claim["head"],
|
|
"run_attempt": 1,
|
|
"runner_id": claim["runner"],
|
|
"started_at": start,
|
|
"status": "completed",
|
|
"conclusion": conclusion,
|
|
}
|
|
]
|
|
# Pagination includes unrelated metadata but never returns it publicly.
|
|
tasks.extend({"id": i} for i in range(50000, 50055))
|
|
calls = []
|
|
|
|
def api(path, **kwargs):
|
|
calls.append(path)
|
|
if "/actions/tasks?" in path:
|
|
page = int(path.rsplit("=", 1)[-1])
|
|
return {
|
|
"total_count": len(tasks),
|
|
"workflow_runs": copy.deepcopy(tasks[(page - 1) * 50 : page * 50]),
|
|
}
|
|
run = int(path.split("/runs/")[1].split("/")[0])
|
|
return (
|
|
{"total_count": len(jobs[run]), "jobs": copy.deepcopy(jobs[run])}
|
|
if "/jobs?" in path
|
|
else copy.deepcopy(runs[run])
|
|
)
|
|
|
|
monkeypatch.setattr(module.fleet, "_api_json", api)
|
|
return tasks, runs, jobs, calls
|
|
|
|
|
|
def test_native_documented_task_binding_current_attempt_and_null_run_start(
|
|
module, native_origins
|
|
):
|
|
result = module.ControllerDriver({})._origins()
|
|
module._attest_origins(result)
|
|
assert all(row["attempt"] == 1 and row["jobsTerminal"] for row in result.values())
|
|
assert len([path for path in native_origins[3] if "/tasks?" in path]) == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
[
|
|
"task-url",
|
|
"task-head",
|
|
"task-workflow",
|
|
"task-time",
|
|
"foreign-job",
|
|
"duplicate-job-start",
|
|
"later-attempt",
|
|
"active-job",
|
|
"missing-job",
|
|
"missing-attempt",
|
|
"terminal-task-changed",
|
|
"duplicate-task",
|
|
],
|
|
)
|
|
def test_native_origin_drift_refuses_authority(module, native_origins, change):
|
|
tasks, runs, jobs, _ = native_origins
|
|
claim = module.ORIGINS[next(iter(module.ORIGINS))]
|
|
task = next(row for row in tasks if row["id"] == claim["task"])
|
|
job = jobs[claim["run"]][0]
|
|
if change == "task-url":
|
|
task["url"] = task["url"].replace("127.0.0.1", "foreign.invalid")
|
|
elif change == "task-head":
|
|
task["head_sha"] = "f" * 40
|
|
elif change == "task-workflow":
|
|
task["workflow_id"] = "other.yml"
|
|
elif change == "task-time":
|
|
task["created_at"] = "2026-10-02T00:00:00Z"
|
|
elif change == "foreign-job":
|
|
job["runner_id"] = 800
|
|
elif change == "duplicate-job-start":
|
|
jobs[claim["run"]].append(copy.deepcopy(job))
|
|
elif change == "later-attempt":
|
|
runs[claim["run"]]["run_attempt"] = 2
|
|
elif change == "active-job":
|
|
job.update(status="in_progress", conclusion=None)
|
|
elif change == "missing-job":
|
|
jobs[claim["run"]] = []
|
|
elif change == "missing-attempt":
|
|
runs[claim["run"]].pop("run_attempt")
|
|
elif change == "terminal-task-changed":
|
|
task["status"] = "success"
|
|
elif change == "duplicate-task":
|
|
tasks[-1] = copy.deepcopy(tasks[0])
|
|
with pytest.raises(module.QuiescenceError):
|
|
module._attest_origins(module.ControllerDriver({})._origins())
|
|
|
|
|
|
def test_exact_inspect_inventory_cannot_omit_foreign_volume_attachment(
|
|
module, monkeypatch
|
|
):
|
|
driver = module.ControllerDriver({})
|
|
ids = ["a" * 64, "b" * 64]
|
|
monkeypatch.setattr(driver, "_docker", lambda *args: [{"Id": ids[0]}])
|
|
with pytest.raises(module.QuiescenceError, match="legacy_inventory_unavailable"):
|
|
driver._snapshot_from_ids({}, ids)
|
|
|
|
|
|
@pytest.fixture
|
|
def product(module, monkeypatch):
|
|
source = "/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh"
|
|
names = [
|
|
"workmesh-postgres-1",
|
|
"workmesh-api-1",
|
|
"workmesh-n8n-1",
|
|
"workmesh-workmesh-1",
|
|
]
|
|
target = {
|
|
"slug": "workmesh",
|
|
"branch": "main",
|
|
"exact_source": True,
|
|
"source_dir": source,
|
|
"env_source": "/srv/example/appdata/workmesh/.env",
|
|
"compose_files": ["compose.yaml"],
|
|
"compose_project": "workmesh",
|
|
"containers": names,
|
|
}
|
|
rows = {
|
|
name: {
|
|
"Id": str(i + 1) * 64,
|
|
"Name": "/" + name,
|
|
"Image": "sha256:" + str(i + 1) * 64,
|
|
"Created": "2026-09-01T00:00:00Z",
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
"Config": {
|
|
"Labels": {
|
|
"com.docker.compose.project": "workmesh",
|
|
"com.docker.compose.service": name.removeprefix(
|
|
"workmesh-"
|
|
).removesuffix("-1"),
|
|
"com.docker.compose.project.working_dir": source,
|
|
"com.docker.compose.project.config_files": source + "/compose.yaml",
|
|
}
|
|
},
|
|
"Mounts": [
|
|
{
|
|
"Type": "volume",
|
|
"Name": "production_data",
|
|
"Source": "/var/lib/docker/volumes/production_data/_data",
|
|
"Destination": "/data",
|
|
"RW": True,
|
|
}
|
|
],
|
|
"NetworkSettings": {
|
|
"Networks": {"workmesh_default": {"NetworkID": "e" * 64}}
|
|
},
|
|
}
|
|
for i, name in enumerate(names)
|
|
}
|
|
info = {"ID": "host-engine", "DockerRootDir": "/var/lib/docker"}
|
|
inventory = [row["Id"] for row in rows.values()]
|
|
|
|
def run(argv, **kw):
|
|
import json
|
|
|
|
if argv[1] == "info":
|
|
return json.dumps(info)
|
|
if argv[1] == "inspect":
|
|
return json.dumps([rows[argv[2]]])
|
|
assert argv[1:3] == ["ps", "-a"]
|
|
return "\n".join(inventory)
|
|
|
|
monkeypatch.setattr(module.fleet, "_run", run)
|
|
return target, rows, info, inventory
|
|
|
|
|
|
def test_manual_product_observation_protects_exact_healthy_target_without_fake_receipt(
|
|
module, product
|
|
):
|
|
target, _, _, _ = product
|
|
proof = module.ControllerDriver({"jens/workmesh": target})._product_binding(
|
|
{"engineId": "private"}, []
|
|
)
|
|
assert (
|
|
len(proof["rows"]) == 4
|
|
and proof["receiptStatus"] == "legacy-physical-observation"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
["source", "engine", "root", "cid", "health", "mount", "network", "extra-product"],
|
|
)
|
|
def test_product_disjointness_fails_on_changed_identity_or_overlap(
|
|
module, product, change
|
|
):
|
|
target, rows, info, inventory = product
|
|
row = next(iter(rows.values()))
|
|
private = [
|
|
{
|
|
"id": "a" * 64,
|
|
"mounts": [{"Source": "/private-data"}],
|
|
"networks": {"private": "b" * 64},
|
|
}
|
|
]
|
|
if change == "source":
|
|
row["Config"]["Labels"]["com.docker.compose.project.config_files"] = (
|
|
"/foreign/compose.yaml"
|
|
)
|
|
elif change == "engine":
|
|
info["ID"] = "private"
|
|
elif change == "root":
|
|
info["DockerRootDir"] = module.ROOTLESS_STORAGE
|
|
elif change == "cid":
|
|
row["Id"] = "a" * 64
|
|
elif change == "health":
|
|
row["State"]["Health"]["Status"] = "unhealthy"
|
|
elif change == "mount":
|
|
row["Mounts"][0]["Source"] = "/private-data"
|
|
elif change == "network":
|
|
row["NetworkSettings"]["Networks"]["workmesh_default"]["NetworkID"] = "b" * 64
|
|
elif change == "extra-product":
|
|
inventory.append("f" * 64)
|
|
with pytest.raises(module.QuiescenceError):
|
|
module.ControllerDriver({"jens/workmesh": target})._product_binding(
|
|
{"engineId": "private"}, private
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def real_guards(module, proven, monkeypatch, tmp_path):
|
|
import hashlib
|
|
|
|
driver = module.ControllerDriver({})
|
|
monkeypatch.setattr(driver, "assert_barrier", lambda: None)
|
|
monkeypatch.setattr(
|
|
driver, "registrations", lambda: copy.deepcopy(proven[0].runners)
|
|
)
|
|
monkeypatch.setattr(driver, "attest_registrations", lambda rows: None)
|
|
monkeypatch.setattr(driver, "pending_jobs", lambda: [])
|
|
sources = {}
|
|
for name in ("runner", "host"):
|
|
stanza = (
|
|
"if ! flock -n 8; then\nexit 0\nfi"
|
|
if name == "runner"
|
|
else "flock -n 8 || exit 0"
|
|
)
|
|
raw = (
|
|
'#!/bin/bash\nreadonly MAINTENANCE_LOCK="/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock"\nexec 8>"$MAINTENANCE_LOCK"\n'
|
|
+ stanza
|
|
+ "\n# private SECRET-CANARY\n"
|
|
).encode()
|
|
path = tmp_path / name
|
|
path.write_bytes(raw)
|
|
sources[name] = (path, hashlib.sha256(raw).hexdigest())
|
|
monkeypatch.setattr(module, "GUARDS", sources)
|
|
monkeypatch.setattr(
|
|
module.fleet, "_read_source_file", lambda path, **kw: path.read_bytes()
|
|
)
|
|
|
|
def write(path, raw, *, mode, expected_sha256):
|
|
module._require(
|
|
hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256,
|
|
"guard_cas_failed",
|
|
)
|
|
path.write_bytes(raw)
|
|
|
|
monkeypatch.setattr(module.fleet, "_atomic_write_owned", write)
|
|
# Only the filesystem-independent binding is exercised on Windows here.
|
|
monkeypatch.setattr(
|
|
module.os, "fstat", lambda fd: type("Info", (), {"st_dev": 1, "st_ino": 2})()
|
|
)
|
|
return driver, sources, proven[1]
|
|
|
|
|
|
def test_actual_guard_wrapper_owned_cas_and_exact_restore(module, real_guards):
|
|
driver, sources, store = real_guards
|
|
captured = driver.capture_guards()
|
|
value = {
|
|
"guards": captured,
|
|
"guardProgress": {},
|
|
"registrations": driver.registrations(),
|
|
"registrationProgress": {},
|
|
"progress": {row["id"]: "quiesced" for row in module._fixed_rows()},
|
|
}
|
|
store["value"] = copy.deepcopy(value)
|
|
|
|
def persist():
|
|
store["value"] = copy.deepcopy(value)
|
|
|
|
driver.install_guards(value, persist)
|
|
assert all(
|
|
module.GUARD_MARKER.encode() in path.read_bytes()
|
|
for path, _ in sources.values()
|
|
)
|
|
value["registrationProgress"] = {"9": "restored", "22": "restored"}
|
|
persist()
|
|
driver.restore_guards(value, persist)
|
|
assert value["guardProgress"] == {"runner": "restored", "host": "restored"}
|
|
assert all(
|
|
module.hashlib.sha256(path.read_bytes()).hexdigest() == digest
|
|
for path, digest in sources.values()
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"stage", [None, "install_intent", "installed", "restore_intent", "restored"]
|
|
)
|
|
def test_foreign_guard_changes_are_never_adopted_or_overwritten(
|
|
module, real_guards, stage
|
|
):
|
|
driver, sources, store = real_guards
|
|
captured = driver.capture_guards()
|
|
store["value"] = {
|
|
"guards": captured,
|
|
"guardProgress": {"runner": stage} if stage else {},
|
|
}
|
|
path = sources["runner"][0]
|
|
path.write_bytes(path.read_bytes() + b"foreign-change")
|
|
before = path.read_bytes()
|
|
with pytest.raises(module.QuiescenceError, match="dispatch_guard_changed"):
|
|
driver._guard_binding()
|
|
assert path.read_bytes() == before
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"failure", ["short-write", "temp-fsync", "before-link", "after-link"]
|
|
)
|
|
def test_initial_receipt_atomic_publication_faults(
|
|
module, monkeypatch, tmp_path, failure
|
|
):
|
|
import json
|
|
|
|
monkeypatch.setattr(module, "STATE_PATH", tmp_path / "receipt.json")
|
|
monkeypatch.setattr(module.fleet, "_trusted_directory", lambda path: None)
|
|
monkeypatch.setattr(
|
|
module.fleet, "_trusted_private_file", lambda path, **kw: path.read_bytes()
|
|
)
|
|
receipt = {
|
|
"schema": module.SCHEMA,
|
|
"revision": "d" * 40,
|
|
"manifestDigest": module._digest(module.OBSERVATION),
|
|
"phase": "guarding",
|
|
"baseline": {},
|
|
"registrations": [],
|
|
"registrationProgress": {},
|
|
"progress": {},
|
|
"guards": {"runner": {}, "host": {}},
|
|
"guardProgress": {},
|
|
}
|
|
real_open, real_link, real_fsync = module.os.fdopen, module.os.link, module.os.fsync
|
|
if failure == "short-write":
|
|
|
|
class Short:
|
|
def __init__(self, fd, mode):
|
|
self.stream = real_open(fd, mode)
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self.stream.close()
|
|
|
|
def write(self, raw):
|
|
return self.stream.write(raw[:1])
|
|
|
|
monkeypatch.setattr(module.os, "fdopen", Short)
|
|
elif failure == "temp-fsync":
|
|
monkeypatch.setattr(
|
|
module.os, "fsync", lambda fd: (_ for _ in ()).throw(OSError("fault"))
|
|
)
|
|
else:
|
|
|
|
def link(src, dst, **kw):
|
|
if failure == "after-link":
|
|
real_link(src, dst, **kw)
|
|
raise OSError("fault")
|
|
|
|
monkeypatch.setattr(module.os, "link", link)
|
|
with pytest.raises((OSError, module.QuiescenceError)):
|
|
module._save_receipt(receipt, None)
|
|
if failure == "after-link":
|
|
assert json.loads(module.STATE_PATH.read_bytes()) == receipt
|
|
assert module._read_receipt() == receipt
|
|
else:
|
|
assert not module.STATE_PATH.exists()
|
|
assert list(tmp_path.glob(".acceptance-receipt-*")) == []
|
|
|
|
|
|
def test_deadline_expires_before_native_read_or_effect(module, monkeypatch):
|
|
driver = module.ControllerDriver({})
|
|
driver._deadline = module.time.monotonic() - 1
|
|
monkeypatch.setattr(
|
|
module.fleet, "_run", lambda *a, **kw: pytest.fail("no effect after deadline")
|
|
)
|
|
monkeypatch.setattr(
|
|
module.fleet,
|
|
"_api_json",
|
|
lambda *a, **kw: pytest.fail("no read after deadline"),
|
|
)
|
|
with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"):
|
|
driver._run(["docker", "info"])
|
|
with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"):
|
|
driver._api_json("/admin/actions/runners")
|
|
|
|
|
|
def test_readonly_poll_reports_real_partial_and_complete_without_resuming(
|
|
module, proven
|
|
):
|
|
driver, receipt = proven
|
|
driver.timeout = True
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert module.operation_status()["status"] == "partial"
|
|
assert module.operation_status(active=True)["status"] == "in_progress"
|
|
before = len(driver.effects)
|
|
assert module.operation_status()["phase"] == "quiescing"
|
|
assert len(driver.effects) == before
|
|
driver.timeout = False
|
|
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
|
assert module.operation_status()["status"] == "quiesced"
|
|
assert module.operation_status()["completedContainers"] == 18
|
|
|
|
|
|
def test_external_migration_lock_is_nonblocking_and_readonly_busy(module, proven, monkeypatch):
|
|
def busy(**kwargs):
|
|
assert kwargs == {"blocking": False}
|
|
raise module.fleet.RunnerFleetSpecializeError("runner_fleet_specialization_recovery_busy")
|
|
monkeypatch.setattr(module.fleet, "_migration_lock", busy)
|
|
driver, _ = proven
|
|
result = module.preflight(revision="d"*40, projects={}, driver=driver)
|
|
assert result["status"] == "in_progress" and result["effectsPerformed"] is False
|
|
assert driver.effects == []
|