from __future__ import annotations import copy import importlib.util import sys from contextlib import nullcontext from pathlib import Path import pytest BROKER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker" @pytest.fixture def module(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) spec = importlib.util.spec_from_file_location( "acceptance_quiescence_test", BROKER / "runner_acceptance_quiescence.py" ) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) monkeypatch.setattr(module.fleet, "_migration_lock", lambda **kwargs: nullcontext()) return module class Driver: def __init__(self, module): self.module = module self.state = { "daemon": { "legacyAttested": True, "engineId": "fixed-private-engine", "outerId": "a" * 64, "dockerRootDir": module.ROOTLESS_STORAGE, "rootless": True, }, "productExclusionVerified": True, "dispatchBarrierVerified": True, "resources": { "attachmentsVerified": True, "ownershipVerified": True, "networks": ["fixed-network"], "volumes": ["preserved-data"], }, "rows": copy.deepcopy(module._fixed_rows()), "origins": {}, } for row in self.state["rows"]: row["ownerLabelDigest"] = "b" * 64 if row["ownerLabelPresent"] else None row["labelDigest"] = module._digest(row["composeLabels"]) for project, claim in module.ORIGINS.items(): if claim is None: self.state["origins"][project] = None continue attempt = claim["attempt"] if claim["attempt"] is not None else 0 self.state["origins"][project] = { "repository": "jens/workmesh", "run": claim["run"], "head": claim["head"], "status": "completed", "conclusion": "failure", "attempt": attempt, "task": claim["task"], "taskHead": claim["head"], "taskAttempt": attempt, "jobsTerminal": True, } self.runners = [ { "id": 9, "name": "legacy-linux-1", "labels": ["linux", "docker"], "busy": False, "disabled": False, }, { "id": 22, "name": "legacy-linux-2", "labels": ["linux", "docker"], "busy": False, "disabled": False, }, { "id": 700, "name": "windows-preserved", "labels": ["windows"], "busy": False, "disabled": False, }, ] self.effects = [] self.pending = [] self.timeout = False self.after_term = None def maintenance(self): return nullcontext() def capture_guards(self): return {"runner": {"synthetic": True}, "host": {"synthetic": True}} def install_guards(self, receipt, persist): for name in ("runner", "host"): if receipt["guardProgress"].get(name) in { "installed", "restore_intent", "restored", }: continue receipt["guardProgress"][name] = "install_intent" persist() receipt["guardProgress"][name] = "installed" persist() def restore_guards(self, receipt, persist): for name in ("runner", "host"): if receipt["guardProgress"].get(name) == "restored": continue receipt["guardProgress"][name] = "restore_intent" persist() receipt["guardProgress"][name] = "restored" persist() def snapshot(self): return copy.deepcopy(self.state) def registrations(self): return copy.deepcopy(self.runners) def attest_registrations(self, rows): assert {row["id"] for row in rows if "linux" in row["labels"]} == {9, 22} def pending_jobs(self): return copy.deepcopy(self.pending) def pause(self): pass def set_registration_disabled(self, original, value): self.effects.append(("registration", original["id"], value)) next(row for row in self.runners if row["id"] == original["id"])["disabled"] = ( value ) def disable_restart(self, cid): self.effects.append(("restart", cid)) next(row for row in self.state["rows"] if row["id"] == cid)["restartPolicy"] = { "Name": "no", "MaximumRetryCount": 0, } def term(self, cid): row = next(row for row in self.state["rows"] if row["id"] == cid) if row["state"]["Running"]: self.effects.append(("TERM", cid)) if not self.timeout: row["state"].update( { "Running": False, "Status": "exited", "Pid": 0, "FinishedAt": "2026-10-09T20:00:00Z", } ) if self.after_term: self.after_term(self) def wait_exited(self, cid): return not self.timeout @pytest.fixture def proven(module, monkeypatch): receipt = {"value": None} monkeypatch.setattr( module, "_read_receipt", lambda: copy.deepcopy(receipt["value"]) ) def save(value, expected): current = receipt["value"] actual = module._digest(current) if current is not None else None module._require(actual == expected, "maintenance_receipt_cas_failed") receipt["value"] = copy.deepcopy(value) return module._digest(value) monkeypatch.setattr(module, "_save_receipt", save) return Driver(module), receipt def test_missing_backup_origin_never_becomes_authority(module, monkeypatch): monkeypatch.setitem(module.ORIGINS, "workmesh-backup-exercise-39a1a242e4b9", None) monkeypatch.setattr(module, "_read_receipt", lambda: None) driver = Driver(module) result = module.preflight(revision="d" * 40, projects={}, driver=driver) assert result["status"] == "blocked" assert result["effectsPerformed"] is False assert driver.effects == [] def test_positive_preserves_resources_and_windows_then_idempotent(module, proven): driver, receipt = proven data_before = copy.deepcopy(driver.state["resources"]) ids_before = {row["id"] for row in driver.state["rows"]} assert ( module.preflight(revision="d" * 40, projects={}, driver=driver)["status"] == "ready" ) result = module.quiesce(revision="d" * 40, projects={}, driver=driver) assert result["status"] == "quiesced" assert result["preservedData"] is True assert driver.state["resources"] == data_before assert {row["id"] for row in driver.state["rows"]} == ids_before assert all(not row["state"]["Running"] for row in driver.state["rows"]) assert sum(effect[0] == "TERM" for effect in driver.effects) == 12 assert driver.runners[2]["disabled"] is False assert all( effect[0] in {"registration", "restart", "TERM"} for effect in driver.effects ) assert receipt["value"]["phase"] == "complete" count = len(driver.effects) assert ( module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"] == "already_quiesced" ) assert len(driver.effects) == count with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"): module.quiesce(revision="e" * 40, projects={}, driver=driver) @pytest.fixture def smart_database(proven): # Synthetic dependency graph over two existing running identities; this # models retained sessions, not a claim that mock-ats uses this production DB. driver, receipt = proven project = "workmesh-runtime-19421" pg = next(row for row in driver.state["rows"] if row["composeLabels"]["com.docker.compose.project"] == project and row["composeLabels"]["com.docker.compose.service"] == "postgres") clients = [row for row in driver.state["rows"] if row["composeLabels"]["com.docker.compose.project"] == project and row["composeLabels"]["com.docker.compose.service"] in {"api", "mock-ats"}] assert len(clients) == 2 and all(row["state"]["Running"] for row in clients) driver.state["rows"] = [pg] + [row for row in driver.state["rows"] if row is not pg] original_term = driver.term pending = False waits = [] def term(cid): nonlocal pending if cid == pg["id"] and pg["state"]["Running"]: pending = True driver.effects.append(("TERM", cid)) else: original_term(cid) if pending and all(not row["state"]["Running"] for row in clients): pg["state"].update(Running=False, Status="exited", Pid=0, FinishedAt="2026-10-10T00:00:00Z") def wait(cid): waits.append(sum(effect[0] == "TERM" for effect in driver.effects)) return not next(row for row in driver.state["rows"] if row["id"] == cid)["state"]["Running"] driver.term, driver.wait_exited = term, wait return driver, receipt, pg, clients, waits def test_two_clients_receive_term_before_waiting_for_smart_database_shutdown(module, smart_database): driver, receipt, pg, clients, waits = smart_database result = module.quiesce(revision="d" * 40, projects={}, driver=driver) assert result["status"] == "quiesced" assert all(not row["state"]["Running"] for row in [pg, *clients]) assert waits and all(count == 12 for count in waits) assert receipt["value"]["phase"] == "complete" assert sum(effect[0] == "TERM" for effect in driver.effects) == 12 def test_old_partial_receipt_resumes_two_passes_without_rewriting_origin(module, smart_database, monkeypatch): from types import SimpleNamespace driver, holder, pg, clients, _ = smart_database baseline = copy.deepcopy(driver.state) original = { "schema": module.SCHEMA, "revision": module.REVIEWED_RECEIPT_PREDECESSOR, "manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing", "baseline": baseline, "registrations": copy.deepcopy(driver.runners[:2]), "registrationProgress": {"9": "disabled", "22": "disabled"}, "progress": {pg["id"]: "term_intent"}, "guards": driver.capture_guards(), "guardProgress": {"runner": "installed", "host": "installed"}, } driver.disable_restart(pg["id"]) driver.term(pg["id"]) assert pg["state"]["Running"] and all(row["state"]["Running"] for row in clients) for runner in driver.runners[:2]: runner["disabled"] = True holder["value"] = original frozen = (original["revision"], original["manifestDigest"], module._digest(baseline)) release = {"controllerVersion": "2.1.21", "sourceRepository": module.fleet.SOURCE_REPOSITORY, "sourceRevision": "e" * 40, "sourceTree": "f" * 40} monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release)) driver.accepts_reviewed_receipt_predecessor = module.ControllerDriver({}).accepts_reviewed_receipt_predecessor assert module.preflight(revision="e" * 40, projects={}, driver=driver)["status"] == "partial" assert module.quiesce(revision="e" * 40, projects={}, driver=driver)["status"] == "quiesced" final = holder["value"] assert (final["revision"], final["manifestDigest"], module._digest(final["baseline"])) == frozen assert all(not row["state"]["Running"] for row in [pg, *clients]) assert module.quiesce(revision="e" * 40, projects={}, driver=driver)["status"] == "already_quiesced" @pytest.mark.parametrize("change", ["origin", "manifest", "revision", "tree", "repository", "version", "missing-reader"]) def test_predecessor_resume_rejects_foreign_receipt_or_current_source(module, proven, monkeypatch, change): from types import SimpleNamespace driver, holder = proven driver.timeout = True module.quiesce(revision=module.REVIEWED_RECEIPT_PREDECESSOR, projects={}, driver=driver) release = {"controllerVersion": "2.1.21", "sourceRepository": module.fleet.SOURCE_REPOSITORY, "sourceRevision": "e" * 40, "sourceTree": "f" * 40} monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release)) if change == "origin": holder["value"]["revision"] = "a" * 40 elif change == "manifest": holder["value"]["manifestDigest"] = "a" * 64 elif change == "revision": release["sourceRevision"] = "a" * 40 elif change == "tree": release["sourceTree"] = "invalid" elif change == "repository": release["sourceRepository"] = "jens/foreign" elif change == "version": release["controllerVersion"] = "2.1.22" if change != "missing-reader": driver.accepts_reviewed_receipt_predecessor = module.ControllerDriver({}).accepts_reviewed_receipt_predecessor before = len(driver.effects) with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"): module.quiesce(revision="e" * 40, projects={}, driver=driver) assert len(driver.effects) == before def test_predecessor_failure_projection_preserves_effect_uncertainty(module, proven): driver, _ = proven driver.timeout = True module.quiesce(revision=module.REVIEWED_RECEIPT_PREDECESSOR, projects={}, driver=driver) result = module.failure_projection("e" * 40) assert result["status"] == "partial" and result["guardHeld"] is True assert result["effectsPerformed"] is True def test_mount_order_is_canonical_for_fixed_current_and_resume_baseline(module, proven): driver, _ = proven baseline = copy.deepcopy(driver.state) row = next(row for row in driver.state["rows"] if len(row["mounts"]) >= 2) row["mounts"].reverse() module.attest_snapshot(driver.state) module.attest_snapshot(driver.state, baseline=baseline, progress={}) assert module.preflight(revision="d" * 40, projects={}, driver=driver)["status"] == "ready" assert module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"] == "quiesced" @pytest.mark.parametrize("field,value", [("Source", "/foreign"), ("Destination", "/foreign"), ("Type", "bind"), ("Name", "foreign-volume"), ("RW", False)]) def test_mount_order_normalization_preserves_all_identity_fields(module, proven, field, value): driver, _ = proven row = next(row for row in driver.state["rows"] if any(mount["Type"] == "volume" and mount["RW"] is True for mount in row["mounts"])) mount = next(mount for mount in row["mounts"] if mount["Type"] == "volume" and mount["RW"] is True) mount[field] = value row["mounts"].reverse() with pytest.raises(module.QuiescenceError, match="container_identity_changed"): module.attest_snapshot(driver.state) assert driver.effects == [] def test_mount_normalization_rejects_duplicate_destinations(module, proven): driver, _ = proven row = next(row for row in driver.state["rows"] if row["mounts"]) row["mounts"].append(copy.deepcopy(row["mounts"][0])) with pytest.raises(module.QuiescenceError, match="container_mount_inventory_invalid"): module.attest_snapshot(driver.state) assert driver.effects == [] @pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"]) def test_initial_new_work_refuses_all_effects(module, proven, kind): driver, _ = proven driver.pending = [{"id": 99, "status": kind}] with pytest.raises(module.QuiescenceError, match="global_ci_work_present"): module.quiesce(revision="d" * 40, projects={}, driver=driver) assert driver.effects == [] @pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"]) def test_new_work_during_execution_blocks_next_container(module, proven, kind): driver, receipt = proven driver.after_term = lambda self: setattr( self, "pending", [{"id": 99, "status": kind}] ) with pytest.raises(module.QuiescenceError, match="global_ci_work_present"): module.quiesce(revision="d" * 40, projects={}, driver=driver) assert sum(e[0] == "restart" for e in driver.effects) == 1 assert receipt["value"]["phase"] != "complete" assert all(row["disabled"] for row in driver.runners[:2]) @pytest.mark.parametrize( "change", [ lambda s: s["daemon"].update({"rootless": False}), lambda s: s["daemon"].update({"dockerRootDir": "/srv/example/appdata/production"}), lambda s: s["rows"][0].update({"id": "f" * 64}), lambda s: s["rows"][0].update({"imageId": "sha256:" + "f" * 64}), lambda s: s["rows"][0]["composeLabels"].update( {"com.docker.compose.config-hash": "f" * 64} ), lambda s: s["rows"][0]["composeLabels"].update( {"com.docker.compose.project.working_dir": "/production"} ), lambda s: s["rows"][0]["mounts"].append( {"Type": "bind", "Source": "/production", "RW": True} ), lambda s: s["rows"][0]["networks"].update({"foreign": "f" * 64}), lambda s: s["resources"].update({"attachmentsVerified": False}), lambda s: s["resources"].update({"ownershipVerified": False}), lambda s: s.update({"productExclusionVerified": False}), lambda s: s.update({"dispatchBarrierVerified": False}), lambda s: s["rows"].append(copy.deepcopy(s["rows"][0])), lambda s: s["rows"][0].update({"ownerLabelDigest": "e" * 64}), ], ) def test_identity_or_scope_drift_has_no_effect(module, proven, change): driver, _ = proven change(driver.state) with pytest.raises(module.QuiescenceError): module.quiesce(revision="d" * 40, projects={}, driver=driver) assert driver.effects == [] @pytest.mark.parametrize( "field,value", [ ("attempt", None), ("attempt", 2), ("taskAttempt", 3), ("task", 999), ("head", "e" * 40), ("jobsTerminal", False), ], ) def test_origin_drift_refuses_effects(module, proven, field, value): driver, _ = proven driver.state["origins"]["workmesh-product-21300-7bace5f2edf0d259"][field] = value with pytest.raises(module.QuiescenceError): module.quiesce(revision="d" * 40, projects={}, driver=driver) assert driver.effects == [] def test_legacy_attempt_zero_is_actual_sentinel_and_missing_is_not_assumed( module, proven ): driver, _ = proven module.ORIGINS["workmesh-runtime-19421"]["attempt"] = 0 driver.state["origins"]["workmesh-runtime-19421"].update(attempt=0, taskAttempt=0) module.attest_snapshot(driver.snapshot()) driver.state["origins"]["workmesh-runtime-19421"]["attempt"] = None with pytest.raises(module.QuiescenceError, match="attempt_unavailable"): module.attest_snapshot(driver.snapshot()) def test_timeout_retains_owned_guard_and_resumes_only_own_transition(module, proven): driver, receipt = proven driver.timeout = True result = module.quiesce(revision="d" * 40, projects={}, driver=driver) assert result["status"] == "partial" and result["guardHeld"] is True assert receipt["value"]["progress"][driver.state["rows"][0]["id"]] == "term_intent" driver.timeout = False result = module.quiesce(revision="d" * 40, projects={}, driver=driver) assert result["status"] == "quiesced" def test_registration_drift_after_partial_never_restores_foreign_identity( module, proven ): driver, _ = proven driver.timeout = True module.quiesce(revision="d" * 40, projects={}, driver=driver) before = len(driver.effects) driver.runners[0]["labels"].append("foreign-project") with pytest.raises(module.QuiescenceError, match="legacy_registration_changed"): module.quiesce(revision="d" * 40, projects={}, driver=driver) assert len(driver.effects) == before def test_no_arbitrary_or_destructive_docker_effect(module, monkeypatch): calls = [] class Result: returncode = 0 monkeypatch.setattr( module.fleet, "_run", lambda args, **kwargs: calls.append(args) or Result() ) driver = module.ControllerDriver({}) monkeypatch.setattr(driver, "assert_barrier", lambda: None) monkeypatch.setattr(driver, "_guard_binding", lambda: {}) monkeypatch.setattr(driver, "_effect_admission", lambda cid, args: None) cid = module._fixed_rows()[0]["id"] driver.disable_restart(cid) assert calls == [ [ "docker", "exec", "-e", "DOCKER_HOST=tcp://127.0.0.1:2375", "gitea-runner-dind", "docker", "update", "--restart=no", cid, ] ] with pytest.raises(module.QuiescenceError, match="container_target_invalid"): driver.disable_restart("user-selected") def test_public_preflight_never_serializes_private_canary(module): class BadDriver: def maintenance(self): return nullcontext() def pending_jobs(self): raise ValueError("Bearer SECRET-CANARY") def proof_gaps(self): return ["terminal_origin_unavailable"] result = module.preflight(revision="d" * 40, projects={}, driver=BadDriver()) assert "SECRET-CANARY" not in str(result) assert result["effectsPerformed"] is False @pytest.mark.parametrize("boundary", range(1, 93)) def test_every_durable_write_boundary_resumes_without_adoption( module, proven, monkeypatch, boundary ): driver, receipt = proven save = module._save_receipt count = 0 def crash(value, expected): nonlocal count count += 1 result = save(value, expected) if count == boundary: raise OSError("simulated power loss after durable commit") return result monkeypatch.setattr(module, "_save_receipt", crash) try: module.quiesce(revision="d" * 40, projects={}, driver=driver) except OSError: pass monkeypatch.setattr(module, "_save_receipt", save) result = module.quiesce(revision="d" * 40, projects={}, driver=driver) assert result["status"] in {"quiesced", "already_quiesced"} assert receipt["value"]["phase"] == "complete" assert all(not row["state"]["Running"] for row in driver.state["rows"]) assert sum(e[0] == "TERM" for e in driver.effects) == 12 @pytest.mark.parametrize( "effect", ["set_registration_disabled", "disable_restart", "term"] ) def test_effect_commit_before_receipt_completion_is_owned_resume( module, proven, monkeypatch, effect ): driver, _ = proven original = getattr(driver, effect) crashed = False def wrapper(*args): nonlocal crashed original(*args) if not crashed: crashed = True raise OSError("simulated loss after external effect") monkeypatch.setattr(driver, effect, wrapper) with pytest.raises(OSError): module.quiesce(revision="d" * 40, projects={}, driver=driver) monkeypatch.setattr(driver, effect, original) assert ( module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"] == "quiesced" ) def test_current_fixed_origins_include_reviewed_backup_and_attempts(module): assert module.ControllerDriver({}).proof_gaps() == [] assert module.ORIGINS["workmesh-backup-exercise-39a1a242e4b9"]["run"] == 20184 assert all(row["attempt"] == 1 for row in module.ORIGINS.values()) def test_malformed_complete_receipt_cannot_reenable_maintenance(module, proven): driver, receipt = proven module.quiesce(revision="d" * 40, projects={}, driver=driver) assert module.maintenance_fenced() is False receipt["value"]["progress"].pop(next(iter(receipt["value"]["progress"]))) assert module.maintenance_fenced() is True def test_failure_projection_never_misreports_partial_effects(module, proven): driver, _ = proven assert module.failure_projection("d" * 40)["effectsPerformed"] is False driver.timeout = True module.quiesce(revision="d" * 40, projects={}, driver=driver) result = module.failure_projection("d" * 40) assert result["status"] == "partial" assert result["effectsPerformed"] is True and result["guardHeld"] is True @pytest.fixture def native_origins(module, monkeypatch): dates = { 19421: ("2026-09-29T00:53:46Z", "2026-09-29T00:57:44Z", 768), 19617: ("2026-09-29T07:36:50Z", "2026-09-29T07:41:39Z", 834), 20184: ("2026-09-29T22:34:11Z", "2026-09-29T22:35:35Z", 941), 21300: ("2026-10-01T20:53:43Z", "2026-10-01T20:54:47Z", 1128), } tasks, runs, jobs = [], {}, {} for claim in module.ORIGINS.values(): start, end, number = dates[claim["run"]] conclusion = "cancelled" if claim["run"] == 20184 else "failure" tasks.append( { "id": claim["task"], "run_number": number, "head_sha": claim["head"], "workflow_id": "runtime-acceptance.yml", "status": conclusion, "created_at": start, "run_started_at": start, "updated_at": end, "url": f"http://127.0.0.1:3000/Jens/WorkMesh/actions/runs/{claim['run']}", } ) runs[claim["run"]] = { "id": claim["run"], "run_number": number, "head_sha": claim["head"], "status": "completed", "conclusion": conclusion, "run_attempt": 1, "run_started_at": None, "updated_at": end, } jobs[claim["run"]] = [ { "id": claim["job"], "run_id": claim["run"], "head_sha": claim["head"], "run_attempt": 1, "runner_id": claim["runner"], "started_at": start, "status": "completed", "conclusion": conclusion, } ] # Pagination includes unrelated metadata but never returns it publicly. tasks.extend({"id": i} for i in range(50000, 50055)) calls = [] def api(path, **kwargs): calls.append(path) if "/actions/tasks?" in path: page = int(path.rsplit("=", 1)[-1]) return { "total_count": len(tasks), "workflow_runs": copy.deepcopy(tasks[(page - 1) * 50 : page * 50]), } run = int(path.split("/runs/")[1].split("/")[0]) return ( {"total_count": len(jobs[run]), "jobs": copy.deepcopy(jobs[run])} if "/jobs?" in path else copy.deepcopy(runs[run]) ) monkeypatch.setattr(module.fleet, "_api_json", api) return tasks, runs, jobs, calls def test_native_documented_task_binding_current_attempt_and_null_run_start( module, native_origins ): result = module.ControllerDriver({})._origins() module._attest_origins(result) assert all(row["attempt"] == 1 and row["jobsTerminal"] for row in result.values()) assert len([path for path in native_origins[3] if "/tasks?" in path]) == 2 @pytest.mark.parametrize( "change", [ "task-url", "task-head", "task-workflow", "task-time", "foreign-job", "duplicate-job-start", "later-attempt", "active-job", "missing-job", "missing-attempt", "terminal-task-changed", "duplicate-task", ], ) def test_native_origin_drift_refuses_authority(module, native_origins, change): tasks, runs, jobs, _ = native_origins claim = module.ORIGINS[next(iter(module.ORIGINS))] task = next(row for row in tasks if row["id"] == claim["task"]) job = jobs[claim["run"]][0] if change == "task-url": task["url"] = task["url"].replace("127.0.0.1", "foreign.invalid") elif change == "task-head": task["head_sha"] = "f" * 40 elif change == "task-workflow": task["workflow_id"] = "other.yml" elif change == "task-time": task["created_at"] = "2026-10-02T00:00:00Z" elif change == "foreign-job": job["runner_id"] = 800 elif change == "duplicate-job-start": jobs[claim["run"]].append(copy.deepcopy(job)) elif change == "later-attempt": runs[claim["run"]]["run_attempt"] = 2 elif change == "active-job": job.update(status="in_progress", conclusion=None) elif change == "missing-job": jobs[claim["run"]] = [] elif change == "missing-attempt": runs[claim["run"]].pop("run_attempt") elif change == "terminal-task-changed": task["status"] = "success" elif change == "duplicate-task": tasks[-1] = copy.deepcopy(tasks[0]) with pytest.raises(module.QuiescenceError): module._attest_origins(module.ControllerDriver({})._origins()) def test_exact_inspect_inventory_cannot_omit_foreign_volume_attachment( module, monkeypatch ): driver = module.ControllerDriver({}) ids = ["a" * 64, "b" * 64] monkeypatch.setattr(driver, "_docker", lambda *args: [{"Id": ids[0]}]) with pytest.raises(module.QuiescenceError, match="legacy_inventory_unavailable"): driver._snapshot_from_ids({}, ids) @pytest.fixture def product(module, monkeypatch): source = "/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh" names = [ "workmesh-postgres-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1", ] target = { "slug": "workmesh", "branch": "main", "exact_source": True, "source_dir": source, "env_source": "/srv/example/appdata/workmesh/.env", "compose_files": ["compose.yaml"], "compose_project": "workmesh", "containers": names, } rows = { name: { "Id": str(i + 1) * 64, "Name": "/" + name, "Image": "sha256:" + str(i + 1) * 64, "Created": "2026-09-01T00:00:00Z", "State": {"Running": True, "Health": {"Status": "healthy"}}, "Config": { "Labels": { "com.docker.compose.project": "workmesh", "com.docker.compose.service": name.removeprefix( "workmesh-" ).removesuffix("-1"), "com.docker.compose.project.working_dir": source, "com.docker.compose.project.config_files": source + "/compose.yaml", } }, "Mounts": [ { "Type": "volume", "Name": "production_data", "Source": "/var/lib/docker/volumes/production_data/_data", "Destination": "/data", "RW": True, } ], "NetworkSettings": { "Networks": {"workmesh_default": {"NetworkID": "e" * 64}} }, } for i, name in enumerate(names) } info = {"ID": "host-engine", "DockerRootDir": "/var/lib/docker"} inventory = [row["Id"] for row in rows.values()] def run(argv, **kw): import json if argv[1] == "info": return json.dumps(info) if argv[1] == "inspect": return json.dumps([rows[argv[2]]]) assert argv[1:3] == ["ps", "-a"] return "\n".join(inventory) monkeypatch.setattr(module.fleet, "_run", run) return target, rows, info, inventory def test_manual_product_observation_protects_exact_healthy_target_without_fake_receipt( module, product ): target, _, _, _ = product proof = module.ControllerDriver({"jens/workmesh": target})._product_binding( {"engineId": "private"}, [] ) assert ( len(proof["rows"]) == 4 and proof["receiptStatus"] == "legacy-physical-observation" ) def test_product_baseline_mount_order_is_stable_without_discarding_fields(module, product): target, rows, _, _ = product row = next(iter(rows.values())) row["Mounts"].append({"Type": "bind", "Source": "/production/bootstrap.sh", "Destination": "/bootstrap.sh", "RW": False}) driver = module.ControllerDriver({"jens/workmesh": target}) baseline = driver._product_binding({"engineId": "private"}, []) row["Mounts"].reverse() assert driver._product_binding({"engineId": "private"}, []) == baseline row["Mounts"][0]["RW"] = True assert driver._product_binding({"engineId": "private"}, []) != baseline @pytest.fixture def product_state(module, monkeypatch, tmp_path): import os import stat from types import SimpleNamespace path = tmp_path / "workmesh.json" path.write_bytes(b'{"status":"healthy","live_legacy_provenance":true}') path.chmod(0o644) monkeypatch.setattr(module, "WORKMESH_STATE_PATH", path) original_lstat, original_fstat = Path.lstat, os.fstat changes = {"st_uid": 0, "st_mode": stat.S_IFREG | 0o644, "st_nlink": 1} fields = ("st_dev", "st_ino", "st_uid", "st_gid", "st_mode", "st_nlink", "st_size", "st_mtime_ns", "st_ctime_ns") def metadata(info): # Windows cannot express POSIX ownership/modes; retain real file # identity/size/timestamps and exercise the same strict native contract. values = {name: getattr(info, name) for name in fields} if os.name == "nt": # Windows reports different ctime precision for path/FD queries. # The deployed Linux reader still compares the real nanosecond value. values["st_ctime_ns"] = original_lstat(path).st_ctime_ns return SimpleNamespace(**{**values, **changes}) monkeypatch.setattr(Path, "lstat", lambda self, **kw: metadata(original_lstat(self, **kw)) if self == path else original_lstat(self, **kw)) monkeypatch.setattr(os, "fstat", lambda fd: metadata(original_fstat(fd))) return path, changes @pytest.mark.parametrize("mode", [0o600, 0o644]) def test_product_state_accepts_exact_normal_producer_modes(module, product, product_state, mode): import stat target, _, _, _ = product _, changes = product_state changes["st_mode"] = stat.S_IFREG | mode assert module.ControllerDriver({})._product_receipt(target, []) == "legacy-physical-observation" def test_actual_posix_producer_0644_is_readable_without_metadata_shims(module, monkeypatch, tmp_path): import os if os.name == "nt" or os.geteuid() != 0: pytest.skip("actual root-owned POSIX producer proof requires the native root harness") path = tmp_path / "workmesh.json" path.write_bytes(b'{"status":"healthy","live_legacy_provenance":true}') path.chmod(0o644) monkeypatch.setattr(module, "WORKMESH_STATE_PATH", path) assert module.ControllerDriver({})._product_receipt({}, []) == "legacy-physical-observation" with pytest.raises(module.fleet.RunnerFleetSpecializeError): module.fleet._trusted_private_file(path) @pytest.mark.parametrize("mode", [0o400, 0o640, 0o664, 0o666, 0o700, 0o755, 0o4644]) def test_product_metadata_does_not_accept_other_modes(module, product_state, mode): import stat _, changes = product_state changes["st_mode"] = stat.S_IFREG | mode with pytest.raises(module.QuiescenceError, match="active_product_receipt_untrusted"): module._read_workmesh_product_state() @pytest.mark.parametrize("change", ["owner", "symlink", "directory", "hardlink", "empty", "oversized"]) def test_product_metadata_rejects_unsafe_objects(module, product_state, change): import stat _, changes = product_state changes.update({ "owner": {"st_uid": 1}, "symlink": {"st_mode": stat.S_IFLNK | 0o644}, "directory": {"st_mode": stat.S_IFDIR | 0o644}, "hardlink": {"st_nlink": 2}, "empty": {"st_size": 0}, "oversized": {"st_size": 512 * 1024 + 1}, }[change]) with pytest.raises(module.QuiescenceError, match="active_product_receipt_untrusted"): module._read_workmesh_product_state() @pytest.mark.parametrize("stage", ["open", "read", "path"]) @pytest.mark.parametrize("field", ["st_ino", "st_dev", "st_uid", "st_size", "st_mtime_ns", "st_ctime_ns"]) def test_product_metadata_rejects_open_read_and_replace_races(module, product_state, monkeypatch, stage, field): path, _ = product_state original_fstat, original_lstat = module.os.fstat, Path.lstat calls = 0 def fstat(fd): nonlocal calls calls += 1 info = original_fstat(fd) if (stage == "open" and calls == 1) or (stage == "read" and calls == 2): setattr(info, field, getattr(info, field) + 1) return info def lstat(self, **kw): info = original_lstat(self, **kw) if stage == "path" and self == path and calls >= 2: setattr(info, field, getattr(info, field) + 1) return info monkeypatch.setattr(module.os, "fstat", fstat) monkeypatch.setattr(Path, "lstat", lstat) with pytest.raises(module.QuiescenceError, match="active_product_receipt_untrusted"): module._read_workmesh_product_state() def test_product_metadata_open_is_nofollow_and_size_bounded(module, product_state, monkeypatch): original_open = module.os.open flags_seen = [] def opening(path, flags, *args, **kw): flags_seen.append(flags) return original_open(path, flags, *args, **kw) monkeypatch.setattr(module.os, "open", opening) assert len(module._read_workmesh_product_state()) < 512 * 1024 assert flags_seen == [module.os.O_RDONLY | getattr(module.os, "O_NOFOLLOW", 0) | getattr(module.os, "O_NONBLOCK", 0)] @pytest.mark.parametrize("valid", [True, False]) def test_public_product_metadata_keeps_existing_receipt_verifier(module, product_state, monkeypatch, valid): import json from types import SimpleNamespace path, _ = product_state path.write_text(json.dumps({"live_receipt": {"receipt": "existing"}}), encoding="utf-8") calls = [] rows = [{"name": "/workmesh-api-1", "imageId": "sha256:" + "a" * 64}] def verified(state, repository, revision, project): calls.append((state, repository, revision, project)) return {"containers": {"workmesh-api-1": {"image_id": rows[0]["imageId"]}}} if valid else None monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace( _live_revision=lambda state: "a" * 40, _verified_receipt=verified)) if valid: assert module.ControllerDriver({})._product_receipt({}, rows)["status"] == "verified-existing" else: with pytest.raises(module.QuiescenceError, match="active_product_receipt_untrusted"): module.ControllerDriver({})._product_receipt({}, rows) assert len(calls) == 1 and calls[0][1] == "jens/workmesh" @pytest.mark.parametrize( "change", ["source", "engine", "root", "cid", "health", "mount", "network", "extra-product"], ) def test_product_disjointness_fails_on_changed_identity_or_overlap( module, product, change ): target, rows, info, inventory = product row = next(iter(rows.values())) private = [ { "id": "a" * 64, "mounts": [{"Source": "/private-data"}], "networks": {"private": "b" * 64}, } ] if change == "source": row["Config"]["Labels"]["com.docker.compose.project.config_files"] = ( "/foreign/compose.yaml" ) elif change == "engine": info["ID"] = "private" elif change == "root": info["DockerRootDir"] = module.ROOTLESS_STORAGE elif change == "cid": row["Id"] = "a" * 64 elif change == "health": row["State"]["Health"]["Status"] = "unhealthy" elif change == "mount": row["Mounts"][0]["Source"] = "/private-data" elif change == "network": row["NetworkSettings"]["Networks"]["workmesh_default"]["NetworkID"] = "b" * 64 elif change == "extra-product": inventory.append("f" * 64) with pytest.raises(module.QuiescenceError): module.ControllerDriver({"jens/workmesh": target})._product_binding( {"engineId": "private"}, private ) @pytest.fixture def real_guards(module, proven, monkeypatch, tmp_path): import hashlib driver = module.ControllerDriver({}) monkeypatch.setattr(driver, "assert_barrier", lambda: None) monkeypatch.setattr( driver, "registrations", lambda: copy.deepcopy(proven[0].runners) ) monkeypatch.setattr(driver, "attest_registrations", lambda rows: None) monkeypatch.setattr(driver, "pending_jobs", lambda: []) sources = {} for name in ("runner", "host"): stanza = ( "if ! flock -n 8; then\nexit 0\nfi" if name == "runner" else "flock -n 8 || exit 0" ) raw = ( '#!/bin/bash\nreadonly MAINTENANCE_LOCK="/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock"\nexec 8>"$MAINTENANCE_LOCK"\n' + stanza + "\n# private SECRET-CANARY\n" ).encode() path = tmp_path / name path.write_bytes(raw) sources[name] = (path, hashlib.sha256(raw).hexdigest()) monkeypatch.setattr(module, "GUARDS", sources) monkeypatch.setattr( module.fleet, "_read_source_file", lambda path, **kw: path.read_bytes() ) def write(path, raw, *, mode, expected_sha256): module._require( hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256, "guard_cas_failed", ) path.write_bytes(raw) monkeypatch.setattr(module.fleet, "_atomic_write_owned", write) # Only the filesystem-independent binding is exercised on Windows here. monkeypatch.setattr( module.os, "fstat", lambda fd: type("Info", (), {"st_dev": 1, "st_ino": 2})() ) return driver, sources, proven[1] def test_actual_guard_wrapper_owned_cas_and_exact_restore(module, real_guards): driver, sources, store = real_guards captured = driver.capture_guards() value = { "guards": captured, "guardProgress": {}, "registrations": driver.registrations(), "registrationProgress": {}, "progress": {row["id"]: "quiesced" for row in module._fixed_rows()}, } store["value"] = copy.deepcopy(value) def persist(): store["value"] = copy.deepcopy(value) driver.install_guards(value, persist) assert all( module.GUARD_MARKER.encode() in path.read_bytes() for path, _ in sources.values() ) value["registrationProgress"] = {"9": "restored", "22": "restored"} persist() driver.restore_guards(value, persist) assert value["guardProgress"] == {"runner": "restored", "host": "restored"} assert all( module.hashlib.sha256(path.read_bytes()).hexdigest() == digest for path, digest in sources.values() ) @pytest.mark.parametrize( "stage", [None, "install_intent", "installed", "restore_intent", "restored"] ) def test_foreign_guard_changes_are_never_adopted_or_overwritten( module, real_guards, stage ): driver, sources, store = real_guards captured = driver.capture_guards() store["value"] = { "guards": captured, "guardProgress": {"runner": stage} if stage else {}, } path = sources["runner"][0] path.write_bytes(path.read_bytes() + b"foreign-change") before = path.read_bytes() with pytest.raises(module.QuiescenceError, match="dispatch_guard_changed"): driver._guard_binding() assert path.read_bytes() == before @pytest.mark.parametrize( "failure", ["short-write", "temp-fsync", "before-link", "after-link"] ) def test_initial_receipt_atomic_publication_faults( module, monkeypatch, tmp_path, failure ): import json monkeypatch.setattr(module, "STATE_PATH", tmp_path / "receipt.json") monkeypatch.setattr(module.fleet, "_trusted_directory", lambda path: None) monkeypatch.setattr( module.fleet, "_trusted_private_file", lambda path, **kw: path.read_bytes() ) receipt = { "schema": module.SCHEMA, "revision": "d" * 40, "manifestDigest": module._digest(module.OBSERVATION), "phase": "guarding", "baseline": {}, "registrations": [], "registrationProgress": {}, "progress": {}, "guards": {"runner": {}, "host": {}}, "guardProgress": {}, } real_open, real_link, real_fsync = module.os.fdopen, module.os.link, module.os.fsync if failure == "short-write": class Short: def __init__(self, fd, mode): self.stream = real_open(fd, mode) def __enter__(self): return self def __exit__(self, *args): self.stream.close() def write(self, raw): return self.stream.write(raw[:1]) monkeypatch.setattr(module.os, "fdopen", Short) elif failure == "temp-fsync": monkeypatch.setattr( module.os, "fsync", lambda fd: (_ for _ in ()).throw(OSError("fault")) ) else: def link(src, dst, **kw): if failure == "after-link": real_link(src, dst, **kw) raise OSError("fault") monkeypatch.setattr(module.os, "link", link) with pytest.raises((OSError, module.QuiescenceError)): module._save_receipt(receipt, None) if failure == "after-link": assert json.loads(module.STATE_PATH.read_bytes()) == receipt assert module._read_receipt() == receipt else: assert not module.STATE_PATH.exists() assert list(tmp_path.glob(".acceptance-receipt-*")) == [] def test_deadline_expires_before_native_read_or_effect(module, monkeypatch): driver = module.ControllerDriver({}) driver._deadline = module.time.monotonic() - 1 monkeypatch.setattr( module.fleet, "_run", lambda *a, **kw: pytest.fail("no effect after deadline") ) monkeypatch.setattr( module.fleet, "_api_json", lambda *a, **kw: pytest.fail("no read after deadline"), ) with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"): driver._run(["docker", "info"]) with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"): driver._api_json("/admin/actions/runners") def test_readonly_poll_reports_real_partial_and_complete_without_resuming( module, proven ): driver, receipt = proven driver.timeout = True module.quiesce(revision="d" * 40, projects={}, driver=driver) assert module.operation_status()["status"] == "partial" assert module.operation_status(active=True)["status"] == "in_progress" before = len(driver.effects) assert module.operation_status()["phase"] == "quiescing" assert len(driver.effects) == before driver.timeout = False module.quiesce(revision="d" * 40, projects={}, driver=driver) assert module.operation_status()["status"] == "quiesced" assert module.operation_status()["completedContainers"] == 18 def test_external_migration_lock_is_nonblocking_and_readonly_busy(module, proven, monkeypatch): def busy(**kwargs): assert kwargs == {"blocking": False} raise module.fleet.RunnerFleetSpecializeError("runner_fleet_specialization_recovery_busy") monkeypatch.setattr(module.fleet, "_migration_lock", busy) driver, _ = proven result = module.preflight(revision="d"*40, projects={}, driver=driver) assert result["status"] == "in_progress" and result["effectsPerformed"] is False assert driver.effects == []