Publish curated Unraid-AppOps-Gateway source from 945a919d1a3bbd2bb5bd21eb221fc9b58d795d80

backend/tests/test_runner_fleet_specialize.py

@@ -4308,16 +4308,18 @@ def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specia
         specialize._check_legacy_queued_jobs()
 
 
-def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url():
+def test_image_workflow_builds_from_protected_appops_main_with_private_https_checkout_url():
     workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8")
     build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8")
     frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8")
 
-    assert "github-server-url: http://gitea-runner-internal:3000" in workflow
-    assert "runs-on: linux-validation" in workflow
-    assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output"' in workflow
-    assert "runs-on: itworx-ci-build" not in workflow
-    assert "--job-local" not in workflow
+    assert "github-server-url: https://gitea.example.com" in workflow
+    for name in ("managed-validation.yml", "runner-build-acceptance.yml", "ci-task-images.yml"):
+        content = (ROOT / ".gitea" / "workflows" / name).read_text(encoding="utf-8")
+        assert "gitea-runner-internal" not in content
+    assert "runs-on: itworx-ci-build" in workflow
+    assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output" --job-local' in workflow
+    assert "runs-on: linux-validation" not in workflow
     assert "jens/unraid-appops-gateway" in build
     assert "Jens/Unraid-AppOps-Gateway" in build
     assert "corepack prepare pnpm@11.15.0 --activate" in frontend
@@ -4375,6 +4377,9 @@ def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity(
 
 def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True):
     rows, containers, dind, journal, model = _active_recovery_fixture(specialize)
+    dind.update({"Id": "a" * 64, "Image": specialize._DIND_IMAGE_ID})
+    for profile in specialize.RUNNER_PROFILES.values():
+        containers[profile.container_name]["State"]["Pid"] = 0
     events = []
 
     class Volume:
@@ -4425,6 +4430,10 @@ def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True):
     )
     monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy"))
     monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True)
+    monkeypatch.setattr(specialize, "_bind_active_dind_family",
+                        lambda *_args: events.append("owned-populated-family"))
+    monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "")
+    monkeypatch.setattr(specialize, "_run", lambda args, **_kwargs: events.append(("docker", args)) or "")
     monkeypatch.setattr(
         specialize,
         "_resource_proof",
@@ -4433,7 +4442,8 @@ def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True):
 
     def inspect(name):
         if name == specialize.DIND_NAME:
-            dind["State"] = {"Running": True, "Health": {"Status": "healthy"}}
+            dind["State"] = {"Running": True, "Pid": 77,
+                             "StartedAt": "2026-10-12T00:00:00Z", "Health": {"Status": "healthy"}}
             return dind
         return None
 
@@ -4461,11 +4471,270 @@ def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates(
     assert compose_bytes == b"captured-compose"
     assert args == ("up", "-d", "--no-deps", "runner-dind")
     assert kwargs["timeout"] <= 30
-    assert events.count("fresh-host-gate") == 2
+    assert events.count("fresh-host-gate") == 3
     assert events.index("fresh-host-gate") < events.index("healthy")
 
 
-def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind(
+def test_active_recovery_survives_disappearing_empty_family(specialize, monkeypatch):
+    events, _rows, _containers, _dind = _stub_active_recovery(
+        specialize, monkeypatch, family_exists=False
+    )
+    monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {
+        "exists": False,
+        "safeToStartDind": False,
+        "safeToStartDindReason": "resource_cgroup_family_missing",
+        "rootSubtreeControllers": ["cpu", "memory", "pids"],
+    })
+    monkeypatch.setattr(specialize, "_bind_active_dind_family",
+                        lambda *_args: events.append("owned-populated-family"), raising=False)
+
+    result = specialize.ensure_active_fleet_ready()
+
+    assert result == {"ready": True, "reason": None}
+    assert "cgroup-create" not in events
+    assert events.index("healthy") < events.index("owned-populated-family")
+
+
+@pytest.mark.parametrize("failure", ["health", "inspect-after-binding", "binding", "images", "envelope", "guard"])
+def test_active_recovery_stops_only_its_new_empty_start_on_failed_acceptance(
+    specialize, monkeypatch, failure,
+):
+    events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    if failure == "health":
+        def fail_health(*_args, **_kwargs):
+            raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_dind_recovery_unverified")
+        monkeypatch.setattr(specialize, "_wait_healthy", fail_health)
+    elif failure == "inspect-after-binding":
+        original_inspect = specialize._inspect
+        count = [0]
+        def fail_inspect(name):
+            count[0] += 1
+            if count[0] == 2:
+                raise RuntimeError("sentinel-secret")
+            return original_inspect(name)
+        monkeypatch.setattr(specialize, "_inspect", fail_inspect)
+    elif failure == "binding":
+        def fail_binding(*_args):
+            raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable")
+        monkeypatch.setattr(specialize, "_bind_active_dind_family", fail_binding)
+    elif failure == "images":
+        monkeypatch.setattr(specialize, "verify_profile_task_images", lambda *_args: False)
+    elif failure == "envelope":
+        def fail_proof(*_args):
+            raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_resource_envelope_unready")
+        monkeypatch.setattr(specialize, "_resource_proof", fail_proof)
+    else:
+        count = [0]
+        def fail_guard():
+            count[0] += 1
+            if count[0] == 3:
+                raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_host_guard_paused")
+        monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fail_guard)
+
+    result = specialize.ensure_active_fleet_ready()
+
+    assert result["ready"] is False
+    assert "sentinel-secret" not in str(result)
+    assert ("docker", ["docker", "stop", "--time", "30", "a" * 64]) in events
+
+
+@pytest.mark.parametrize("drift", ["id", "pid", "started", "job", "profile", "profile-pid", "registration", "busy", "enabled", "volume"])
+def test_active_recovery_rollback_refuses_changed_or_foreign_state(specialize, monkeypatch, drift):
+    events, rows, containers, dind = _stub_active_recovery(specialize, monkeypatch)
+    inspected = specialize._inspect(specialize.DIND_NAME)
+    binding = specialize._active_dind_binding(inspected, {})
+    marker, journal, model, compose = specialize._active_release_bundle()
+    identity = specialize._active_recovery_identity(marker, journal, compose)
+    if drift == "id":
+        dind["Id"] = "b" * 64
+    elif drift in {"pid", "started"}:
+        key, value = ("Pid", 88) if drift == "pid" else ("StartedAt", "2026-10-12T00:01:00Z")
+        def changed_inspect(_name):
+            changed = copy.deepcopy(dind)
+            changed["State"][key] = value
+            return changed
+        monkeypatch.setattr(specialize, "_inspect", changed_inspect)
+    elif drift == "job":
+        monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
+    elif drift == "profile":
+        containers[specialize.RUNNER_PROFILES["backend"].container_name]["State"]["Running"] = True
+    elif drift == "profile-pid":
+        containers[specialize.RUNNER_PROFILES["backend"].container_name]["State"]["Pid"] = 88
+    elif drift == "registration":
+        rows[0]["id"] += 1000
+    elif drift == "busy":
+        rows[0]["busy"] = True
+    elif drift == "enabled":
+        rows[0]["disabled"] = False
+    else:
+        def changed_bundle():
+            raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_active_release_untrusted")
+        monkeypatch.setattr(specialize, "_active_release_bundle", changed_bundle)
+    try:
+        specialize._stop_new_active_dind(binding, model, identity)
+    except specialize.RunnerFleetSpecializeError:
+        pass
+    assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
+
+
+@pytest.mark.parametrize("case", ["missing-original", "replaced", "same-cid-restarted", "uninspectable", "inspect-once", "nonempty"])
+def test_active_recovery_failed_start_never_infers_cleanup_ownership(specialize, monkeypatch, case):
+    events, _rows, containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    original_inspect = specialize._inspect
+    if case == "missing-original":
+        containers.pop(specialize.DIND_NAME)
+    elif case == "replaced":
+        def replaced(name):
+            item = copy.deepcopy(original_inspect(name))
+            item["Id"] = "b" * 64
+            return item
+        monkeypatch.setattr(specialize, "_inspect", replaced)
+    elif case == "uninspectable":
+        def fail_inspect(_name):
+            raise RuntimeError("sentinel-secret")
+        monkeypatch.setattr(specialize, "_inspect", fail_inspect)
+    elif case == "inspect-once":
+        count = [0]
+        def fail_once(name):
+            count[0] += 1
+            if count[0] == 1:
+                raise RuntimeError("sentinel-secret")
+            return original_inspect(name)
+        monkeypatch.setattr(specialize, "_inspect", fail_once)
+    elif case == "same-cid-restarted":
+        def restarted(name):
+            item = copy.deepcopy(original_inspect(name))
+            item["State"].update({"Pid": 88, "StartedAt": "2026-10-12T00:01:00Z"})
+            return item
+        monkeypatch.setattr(specialize, "_inspect", restarted)
+    else:
+        monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
+    def fail_start(*_args, **_kwargs):
+        raise RuntimeError("sentinel-secret")
+    if case != "inspect-once":
+        monkeypatch.setattr(specialize, "_run_compose", fail_start)
+
+    result = specialize.ensure_active_fleet_ready()
+
+    assert result["ready"] is False
+    assert "sentinel-secret" not in str(result)
+    assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
+
+
+def test_active_binder_uses_exact_owned_population_without_migration_journal_writes(specialize, monkeypatch):
+    binder = specialize._bind_active_dind_family
+    events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    # Exercise the real binder rather than the high-level lifecycle stub.
+    item = specialize._inspect(specialize.DIND_NAME)
+    calls = []
+    monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\n77\n78\n")
+    monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", lambda **kwargs: calls.append(kwargs))
+    monkeypatch.setattr(specialize, "_write_journal", lambda *_args: pytest.fail("migration evidence must remain intact"))
+    marker, journal, model, compose = specialize._active_release_bundle()
+    binder(item, model, specialize._active_recovery_identity(marker, journal, compose))
+
+    assert calls == [{"expected_container_id": "a" * 64, "expected_host_pid": 77,
+                      "expected_process_ids": [77, 78], "allow_owned_partial": True}]
+    assert "cgroup-create" not in events
+
+
+@pytest.mark.parametrize("case", ["job", "job-after", "duplicate", "zero", "missing-init", "unknown-pid", "foreign-family", "replaced", "replaced-before", "restarted-before"])
+def test_active_binder_rejects_untrusted_or_nonempty_population(specialize, monkeypatch, case):
+    binder = specialize._bind_active_dind_family
+    _events, _rows, _containers, dind = _stub_active_recovery(specialize, monkeypatch)
+    item = copy.deepcopy(specialize._inspect(specialize.DIND_NAME))
+    if case == "replaced-before":
+        dind["Id"] = "b" * 64
+    elif case == "restarted-before":
+        original_inspect = specialize._inspect
+        def restarted(name):
+            changed = copy.deepcopy(original_inspect(name))
+            changed["State"].update({"Pid": 88, "StartedAt": "2026-10-12T00:01:00Z"})
+            return changed
+        monkeypatch.setattr(specialize, "_inspect", restarted)
+    text = {"duplicate": "PID\n77\n77\n", "zero": "PID\n77\n0\n",
+            "missing-init": "PID\n78\n", "unknown-pid": "PID\nsentinel-secret\n"}.get(case, "PID\n77\n78\n")
+    monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: text)
+    if case == "job":
+        monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
+    calls = []
+    def bind(**kwargs):
+        calls.append(kwargs)
+        if case == "foreign-family":
+            raise RuntimeError("sentinel-secret")
+        if case == "replaced":
+            dind["Id"] = "b" * 64
+        elif case == "job-after":
+            monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
+    monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", bind)
+    marker, journal, model, compose = specialize._active_release_bundle()
+    with pytest.raises(specialize.RunnerFleetSpecializeError) as error:
+        binder(item, model, specialize._active_recovery_identity(marker, journal, compose))
+    assert "sentinel-secret" not in str(error.value)
+    if case not in {"foreign-family", "replaced", "job-after"}:
+        assert calls == []
+
+
+def test_active_recovery_can_finish_existing_empty_bounded_start_without_stopping_it(specialize, monkeypatch):
+    events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    specialize._inspect(specialize.DIND_NAME)
+    ready = iter([{"ready": False}, {"ready": True}])
+    monkeypatch.setattr(specialize, "verified_topology", lambda: next(ready))
+    assert specialize.ensure_active_fleet_ready() == {"ready": True, "reason": None}
+    assert "owned-populated-family" in events
+    assert not any(isinstance(item, tuple) and item[0] in {"compose", "docker"} for item in events)
+
+
+@pytest.mark.parametrize("drift", ["sourceScriptSha256", "imageSourceRevision", "imageSourceTree", "imageManifestSha256", "compose", "registration"])
+def test_active_binder_and_cleanup_refuse_valid_same_model_release_drift(specialize, monkeypatch, drift):
+    binder = specialize._bind_active_dind_family
+    events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    item = specialize._inspect(specialize.DIND_NAME)
+    binding = specialize._active_dind_binding(item, {})
+    marker, journal, model, compose = specialize._active_release_bundle()
+    identity = specialize._active_recovery_identity(marker, journal, compose)
+    changed_marker, changed_journal, changed_compose = copy.deepcopy(marker), copy.deepcopy(journal), compose
+    if drift == "compose":
+        changed_compose += b" "
+    elif drift == "registration":
+        changed_journal["newRegistrationIds"]["backend"] += 1000
+    else:
+        changed_marker[drift] = "f" * (40 if drift in {"imageSourceRevision", "imageSourceTree"} else 64)
+    monkeypatch.setattr(specialize, "_active_release_bundle", lambda: (changed_marker, changed_journal, model, changed_compose))
+    monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\n77\n78\n")
+    monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", lambda **_kwargs: pytest.fail("changed source cannot bind"))
+    with pytest.raises(specialize.RunnerFleetSpecializeError, match="active_release_changed"):
+        binder(item, model, identity)
+    monkeypatch.setattr(specialize, "_run", lambda args, **_kwargs: events.append(("docker", args)) or "")
+    with pytest.raises(specialize.RunnerFleetSpecializeError, match="active_release_changed"):
+        specialize._stop_new_active_dind(binding, model, identity)
+    assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
+
+
+@pytest.mark.parametrize("failure", ["binding", "proof"])
+def test_active_recovery_preserves_preexisting_running_daemon_on_failure(specialize, monkeypatch, failure):
+    events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    specialize._inspect(specialize.DIND_NAME)
+    monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False})
+    def fail_bind(*_args):
+        raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable")
+    if failure == "binding":
+        monkeypatch.setattr(specialize, "_bind_active_dind_family", fail_bind)
+    assert specialize.ensure_active_fleet_ready()["ready"] is False
+    assert not any(isinstance(item, tuple) and item[0] in {"compose", "docker"} for item in events)
+
+
+def test_active_recovery_does_not_stop_new_daemon_with_work_appearing_after_binding(specialize, monkeypatch):
+    events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
+    def failed_binding(*_args):
+        monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
+        raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_dind_recovery_unverified")
+    monkeypatch.setattr(specialize, "_bind_active_dind_family", failed_binding)
+    assert specialize.ensure_active_fleet_ready()["ready"] is False
+    assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
+
+
+def test_active_recovery_makes_no_cgroup_or_start_write_if_host_guard_pauses_before_dind(
     specialize, monkeypatch
 ):
     events, _rows, _containers, _dind = _stub_active_recovery(
@@ -4495,9 +4764,8 @@ def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_din
         "ready": False,
         "reason": "runner_fleet_specialization_host_guard_paused",
     }
-    assert rollback == [
-        {"created": True, "initial_controllers": ["cpu", "memory", "pids"]}
-    ]
+    assert rollback == []
+    assert "cgroup-create" not in events
     assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events)