From 07bb0bbb90bdbbf8a93b4fa132c0011711a98cb6 Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Sat, 10 Oct 2026 11:21:58 +0000 Subject: [PATCH] Publish curated Unraid-AppOps-Gateway source from 5572962b172902d61bdd220ceb8fb69426f17b5f --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 11 +- backend/tests/test_runner_fleet_specialize.py | 173 ++++++++++++++++++ ...st_runner_registration_cutover_recovery.py | 171 +++++++++++++++++ 4 files changed, 353 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_runner_registration_cutover_recovery.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 8a7c95b..ce54d88 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `98b19114655572fd0244dbd68420e8491a4337d5`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `5572962b172902d61bdd220ceb8fb69426f17b5f`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index c7708c7..0f974ae 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "98b19114655572fd0244dbd68420e8491a4337d5", + "sourceRevision": "5572962b172902d61bdd220ceb8fb69426f17b5f", "files": [ { "path": ".dockerignore", @@ -1649,8 +1649,8 @@ }, { "path": "backend/tests/test_runner_fleet_specialize.py", - "sha256": "af16899c98919cdf9238315be2dc8d400e283990e716f766fac10ead0663ec79", - "bytes": 158831 + "sha256": "a79cf1f123284ef501fbe03363d0cc1b046df519442af1e63702e6cf8201ccb9", + "bytes": 167920 }, { "path": "backend/tests/test_runner_fleet_v2_execution_surface.py", @@ -1667,6 +1667,11 @@ "sha256": "45a8dba20d1e8d778dc90c20a997630037e2eaa1d462fa62f48671285aa82e0e", "bytes": 3111 }, + { + "path": "backend/tests/test_runner_registration_cutover_recovery.py", + "sha256": "f8d58090d0b43e33067a55780b77a3cde8f4d24e3ab132a646fcb3050cfbf440", + "bytes": 8680 + }, { "path": "backend/tests/test_runner_resource_envelope.py", "sha256": "afa94e168af82c28f91f0423f593dd59f55874bd203ba1ba9ec57c7c4970a3ec", diff --git a/backend/tests/test_runner_fleet_specialize.py b/backend/tests/test_runner_fleet_specialize.py index f883221..05541b1 100644 --- a/backend/tests/test_runner_fleet_specialize.py +++ b/backend/tests/test_runner_fleet_specialize.py @@ -2151,6 +2151,179 @@ def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(s assert shell_calls == [script] +@pytest.fixture +def profile_registration(specialize, monkeypatch): + rows = [] + commands = [] + journals = [] + state = {"check_host_paths": False, "missing_label": None} + token = "private-registration-canary" + monkeypatch.setattr(specialize, "_runner_rows", lambda: copy.deepcopy(rows)) + monkeypatch.setattr(specialize, "_registration_token", lambda: token) + def register(args, data, **kwargs): + assert kwargs == {"timeout": 90, "max_output": 32 * 1024} + assert data == (token + "\n").encode() and token not in " ".join(args) + profile_id = next(key for key, value in specialize.RUNNER_PROFILES.items() if value.container_name == args[args.index("--name") + 1]) + profile = specialize.RUNNER_PROFILES[profile_id] + if state["check_host_paths"]: + mounts = [args[index + 1] for index, argument in enumerate(args) if argument == "--volume"] + assert mounts == [f"{specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)}:/data", + f"{specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / (profile_id + '.yaml'))}:/runner-config.yaml:ro"] + assert all("/controller-hub-releases/" not in mount.replace("\\", "/") for mount in mounts) + assert args[-2] == profile.display_name + assert args[-1] == specialize._profile_labels_for_registration(profile_id, "a" * 40) + required = {profile.runner_label, *profile.legacy_labels} + if state["missing_label"] == profile_id: + required.remove(profile.runner_label) + rows.append({"id": 100 + len(rows), "name": profile.container_name, "displayName": profile.display_name, + "labels": sorted(required), "disabled": False, "busy": False}) + commands.append(profile_id) + return "registered" + monkeypatch.setattr(specialize, "_run_with_input", register) + def disable(identifier, disabled): + assert disabled is True + next(row for row in rows if row["id"] == identifier)["disabled"] = True + monkeypatch.setattr(specialize, "_disable_registration", disable) + def journal(value): + for profile_id, identifier in value["newRegistrationIds"].items(): + matching = next(row for row in rows if row["id"] == identifier) + assert matching["disabled"] is True and matching["busy"] is False + assert matching["name"] == specialize.RUNNER_PROFILES[profile_id].container_name + journals.append(copy.deepcopy(value)) + monkeypatch.setattr(specialize, "_write_journal", journal) + return rows, commands, journals, state + + +def test_new_profile_registration_uses_daemon_visible_volume_children(specialize, profile_registration): + rows, commands, journals, state = profile_registration + state["check_host_paths"] = True + journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} + specialize._register_profiles(journal) + assert commands == list(specialize.PROFILE_IDS) + assert len(rows) == 6 and set(journal["newRegistrationIds"]) == set(specialize.PROFILE_IDS) + assert journals[-1]["stage"] == "registrations_created" + + +@pytest.mark.parametrize("existing", [False, True]) +def test_new_registration_label_verification_is_initialized_for_each_profile(specialize, profile_registration, existing): + rows, commands, journals, _state = profile_registration + journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} + if existing: + key = specialize.PROFILE_IDS[0] + profile = specialize.RUNNER_PROFILES[key] + rows.append({"id": 9, "name": profile.container_name, "displayName": profile.display_name, + "labels": sorted({profile.runner_label, *profile.legacy_labels}), "disabled": True, "busy": False}) + journal["newRegistrationIds"][key] = 9 + specialize._register_profiles(journal) + assert len(rows) == 6 and len(commands) == (5 if existing else 6) + assert set(journal["newRegistrationIds"]) == set(specialize.PROFILE_IDS) + assert journals[-1]["stage"] == "registrations_created" + before = copy.deepcopy(journal) + specialize._register_profiles(journal) + assert journal == before and len(commands) == (5 if existing else 6) + + +@pytest.mark.parametrize("profile_id", ["checks", "backend", "frontend", "security", "browser", "build"]) +def test_new_profile_missing_own_required_label_is_not_recorded(specialize, profile_registration, profile_id): + _rows, commands, journals, state = profile_registration + state["missing_label"] = profile_id + journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} + with pytest.raises(specialize.RunnerFleetSpecializeError, match="registration_create_unverified"): + specialize._register_profiles(journal) + assert profile_id == commands[-1] and profile_id not in journal["newRegistrationIds"] + assert all(profile_id not in value["newRegistrationIds"] for value in journals) + + +@pytest.mark.parametrize("reason", ["command_failed", "command_unavailable"]) +def test_registration_command_failure_is_finite_and_does_not_record_registration( + specialize, profile_registration, monkeypatch, reason, +): + def fail(*_args, **_kwargs): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_" + reason) + monkeypatch.setattr(specialize, "_run_with_input", fail) + journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} + with pytest.raises(specialize.RunnerFleetSpecializeError) as caught: + specialize._register_profiles(journal) + assert str(caught.value) == "runner_fleet_specialization_registration_command_failed" + assert str(caught.value) in specialize._PUBLIC_SPECIALIZATION_FAILURES + assert caught.value.__suppress_context__ is True + assert journal["newRegistrationIds"] == {} + + +@pytest.fixture +def cleanup_workspace(specialize, tmp_path, monkeypatch): + workspace = tmp_path / "workspaces" + profile = workspace / specialize.RUNNER_PROFILES["checks"].container_name + nested = profile / "checkout" + nested.mkdir(parents=True) + (nested / "output").write_bytes(b"preserved-until-owned-cleanup") + monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", workspace) + owners = {profile: 1000, nested: 1000} + original = Path.lstat + def owned(path, *args, **kwargs): + value = list(original(path, *args, **kwargs)) + value[4] = owners.get(path, 0) + return os.stat_result(value) + monkeypatch.setattr(Path, "lstat", owned) + return workspace, profile, nested, owners + + +def test_owned_profile_workspace_cleanup_accepts_prepared_uid1000_directories(specialize, cleanup_workspace): + workspace, _profile, _nested, _owners = cleanup_workspace + specialize._remove_fixed_private_tree(workspace) + assert not workspace.exists() + + +@pytest.mark.parametrize("mutation", ["root_owner", "profile_owner", "nested_owner", "foreign_profile", "outside_workspace"]) +def test_workspace_cleanup_keeps_exact_root_profile_and_owner_boundary(specialize, cleanup_workspace, mutation): + workspace, profile, nested, owners = cleanup_workspace + target = workspace + if mutation == "root_owner": + owners[workspace] = 1000 + elif mutation == "profile_owner": + owners[profile] = 0 + elif mutation == "nested_owner": + owners[nested] = 1001 + elif mutation == "foreign_profile": + foreign = workspace / "foreign-runner" + profile.rename(foreign) + owners[foreign] = 1000 + else: + target = workspace.parent / "runner-data" + target.mkdir() + child = target / "checks" + child.mkdir() + owners[child] = 1000 + with pytest.raises(specialize.RunnerFleetSpecializeError, match="rollback_path_untrusted"): + specialize._remove_fixed_private_tree(target) + assert target.exists() + + +@pytest.mark.skipif(os.name != "posix", reason="requires POSIX symlinks") +def test_workspace_cleanup_refuses_symlink_without_touching_target(specialize, cleanup_workspace, tmp_path): + workspace, _profile, nested, _owners = cleanup_workspace + outside = tmp_path / "outside" + outside.mkdir() + preserved = outside / "preserved" + preserved.write_bytes(b"foreign") + (nested / "foreign-link").symlink_to(outside, target_is_directory=True) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="rollback_path_untrusted"): + specialize._remove_fixed_private_tree(workspace) + assert preserved.read_bytes() == b"foreign" + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") +def test_workspace_cleanup_removes_real_prepared_runner_owned_tree(specialize, tmp_path, monkeypatch): + workspace = tmp_path / "workspaces" + nested = workspace / specialize.RUNNER_PROFILES["checks"].container_name / "checkout" + nested.mkdir(parents=True, mode=0o700) + os.chown(nested.parent, 1000, 1000) + os.chown(nested, 1000, 1000) + monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", workspace) + specialize._remove_fixed_private_tree(workspace) + assert not workspace.exists() + + def test_run_with_input_keeps_registration_secret_out_of_command_arguments(specialize, monkeypatch): observed = {} diff --git a/backend/tests/test_runner_registration_cutover_recovery.py b/backend/tests/test_runner_registration_cutover_recovery.py new file mode 100644 index 0000000..2c61dfe --- /dev/null +++ b/backend/tests/test_runner_registration_cutover_recovery.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import os +import sys +from pathlib import Path + +import pytest + +BROKER = Path(__file__).resolve().parents[2] / 'ops/gitea-deploy-broker' + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def recovery(monkeypatch): + monkeypatch.syspath_prepend(str(BROKER)) + return load('registration_cutover_recovery_test', BROKER / 'runner_registration_cutover_recovery.py') + + +@pytest.fixture +def owned(recovery, monkeypatch, tmp_path): + fixture = load('registration_recovery_shared_fixture', Path(__file__).with_name('test_runner_failed_cutover_recovery.py')) + monkeypatch.setattr(recovery, 'PROBE_ID', 'a' * 64, raising=False) + values = fixture.owned.__wrapped__(recovery, monkeypatch, tmp_path) + fleet, _candidate, _q, _driver, _marker, journal, _probe, _rows, _dind, _events, _pending = values + journal.update(stage='state_prepared', cgroupCreated=True, cgroupEnsurePending=False) + journal['backupDirectory'] = recovery.BACKUP + _marker['imageManifestSha256'] = '0' * 64 + monkeypatch.setattr(recovery, 'MARKER_SHA', hashlib.sha256(json.dumps(_marker).encode()).hexdigest()) + monkeypatch.setattr(recovery, 'JOURNAL_SHA', hashlib.sha256(json.dumps(journal).encode()).hexdigest()) + fleet.TASK_IMAGE_MANIFEST_PATH.unlink() + monkeypatch.setattr(fleet, 'ADMISSION_BOOTSTRAP_RECEIPT_PATH', tmp_path / 'absent-bootstrap') + monkeypatch.setattr(recovery, '__file__', str(recovery.BUNDLE / 'runner_registration_cutover_recovery.py')) + monkeypatch.setattr(recovery, '_workspace_proof', lambda _fleet: {'sixEmptyOwnedWorkspaces': True}) + monkeypatch.setattr(recovery, '_probe_absent', lambda: True) + return values + + +@pytest.mark.parametrize('execute', [False, True]) +@pytest.mark.parametrize('changed', ['marker', 'journal', 'backup']) +def test_same_source_other_incident_is_refused_before_claim_or_lifecycle(recovery, owned, monkeypatch, execute, changed): + fleet, _candidate, _q, _driver, marker, journal, *_rest = owned + if changed == 'marker': + marker['updatedAt'] = 'later-incident' + elif changed == 'journal': + journal['incidentNonce'] = 'later-incident' + else: + journal['backupDirectory'] += '-foreign' + # Even a digest admitted by a test double cannot change the fixed backup. + monkeypatch.setattr(recovery, 'JOURNAL_SHA', hashlib.sha256(json.dumps(journal).encode()).hexdigest()) + monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('foreign incident claimed')) + monkeypatch.setattr(fleet, '_rollback_before_activation_locked', lambda *_: pytest.fail('foreign incident mutated')) + result = recovery.run(execute=execute, confirmation=recovery.CONFIRMATION if execute else None) + assert result['status'] == 'blocked' and result['reason'] == 'incident_identity_changed' + assert result['effectsPerformed'] is False and result['invocationConsumed'] is False + + +def test_default_preflight_uses_both_existing_locks_without_claim_or_lifecycle(recovery, owned, monkeypatch): + *_rest, events, _pending = owned + monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('readonly claimed')) + result = recovery.run() + assert result['status'] == 'ready' + assert result['effectsPerformed'] is False and result['invocationConsumed'] is False + assert events == ['migration-enter', 'maintenance-enter', 'guard-cas-checked', 'guard-cas-checked', + 'maintenance-exit', 'migration-exit'] + + +def test_execute_claims_once_then_only_calls_existing_owned_rollback(recovery, owned, monkeypatch): + fleet, _candidate, _q, _driver, _marker, _journal, _probe, _rows, _dind, events, _pending = owned + monkeypatch.setattr(recovery, '_claim', lambda _proof: events.append('consumed')) + def rollback(_journal, assert_barrier): + assert_barrier() + assert events[-1] == 'consumed' + events.append('existing-rollback-body') + return True + monkeypatch.setattr(fleet, '_rollback_before_activation_locked', rollback) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result['status'] == 'rolled_back' and result['invocationConsumed'] is True + assert events.index('consumed') < events.index('existing-rollback-body') < events.index('maintenance-exit') + + +@pytest.mark.parametrize('change', ['source', 'tree', 'stage', 'active', 'cgroup', 'pending', 'queue', 'busy', + 'registration', 'privatebusy', 'guard', 'probe', 'manifest', 'workspace']) +def test_drift_refused_without_consumption_or_lifecycle(recovery, owned, monkeypatch, change): + fleet, _candidate, _q, driver, marker, journal, _probe, rows, _dind, _events, pending = owned + if change in {'source', 'tree'}: + journal['sourceRevision' if change == 'source' else 'sourceTree'] = 'foreign' + elif change == 'stage': + journal['stage'] = 'registrations_created' + elif change == 'active': + marker['status'] = 'active' + elif change == 'cgroup': + journal['cgroupCreated'] = False + elif change == 'pending': + journal['cgroupEnsurePending'] = True + elif change == 'queue': + pending.append({'id': 1}) + elif change == 'busy': + rows[0]['busy'] = True + elif change == 'registration': + rows[0]['disabled'] = False + elif change == 'privatebusy': + monkeypatch.setattr(driver, '_run', lambda *args, **kwargs: 'f' * 64) + elif change == 'guard': + monkeypatch.setattr(fleet, '_guard_script_plan', lambda _: (_ for _ in ()).throw(RuntimeError('private-secret-canary'))) + elif change == 'probe': + monkeypatch.setattr(recovery, '_probe_absent', lambda: False) + elif change == 'manifest': + fleet.TASK_IMAGE_MANIFEST_PATH.write_text('{}') + else: + monkeypatch.setattr(recovery, '_workspace_proof', lambda _: (_ for _ in ()).throw(recovery.RecoveryRejected('workspace_identity_changed'))) + monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('rejected case claimed')) + monkeypatch.setattr(fleet, '_rollback_before_activation_locked', lambda *_args: pytest.fail('rejected case mutated')) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result['status'] == 'blocked' and result['effectsPerformed'] is False + assert result['invocationConsumed'] is False and 'secret-canary' not in str(result) + + +def test_consumed_registration_claim_blocks_second_invocation(recovery, owned): + recovery.CLAIM.write_text('consumed') + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result['status'] == 'blocked' and result['reason'] == 'invocation_consumed_inspect_only' + + +def test_workspace_second_inspection_detects_identity_change(recovery, owned, monkeypatch): + snapshots = iter([{'identity': 'a'}, {'identity': 'changed'}]) + monkeypatch.setattr(recovery, '_workspace_proof', lambda _: copy.deepcopy(next(snapshots))) + result = recovery.run() + assert result['status'] == 'blocked' and result['effectsPerformed'] is False + + +@pytest.mark.skipif(os.name != 'posix' or getattr(os, 'geteuid', lambda: -1)() != 0, reason='native root ownership fixture') +@pytest.mark.parametrize('change', [None, 'owner', 'mode', 'file', 'foreign', 'symlink']) +def test_native_workspace_admission_requires_exact_six_empty_private_directories(recovery, tmp_path, monkeypatch, change): + root = tmp_path / 'workspaces' + root.mkdir(mode=0o700) + names = tuple('profile-' + str(i) for i in range(6)) + for name in names: + path = root / name + path.mkdir(mode=0o700) + os.chown(path, 1000, 1000) + target = root / names[0] + if change == 'owner': + os.chown(target, 2000, 2000) + elif change == 'mode': + target.chmod(0o755) + elif change == 'file': + (target / 'unexpected').write_text('preserve') + elif change == 'foreign': + (root / 'foreign').mkdir() + elif change == 'symlink': + target.rmdir() + target.symlink_to(tmp_path, target_is_directory=True) + monkeypatch.setattr(recovery, '_parents', lambda _: None) + from types import SimpleNamespace + fleet = SimpleNamespace(_SPECIALIZED_WORKSPACE_ROOT=root, _SPECIALIZED_NAMES=names) + if change: + with pytest.raises(recovery.RecoveryRejected, match='workspace_identity_changed'): + recovery._workspace_proof(fleet) + else: + assert len(recovery._workspace_proof(fleet)['profiles']) == 6