from __future__ import annotations import copy import hashlib import importlib.util import json import os import sys from pathlib import Path import pytest BROKER = Path(__file__).resolve().parents[2] / 'ops/gitea-deploy-broker' def load(name, path): spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module @pytest.fixture def recovery(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) return load('registration_cutover_recovery_test', BROKER / 'runner_registration_cutover_recovery.py') @pytest.fixture def owned(recovery, monkeypatch, tmp_path): fixture = load('registration_recovery_shared_fixture', Path(__file__).with_name('test_runner_failed_cutover_recovery.py')) monkeypatch.setattr(recovery, 'PROBE_ID', 'a' * 64, raising=False) values = fixture.owned.__wrapped__(recovery, monkeypatch, tmp_path) fleet, _candidate, _q, _driver, _marker, journal, _probe, _rows, _dind, _events, _pending = values journal.update(stage='state_prepared', cgroupCreated=True, cgroupEnsurePending=False) journal['backupDirectory'] = recovery.BACKUP _marker['imageManifestSha256'] = '0' * 64 monkeypatch.setattr(recovery, 'MARKER_SHA', hashlib.sha256(json.dumps(_marker).encode()).hexdigest()) monkeypatch.setattr(recovery, 'JOURNAL_SHA', hashlib.sha256(json.dumps(journal).encode()).hexdigest()) fleet.TASK_IMAGE_MANIFEST_PATH.unlink() monkeypatch.setattr(fleet, 'ADMISSION_BOOTSTRAP_RECEIPT_PATH', tmp_path / 'absent-bootstrap') monkeypatch.setattr(recovery, '__file__', str(recovery.BUNDLE / 'runner_registration_cutover_recovery.py')) monkeypatch.setattr(recovery, '_workspace_proof', lambda _fleet: {'sixEmptyOwnedWorkspaces': True}) monkeypatch.setattr(recovery, '_probe_absent', lambda: True) return values @pytest.mark.parametrize('execute', [False, True]) @pytest.mark.parametrize('changed', ['marker', 'journal', 'backup']) def test_same_source_other_incident_is_refused_before_claim_or_lifecycle(recovery, owned, monkeypatch, execute, changed): fleet, _candidate, _q, _driver, marker, journal, *_rest = owned if changed == 'marker': marker['updatedAt'] = 'later-incident' elif changed == 'journal': journal['incidentNonce'] = 'later-incident' else: journal['backupDirectory'] += '-foreign' # Even a digest admitted by a test double cannot change the fixed backup. monkeypatch.setattr(recovery, 'JOURNAL_SHA', hashlib.sha256(json.dumps(journal).encode()).hexdigest()) monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('foreign incident claimed')) monkeypatch.setattr(fleet, '_rollback_before_activation_locked', lambda *_: pytest.fail('foreign incident mutated')) result = recovery.run(execute=execute, confirmation=recovery.CONFIRMATION if execute else None) assert result['status'] == 'blocked' and result['reason'] == 'incident_identity_changed' assert result['effectsPerformed'] is False and result['invocationConsumed'] is False def test_default_preflight_uses_both_existing_locks_without_claim_or_lifecycle(recovery, owned, monkeypatch): *_rest, events, _pending = owned monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('readonly claimed')) result = recovery.run() assert result['status'] == 'ready' assert result['effectsPerformed'] is False and result['invocationConsumed'] is False assert events == ['migration-enter', 'maintenance-enter', 'guard-cas-checked', 'guard-cas-checked', 'maintenance-exit', 'migration-exit'] def test_execute_claims_once_then_only_calls_existing_owned_rollback(recovery, owned, monkeypatch): fleet, _candidate, _q, _driver, _marker, _journal, _probe, _rows, _dind, events, _pending = owned monkeypatch.setattr(recovery, '_claim', lambda _proof: events.append('consumed')) def rollback(_journal, assert_barrier): assert_barrier() assert events[-1] == 'consumed' events.append('existing-rollback-body') return True monkeypatch.setattr(fleet, '_rollback_before_activation_locked', rollback) result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result['status'] == 'rolled_back' and result['invocationConsumed'] is True assert events.index('consumed') < events.index('existing-rollback-body') < events.index('maintenance-exit') @pytest.mark.parametrize('change', ['source', 'tree', 'stage', 'active', 'cgroup', 'pending', 'queue', 'busy', 'registration', 'privatebusy', 'guard', 'probe', 'manifest', 'workspace']) def test_drift_refused_without_consumption_or_lifecycle(recovery, owned, monkeypatch, change): fleet, _candidate, _q, driver, marker, journal, _probe, rows, _dind, _events, pending = owned if change in {'source', 'tree'}: journal['sourceRevision' if change == 'source' else 'sourceTree'] = 'foreign' elif change == 'stage': journal['stage'] = 'registrations_created' elif change == 'active': marker['status'] = 'active' elif change == 'cgroup': journal['cgroupCreated'] = False elif change == 'pending': journal['cgroupEnsurePending'] = True elif change == 'queue': pending.append({'id': 1}) elif change == 'busy': rows[0]['busy'] = True elif change == 'registration': rows[0]['disabled'] = False elif change == 'privatebusy': monkeypatch.setattr(driver, '_run', lambda *args, **kwargs: 'f' * 64) elif change == 'guard': monkeypatch.setattr(fleet, '_guard_script_plan', lambda _: (_ for _ in ()).throw(RuntimeError('private-secret-canary'))) elif change == 'probe': monkeypatch.setattr(recovery, '_probe_absent', lambda: False) elif change == 'manifest': fleet.TASK_IMAGE_MANIFEST_PATH.write_text('{}') else: monkeypatch.setattr(recovery, '_workspace_proof', lambda _: (_ for _ in ()).throw(recovery.RecoveryRejected('workspace_identity_changed'))) monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('rejected case claimed')) monkeypatch.setattr(fleet, '_rollback_before_activation_locked', lambda *_args: pytest.fail('rejected case mutated')) result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result['status'] == 'blocked' and result['effectsPerformed'] is False assert result['invocationConsumed'] is False and 'secret-canary' not in str(result) def test_consumed_registration_claim_blocks_second_invocation(recovery, owned): recovery.CLAIM.write_text('consumed') result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result['status'] == 'blocked' and result['reason'] == 'invocation_consumed_inspect_only' def test_workspace_second_inspection_detects_identity_change(recovery, owned, monkeypatch): snapshots = iter([{'identity': 'a'}, {'identity': 'changed'}]) monkeypatch.setattr(recovery, '_workspace_proof', lambda _: copy.deepcopy(next(snapshots))) result = recovery.run() assert result['status'] == 'blocked' and result['effectsPerformed'] is False @pytest.mark.skipif(os.name != 'posix' or getattr(os, 'geteuid', lambda: -1)() != 0, reason='native root ownership fixture') @pytest.mark.parametrize('change', [None, 'owner', 'mode', 'file', 'foreign', 'symlink']) def test_native_workspace_admission_requires_exact_six_empty_private_directories(recovery, tmp_path, monkeypatch, change): root = tmp_path / 'workspaces' root.mkdir(mode=0o700) names = tuple('profile-' + str(i) for i in range(6)) for name in names: path = root / name path.mkdir(mode=0o700) os.chown(path, 1000, 1000) target = root / names[0] if change == 'owner': os.chown(target, 2000, 2000) elif change == 'mode': target.chmod(0o755) elif change == 'file': (target / 'unexpected').write_text('preserve') elif change == 'foreign': (root / 'foreign').mkdir() elif change == 'symlink': target.rmdir() target.symlink_to(tmp_path, target_is_directory=True) monkeypatch.setattr(recovery, '_parents', lambda _: None) from types import SimpleNamespace fleet = SimpleNamespace(_SPECIALIZED_WORKSPACE_ROOT=root, _SPECIALIZED_NAMES=names) if change: with pytest.raises(recovery.RecoveryRejected, match='workspace_identity_changed'): recovery._workspace_proof(fleet) else: assert len(recovery._workspace_proof(fleet)['profiles']) == 6