172 lines
8.5 KiB
Python
172 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
BROKER = Path(__file__).resolve().parents[2] / 'ops/gitea-deploy-broker'
|
|
|
|
|
|
def load(name, path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def recovery(monkeypatch):
|
|
monkeypatch.syspath_prepend(str(BROKER))
|
|
return load('registration_cutover_recovery_test', BROKER / 'runner_registration_cutover_recovery.py')
|
|
|
|
|
|
@pytest.fixture
|
|
def owned(recovery, monkeypatch, tmp_path):
|
|
fixture = load('registration_recovery_shared_fixture', Path(__file__).with_name('test_runner_failed_cutover_recovery.py'))
|
|
monkeypatch.setattr(recovery, 'PROBE_ID', 'a' * 64, raising=False)
|
|
values = fixture.owned.__wrapped__(recovery, monkeypatch, tmp_path)
|
|
fleet, _candidate, _q, _driver, _marker, journal, _probe, _rows, _dind, _events, _pending = values
|
|
journal.update(stage='state_prepared', cgroupCreated=True, cgroupEnsurePending=False)
|
|
journal['backupDirectory'] = recovery.BACKUP
|
|
_marker['imageManifestSha256'] = '0' * 64
|
|
monkeypatch.setattr(recovery, 'MARKER_SHA', hashlib.sha256(json.dumps(_marker).encode()).hexdigest())
|
|
monkeypatch.setattr(recovery, 'JOURNAL_SHA', hashlib.sha256(json.dumps(journal).encode()).hexdigest())
|
|
fleet.TASK_IMAGE_MANIFEST_PATH.unlink()
|
|
monkeypatch.setattr(fleet, 'ADMISSION_BOOTSTRAP_RECEIPT_PATH', tmp_path / 'absent-bootstrap')
|
|
monkeypatch.setattr(recovery, '__file__', str(recovery.BUNDLE / 'runner_registration_cutover_recovery.py'))
|
|
monkeypatch.setattr(recovery, '_workspace_proof', lambda _fleet: {'sixEmptyOwnedWorkspaces': True})
|
|
monkeypatch.setattr(recovery, '_probe_absent', lambda: True)
|
|
return values
|
|
|
|
|
|
@pytest.mark.parametrize('execute', [False, True])
|
|
@pytest.mark.parametrize('changed', ['marker', 'journal', 'backup'])
|
|
def test_same_source_other_incident_is_refused_before_claim_or_lifecycle(recovery, owned, monkeypatch, execute, changed):
|
|
fleet, _candidate, _q, _driver, marker, journal, *_rest = owned
|
|
if changed == 'marker':
|
|
marker['updatedAt'] = 'later-incident'
|
|
elif changed == 'journal':
|
|
journal['incidentNonce'] = 'later-incident'
|
|
else:
|
|
journal['backupDirectory'] += '-foreign'
|
|
# Even a digest admitted by a test double cannot change the fixed backup.
|
|
monkeypatch.setattr(recovery, 'JOURNAL_SHA', hashlib.sha256(json.dumps(journal).encode()).hexdigest())
|
|
monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('foreign incident claimed'))
|
|
monkeypatch.setattr(fleet, '_rollback_before_activation_locked', lambda *_: pytest.fail('foreign incident mutated'))
|
|
result = recovery.run(execute=execute, confirmation=recovery.CONFIRMATION if execute else None)
|
|
assert result['status'] == 'blocked' and result['reason'] == 'incident_identity_changed'
|
|
assert result['effectsPerformed'] is False and result['invocationConsumed'] is False
|
|
|
|
|
|
def test_default_preflight_uses_both_existing_locks_without_claim_or_lifecycle(recovery, owned, monkeypatch):
|
|
*_rest, events, _pending = owned
|
|
monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('readonly claimed'))
|
|
result = recovery.run()
|
|
assert result['status'] == 'ready'
|
|
assert result['effectsPerformed'] is False and result['invocationConsumed'] is False
|
|
assert events == ['migration-enter', 'maintenance-enter', 'guard-cas-checked', 'guard-cas-checked',
|
|
'maintenance-exit', 'migration-exit']
|
|
|
|
|
|
def test_execute_claims_once_then_only_calls_existing_owned_rollback(recovery, owned, monkeypatch):
|
|
fleet, _candidate, _q, _driver, _marker, _journal, _probe, _rows, _dind, events, _pending = owned
|
|
monkeypatch.setattr(recovery, '_claim', lambda _proof: events.append('consumed'))
|
|
def rollback(_journal, assert_barrier):
|
|
assert_barrier()
|
|
assert events[-1] == 'consumed'
|
|
events.append('existing-rollback-body')
|
|
return True
|
|
monkeypatch.setattr(fleet, '_rollback_before_activation_locked', rollback)
|
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
|
assert result['status'] == 'rolled_back' and result['invocationConsumed'] is True
|
|
assert events.index('consumed') < events.index('existing-rollback-body') < events.index('maintenance-exit')
|
|
|
|
|
|
@pytest.mark.parametrize('change', ['source', 'tree', 'stage', 'active', 'cgroup', 'pending', 'queue', 'busy',
|
|
'registration', 'privatebusy', 'guard', 'probe', 'manifest', 'workspace'])
|
|
def test_drift_refused_without_consumption_or_lifecycle(recovery, owned, monkeypatch, change):
|
|
fleet, _candidate, _q, driver, marker, journal, _probe, rows, _dind, _events, pending = owned
|
|
if change in {'source', 'tree'}:
|
|
journal['sourceRevision' if change == 'source' else 'sourceTree'] = 'foreign'
|
|
elif change == 'stage':
|
|
journal['stage'] = 'registrations_created'
|
|
elif change == 'active':
|
|
marker['status'] = 'active'
|
|
elif change == 'cgroup':
|
|
journal['cgroupCreated'] = False
|
|
elif change == 'pending':
|
|
journal['cgroupEnsurePending'] = True
|
|
elif change == 'queue':
|
|
pending.append({'id': 1})
|
|
elif change == 'busy':
|
|
rows[0]['busy'] = True
|
|
elif change == 'registration':
|
|
rows[0]['disabled'] = False
|
|
elif change == 'privatebusy':
|
|
monkeypatch.setattr(driver, '_run', lambda *args, **kwargs: 'f' * 64)
|
|
elif change == 'guard':
|
|
monkeypatch.setattr(fleet, '_guard_script_plan', lambda _: (_ for _ in ()).throw(RuntimeError('private-secret-canary')))
|
|
elif change == 'probe':
|
|
monkeypatch.setattr(recovery, '_probe_absent', lambda: False)
|
|
elif change == 'manifest':
|
|
fleet.TASK_IMAGE_MANIFEST_PATH.write_text('{}')
|
|
else:
|
|
monkeypatch.setattr(recovery, '_workspace_proof', lambda _: (_ for _ in ()).throw(recovery.RecoveryRejected('workspace_identity_changed')))
|
|
monkeypatch.setattr(recovery, '_claim', lambda _: pytest.fail('rejected case claimed'))
|
|
monkeypatch.setattr(fleet, '_rollback_before_activation_locked', lambda *_args: pytest.fail('rejected case mutated'))
|
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
|
assert result['status'] == 'blocked' and result['effectsPerformed'] is False
|
|
assert result['invocationConsumed'] is False and 'secret-canary' not in str(result)
|
|
|
|
|
|
def test_consumed_registration_claim_blocks_second_invocation(recovery, owned):
|
|
recovery.CLAIM.write_text('consumed')
|
|
result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
|
|
assert result['status'] == 'blocked' and result['reason'] == 'invocation_consumed_inspect_only'
|
|
|
|
|
|
def test_workspace_second_inspection_detects_identity_change(recovery, owned, monkeypatch):
|
|
snapshots = iter([{'identity': 'a'}, {'identity': 'changed'}])
|
|
monkeypatch.setattr(recovery, '_workspace_proof', lambda _: copy.deepcopy(next(snapshots)))
|
|
result = recovery.run()
|
|
assert result['status'] == 'blocked' and result['effectsPerformed'] is False
|
|
|
|
|
|
@pytest.mark.skipif(os.name != 'posix' or getattr(os, 'geteuid', lambda: -1)() != 0, reason='native root ownership fixture')
|
|
@pytest.mark.parametrize('change', [None, 'owner', 'mode', 'file', 'foreign', 'symlink'])
|
|
def test_native_workspace_admission_requires_exact_six_empty_private_directories(recovery, tmp_path, monkeypatch, change):
|
|
root = tmp_path / 'workspaces'
|
|
root.mkdir(mode=0o700)
|
|
names = tuple('profile-' + str(i) for i in range(6))
|
|
for name in names:
|
|
path = root / name
|
|
path.mkdir(mode=0o700)
|
|
os.chown(path, 1000, 1000)
|
|
target = root / names[0]
|
|
if change == 'owner':
|
|
os.chown(target, 2000, 2000)
|
|
elif change == 'mode':
|
|
target.chmod(0o755)
|
|
elif change == 'file':
|
|
(target / 'unexpected').write_text('preserve')
|
|
elif change == 'foreign':
|
|
(root / 'foreign').mkdir()
|
|
elif change == 'symlink':
|
|
target.rmdir()
|
|
target.symlink_to(tmp_path, target_is_directory=True)
|
|
monkeypatch.setattr(recovery, '_parents', lambda _: None)
|
|
from types import SimpleNamespace
|
|
fleet = SimpleNamespace(_SPECIALIZED_WORKSPACE_ROOT=root, _SPECIALIZED_NAMES=names)
|
|
if change:
|
|
with pytest.raises(recovery.RecoveryRejected, match='workspace_identity_changed'):
|
|
recovery._workspace_proof(fleet)
|
|
else:
|
|
assert len(recovery._workspace_proof(fleet)['profiles']) == 6
|