Files
Unraid-AppOps-Gateway-Public/backend/tests/test_privatemuse_f84_adoption.py
T

536 lines
21 KiB
Python

import copy
import hashlib
import json
import os
import sys
from contextlib import contextmanager
from pathlib import Path
import pytest
sys.path.insert(
0,
str(Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"),
)
import adopt_privatemuse_f84_runtime as adoption
import diagnose_privatemuse_f84_receipt as diagnosis
import privatemuse_f84_receipt_diagnosis as incident_contract
def _incident_evidence():
return {
"repository": "jens/privatemuse",
"candidate_revision": diagnosis._FAILED_REVISION,
"live_revision": diagnosis._LIVE_REVISION,
"delivery_state": "failed",
"delivery_message": diagnosis._FAILURE_MESSAGE,
"candidate_container_mutation": False,
"recorded_backend_container_id": adoption._RECORDED_BACKEND_ID,
"observed_backend_container_id": adoption._OBSERVED_BACKEND_ID,
"backend_image_id": incident_contract._EXPECTED_IMAGE_IDS["backend"],
"frontend_image_id": incident_contract._EXPECTED_IMAGE_IDS["frontend"],
"audio_image_id": incident_contract._EXPECTED_IMAGE_IDS["audio"],
"all_recorded_image_ids_match": True,
"frontend_and_audio_container_ids_match": True,
"compose_source_sha256": incident_contract._COMPOSE_SHA256,
"compose_source_hash_matches_receipt": True,
"backend_health": "healthy",
"frontend_state": "running",
"audio_health": "healthy",
"compose_project": "privatemuse",
"backend_service": "backend",
"backend_mounts": sorted(incident_contract._EXPECTED_MOUNTS),
"backend_networks": sorted(incident_contract._EXPECTED_NETWORKS),
"configuration_evidence": {
"old_delivery_updated_at_epoch": 1790271038,
"protected_env_changed_at_utc": "2026-09-24T18:13:33Z",
"backup_filename": ".env.pre-modelforge-ready-1790273613.bak",
"changed_key_names": [diagnosis._HEALTH_PATH_KEY],
"live_value_matches_current_file": True,
"live_value_matches_backup": False,
"historical_effective_runtime_fingerprint": "not_recorded",
"secret_values_recorded": False,
},
"provider_submissions": 0,
"job_retries": 0,
"production_mutations": 0,
"backend_runtime_mount_network_port_and_env_match_current_compose": True,
"backend_frontend_same_compose_project": True,
"backend_frontend_share_network": True,
"backend_alias_visible_to_frontend": True,
"audio_container_identity_matches_contract": True,
}
def _post_adoption_evidence():
evidence = _incident_evidence()
evidence["recorded_backend_container_id"] = adoption._OBSERVED_BACKEND_ID
return evidence
def _fixture_state():
revision = diagnosis._LIVE_REVISION
tree = "a" * 40
built_at = "2026-09-24T17:00:00.000Z"
build = {
"id": revision[:12],
"repository": diagnosis._REPOSITORY,
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": built_at,
}
container_ids = {
"backend": adoption._RECORDED_BACKEND_ID,
"frontend": "1" * 64,
"audio": "2" * 64,
}
containers = {}
for role, name in diagnosis._CONTAINERS.items():
containers[name] = {
"container_id": container_ids[role],
"image_id": incident_contract._EXPECTED_IMAGE_IDS[role],
"image_ref": f"local/privatemuse-{role}:b79",
"container_revision": revision,
"image_revision": revision,
"image_source": diagnosis._REPOSITORY,
"image_tree": tree,
"image_version": revision[:12],
"image_created": built_at,
}
receipt = {
"schema": "gitea-deploy-receipt.v1",
"repository": diagnosis._REPOSITORY,
"revision": revision,
"tree": tree,
"treeDigest": build["treeDigest"],
"build": build,
"containers": copy.deepcopy(containers),
"live": None,
}
artifacts = {
"containers": copy.deepcopy(containers),
"compose_sha256": incident_contract._COMPOSE_SHA256,
}
state = {
"schema_version": 3,
"repository": diagnosis._REPOSITORY,
"slug": "privatemuse",
"compose_project": "privatemuse",
"revision": revision,
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": False,
"live_artifacts": copy.deepcopy(artifacts),
"live_receipt": copy.deepcopy(receipt),
"rollback_snapshot": {},
"status": "failed",
"message": diagnosis._FAILURE_MESSAGE,
"updated_at_epoch": 1790271038,
"artifacts": copy.deepcopy(artifacts),
"last_attempt": {
"revision": diagnosis._FAILED_REVISION,
"status": "failed",
"message": diagnosis._FAILURE_MESSAGE,
"updated_at_epoch": 1790271038,
"metadata": {"operation": "delivery"},
},
}
current_containers = copy.deepcopy(containers)
current_containers["privatemuse-backend-1"]["container_id"] = adoption._OBSERVED_BACKEND_ID
current_receipt = copy.deepcopy(receipt)
current_receipt["containers"] = copy.deepcopy(current_containers)
current_artifacts = {
"containers": copy.deepcopy(current_containers),
"compose_sha256": incident_contract._COMPOSE_SHA256,
}
return state, receipt, current_receipt, current_artifacts
class FakeController:
def __init__(self, current_receipt, current_artifacts):
self.current_receipt = current_receipt
self.current_artifacts = current_artifacts
self.verified_image_maps = []
def repository_path(self, repository, _project):
assert repository == diagnosis._REPOSITORY
return Path("/unused/fixed/repository.git")
def validate_recorded_receipt(self, _project, receipt, repository, revision):
assert receipt["repository"] == repository
assert receipt["revision"] == revision
return receipt["build"]
def verify_deployment(self, _project, _metadata, *, expected_image_ids):
self.verified_image_maps.append(expected_image_ids)
return copy.deepcopy(self.current_receipt)
def deployment_artifacts(self, _project):
return copy.deepcopy(self.current_artifacts)
def _install_fixture(monkeypatch, tmp_path, evidence_sequence=None):
state, old_receipt, current_receipt, current_artifacts = _fixture_state()
state_path = tmp_path / "privatemuse.json"
original_bytes = adoption._json_bytes(state)
state_path.write_bytes(original_bytes)
state_path.chmod(0o600)
transaction_dir = tmp_path / "privatemuse-f84-adoption"
controller = FakeController(current_receipt, current_artifacts)
project = {
"slug": "privatemuse",
"compose_project": "privatemuse",
"containers": list(diagnosis._CONTAINERS.values()),
}
sequence = list(evidence_sequence or [_incident_evidence(), _incident_evidence(), _post_adoption_evidence()])
@contextmanager
def no_controller_locks():
yield
monkeypatch.setattr(adoption, "_STATE_PATH", state_path)
monkeypatch.setattr(adoption, "_INCIDENT_DIR", transaction_dir)
monkeypatch.setattr(adoption, "_BACKUP_PATH", transaction_dir / "original-state.json")
monkeypatch.setattr(adoption, "_RECEIPT_BACKUP_PATH", transaction_dir / "original-live-receipt.json")
monkeypatch.setattr(adoption, "_AUDIT_PATH", transaction_dir / "approval-audit.json")
monkeypatch.setattr(adoption, "_CANDIDATE_PATH", transaction_dir / "baseline-state.json")
monkeypatch.setattr(adoption, "_MANIFEST_PATH", transaction_dir / "transaction-manifest.json")
monkeypatch.setattr(adoption, "_TRANSACTION_FILES", {
"original-state.json",
"original-live-receipt.json",
"approval-audit.json",
"baseline-state.json",
"transaction-manifest.json",
})
monkeypatch.setattr(adoption, "exclusive_controller_locks", no_controller_locks)
monkeypatch.setattr(adoption, "_protected_main_revision", lambda *_args: diagnosis._FAILED_REVISION)
monkeypatch.setattr(diagnosis, "_load_controller_module", lambda: controller)
monkeypatch.setattr(diagnosis, "_expected_project", lambda: copy.deepcopy(project))
monkeypatch.setattr(
diagnosis,
"collect_live_evidence_under_lock",
lambda: copy.deepcopy(sequence.pop(0) if sequence else _post_adoption_evidence()),
)
monkeypatch.setattr(adoption, "_fsync_directory", lambda _path: None)
monkeypatch.setattr(adoption.os, "fchown", lambda *_args: None, raising=False)
return state_path, transaction_dir, original_bytes, state, old_receipt, current_receipt, current_artifacts, controller
def _run_adoption():
return adoption.adopt_current_runtime(
approval_reference=adoption._APPROVAL_REFERENCE,
acknowledge_unknown_historical_configuration=True,
apply=True,
)
def test_adoption_writes_an_audited_current_baseline_and_preserves_failure_history(monkeypatch, tmp_path):
state_path, transaction_dir, original_bytes, original, _old_receipt, current_receipt, artifacts, controller = (
_install_fixture(monkeypatch, tmp_path)
)
result = _run_adoption()
state = json.loads(state_path.read_text(encoding="utf-8"))
audit = json.loads((transaction_dir / "approval-audit.json").read_text(encoding="utf-8"))
assert result["status"] == "adopted"
assert (transaction_dir / "original-state.json").read_bytes() == original_bytes
assert json.loads((transaction_dir / "original-live-receipt.json").read_text()) == original["live_receipt"]
assert state["live_receipt"] == current_receipt
assert state["live_artifacts"] == artifacts
assert state["last_attempt"] == original["last_attempt"]
assert state["status"] == "failed"
assert state["live_revision"] == diagnosis._LIVE_REVISION
assert state["f84_runtime_adoption"]["historical_effective_configuration"] == "unknown"
assert state["f84_runtime_adoption"]["historical_equivalence_claimed"] is False
assert audit["approval_reference"] == adoption._APPROVAL_REFERENCE
assert audit["known_configuration_delta"] == {
"key": diagnosis._HEALTH_PATH_KEY,
"prior_file_value": diagnosis._OLD_HEALTH_PATH,
"adopted_runtime_value": diagnosis._CURRENT_HEALTH_PATH,
}
assert "secret" not in (transaction_dir / "approval-audit.json").read_text().lower()
assert controller.verified_image_maps[-1] == adoption._expected_image_ids_by_name()
def test_adoption_is_idempotent_after_a_verified_commit(monkeypatch, tmp_path):
state_path, _transaction_dir, *_rest = _install_fixture(monkeypatch, tmp_path)
assert _run_adoption()["status"] == "adopted"
adopted_bytes = state_path.read_bytes()
assert _run_adoption()["status"] == "already_adopted"
assert state_path.read_bytes() == adopted_bytes
def test_prepared_partial_transaction_resumes_only_after_fresh_checks(monkeypatch, tmp_path):
bad_second = _incident_evidence()
bad_second["audio_health"] = "unhealthy"
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(
monkeypatch,
tmp_path,
evidence_sequence=[_incident_evidence(), bad_second, _incident_evidence(), _incident_evidence(), _post_adoption_evidence()],
)
with pytest.raises(adoption.AdoptionError, match="fresh live evidence"):
_run_adoption()
assert state_path.read_bytes() == original_bytes
assert (transaction_dir / "approval-audit.json").is_file()
resumed = _run_adoption()
assert resumed["status"] == "adoption_resumed"
assert json.loads(state_path.read_text())["f84_runtime_adoption"]["approval_reference"] == adoption._APPROVAL_REFERENCE
def test_incomplete_transaction_fails_closed_without_replacing_state(monkeypatch, tmp_path):
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
transaction_dir.mkdir()
(transaction_dir / "unexpected-partial-file").write_bytes(b"preserve")
with pytest.raises(adoption.AdoptionError, match="incomplete adoption transaction"):
_run_adoption()
assert state_path.read_bytes() == original_bytes
assert (transaction_dir / "unexpected-partial-file").read_bytes() == b"preserve"
def test_post_write_verification_failure_restores_exact_original_state(monkeypatch, tmp_path):
invalid_post = _post_adoption_evidence()
invalid_post["backend_runtime_mount_network_port_and_env_match_current_compose"] = False
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(
monkeypatch,
tmp_path,
evidence_sequence=[_incident_evidence(), _incident_evidence(), invalid_post],
)
with pytest.raises(adoption.AdoptionError, match="post-adoption runtime"):
_run_adoption()
assert state_path.read_bytes() == original_bytes
def test_third_party_state_after_replace_is_preserved_without_rollback_overwrite(
monkeypatch, tmp_path
):
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
original_collector = diagnosis.collect_live_evidence_under_lock
calls = 0
third_party_bytes = b"third-party-controller-update"
def diverge_after_state_replace():
nonlocal calls
calls += 1
evidence = original_collector()
if calls == 3:
state_path.write_bytes(third_party_bytes)
return evidence
monkeypatch.setattr(diagnosis, "collect_live_evidence_under_lock", diverge_after_state_replace)
with pytest.raises(adoption.AdoptionError, match="diverged after adoption failure"):
_run_adoption()
assert state_path.read_bytes() == third_party_bytes
assert (transaction_dir / "original-state.json").read_bytes() == original_bytes
def test_state_rename_failure_leaves_original_state_and_transaction_evidence(
monkeypatch, tmp_path
):
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
real_replace = adoption.os.replace
def fail_state_rename(source, destination):
if Path(destination) == state_path:
raise OSError("injected state rename failure")
return real_replace(source, destination)
monkeypatch.setattr(adoption.os, "replace", fail_state_rename)
with pytest.raises(adoption.AdoptionError, match="state replacement did not complete"):
_run_adoption()
assert state_path.read_bytes() == original_bytes
assert (transaction_dir / "original-state.json").read_bytes() == original_bytes
@pytest.mark.skipif(sys.platform == "win32", reason="controller directory fsync requires Linux")
def test_directory_fsync_failure_after_rename_restores_original_state(monkeypatch, tmp_path):
real_fsync_directory = adoption._fsync_directory
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
real_replace = adoption.os.replace
state_was_replaced = False
injected = False
def track_state_replace(source, destination):
nonlocal state_was_replaced
real_replace(source, destination)
if Path(destination) == state_path:
state_was_replaced = True
def fail_first_post_replace_fsync(path):
nonlocal injected
if state_was_replaced and Path(path) == state_path.parent and not injected:
injected = True
raise OSError("injected directory fsync failure")
return real_fsync_directory(path)
monkeypatch.setattr(adoption.os, "replace", track_state_replace)
monkeypatch.setattr(adoption, "_fsync_directory", fail_first_post_replace_fsync)
with pytest.raises(adoption.AdoptionError, match="state replacement did not complete"):
_run_adoption()
assert injected is True
assert state_path.read_bytes() == original_bytes
@pytest.mark.parametrize(
("field", "replacement"),
[
("schema", "unexpected-schema"),
("action", "unbounded-adoption"),
("known_configuration_delta", {}),
("container_ids", {"audio": "tampered"}),
("image_ids", {"audio": "sha256:tampered"}),
],
)
def test_candidate_verification_rejects_tampered_audit_fields(
field, replacement
):
original_state, original_receipt, current_receipt, current_artifacts = _fixture_state()
original_bytes = adoption._json_bytes(original_state)
candidate, candidate_bytes, audit, audit_bytes, _manifest, _manifest_bytes = (
adoption._prepare_transaction(
original_bytes,
original_state,
original_receipt,
receipt=current_receipt,
artifacts=current_artifacts,
)
)
tampered_audit = copy.deepcopy(audit)
tampered_audit[field] = replacement
with pytest.raises(adoption.AdoptionError, match="prepared adoption baseline"):
adoption._verify_transaction_candidate(
original_bytes=original_bytes,
original_state=original_state,
original_receipt=original_receipt,
current_receipt=current_receipt,
current_artifacts=current_artifacts,
candidate_bytes=candidate_bytes,
candidate=candidate,
audit_bytes=audit_bytes,
audit=tampered_audit,
)
def test_state_race_immediately_before_replace_does_not_clobber_new_bytes(monkeypatch, tmp_path):
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
original_main_check = adoption._protected_main_revision
calls = 0
def change_state_on_final_head_check(controller, project):
nonlocal calls
calls += 1
if calls == 3:
state_path.write_bytes(b"concurrent-controller-update")
return original_main_check(controller, project)
monkeypatch.setattr(adoption, "_protected_main_revision", change_state_on_final_head_check)
with pytest.raises(adoption.AdoptionError):
_run_adoption()
assert state_path.read_bytes() == b"concurrent-controller-update"
assert (tmp_path / "privatemuse-f84-adoption" / "original-state.json").read_bytes() == original_bytes
def test_mutation_requires_exact_approval_acknowledgement_and_apply(monkeypatch, tmp_path):
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
with pytest.raises(adoption.AdoptionError, match="approval reference"):
adoption.adopt_current_runtime(
approval_reference="other",
acknowledge_unknown_historical_configuration=True,
apply=True,
)
with pytest.raises(adoption.AdoptionError, match="acknowledgement"):
adoption.adopt_current_runtime(
approval_reference=adoption._APPROVAL_REFERENCE,
acknowledge_unknown_historical_configuration=False,
apply=True,
)
with pytest.raises(adoption.AdoptionError, match="--apply"):
adoption.adopt_current_runtime(
approval_reference=adoption._APPROVAL_REFERENCE,
acknowledge_unknown_historical_configuration=True,
apply=False,
)
assert state_path.read_bytes() == original_bytes
def test_fresh_runtime_drift_rejects_before_creating_adoption_files(monkeypatch, tmp_path):
bad = _incident_evidence()
bad["backend_networks"] = ["unexpected_network"]
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(
monkeypatch,
tmp_path,
evidence_sequence=[bad],
)
with pytest.raises(adoption.AdoptionError, match="fresh live evidence"):
_run_adoption()
assert state_path.read_bytes() == original_bytes
assert not transaction_dir.exists()
def test_protected_main_change_rejects_before_state_or_audit_mutation(monkeypatch, tmp_path):
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
monkeypatch.setattr(adoption, "_protected_main_revision", lambda *_args: "0" * 40)
with pytest.raises(adoption.AdoptionError, match="protected main"):
_run_adoption()
assert state_path.read_bytes() == original_bytes
assert not transaction_dir.exists()
@pytest.mark.skipif(sys.platform == "win32", reason="controller locks use Linux flock")
def test_exclusive_controller_locks_are_ordered_and_refuse_contention(monkeypatch, tmp_path):
import fcntl
project_lock = tmp_path / "project.lock"
global_lock = tmp_path / "global.lock"
project_lock.write_bytes(b"")
global_lock.write_bytes(b"")
monkeypatch.setattr(diagnosis, "_PROJECT_LOCK", project_lock)
monkeypatch.setattr(diagnosis, "_GLOBAL_LOCK", global_lock)
real_open = os.open
opened = []
def recording_open(path, *args, **kwargs):
opened.append(Path(path))
return real_open(path, *args, **kwargs)
monkeypatch.setattr(adoption.os, "open", recording_open)
with adoption.exclusive_controller_locks():
assert opened == [project_lock, global_lock]
held_descriptor = real_open(project_lock, os.O_RDWR)
try:
fcntl.flock(held_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
opened.clear()
with pytest.raises(adoption.AdoptionError, match="deployment is active"), adoption.exclusive_controller_locks():
pytest.fail("contended project lock unexpectedly acquired")
assert opened == [project_lock]
finally:
fcntl.flock(held_descriptor, fcntl.LOCK_UN)
os.close(held_descriptor)