import copy import hashlib import json import os import sys from contextlib import contextmanager from pathlib import Path import pytest sys.path.insert( 0, str(Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"), ) import adopt_privatemuse_f84_runtime as adoption import diagnose_privatemuse_f84_receipt as diagnosis import privatemuse_f84_receipt_diagnosis as incident_contract def _incident_evidence(): return { "repository": "jens/privatemuse", "candidate_revision": diagnosis._FAILED_REVISION, "live_revision": diagnosis._LIVE_REVISION, "delivery_state": "failed", "delivery_message": diagnosis._FAILURE_MESSAGE, "candidate_container_mutation": False, "recorded_backend_container_id": adoption._RECORDED_BACKEND_ID, "observed_backend_container_id": adoption._OBSERVED_BACKEND_ID, "backend_image_id": incident_contract._EXPECTED_IMAGE_IDS["backend"], "frontend_image_id": incident_contract._EXPECTED_IMAGE_IDS["frontend"], "audio_image_id": incident_contract._EXPECTED_IMAGE_IDS["audio"], "all_recorded_image_ids_match": True, "frontend_and_audio_container_ids_match": True, "compose_source_sha256": incident_contract._COMPOSE_SHA256, "compose_source_hash_matches_receipt": True, "backend_health": "healthy", "frontend_state": "running", "audio_health": "healthy", "compose_project": "privatemuse", "backend_service": "backend", "backend_mounts": sorted(incident_contract._EXPECTED_MOUNTS), "backend_networks": sorted(incident_contract._EXPECTED_NETWORKS), "configuration_evidence": { "old_delivery_updated_at_epoch": 1790271038, "protected_env_changed_at_utc": "2026-09-24T18:13:33Z", "backup_filename": ".env.pre-modelforge-ready-1790273613.bak", "changed_key_names": [diagnosis._HEALTH_PATH_KEY], "live_value_matches_current_file": True, "live_value_matches_backup": False, "historical_effective_runtime_fingerprint": "not_recorded", "secret_values_recorded": False, }, "provider_submissions": 0, "job_retries": 0, "production_mutations": 0, "backend_runtime_mount_network_port_and_env_match_current_compose": True, "backend_frontend_same_compose_project": True, "backend_frontend_share_network": True, "backend_alias_visible_to_frontend": True, "audio_container_identity_matches_contract": True, } def _post_adoption_evidence(): evidence = _incident_evidence() evidence["recorded_backend_container_id"] = adoption._OBSERVED_BACKEND_ID return evidence def _fixture_state(): revision = diagnosis._LIVE_REVISION tree = "a" * 40 built_at = "2026-09-24T17:00:00.000Z" build = { "id": revision[:12], "repository": diagnosis._REPOSITORY, "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": built_at, } container_ids = { "backend": adoption._RECORDED_BACKEND_ID, "frontend": "1" * 64, "audio": "2" * 64, } containers = {} for role, name in diagnosis._CONTAINERS.items(): containers[name] = { "container_id": container_ids[role], "image_id": incident_contract._EXPECTED_IMAGE_IDS[role], "image_ref": f"local/privatemuse-{role}:b79", "container_revision": revision, "image_revision": revision, "image_source": diagnosis._REPOSITORY, "image_tree": tree, "image_version": revision[:12], "image_created": built_at, } receipt = { "schema": "gitea-deploy-receipt.v1", "repository": diagnosis._REPOSITORY, "revision": revision, "tree": tree, "treeDigest": build["treeDigest"], "build": build, "containers": copy.deepcopy(containers), "live": None, } artifacts = { "containers": copy.deepcopy(containers), "compose_sha256": incident_contract._COMPOSE_SHA256, } state = { "schema_version": 3, "repository": diagnosis._REPOSITORY, "slug": "privatemuse", "compose_project": "privatemuse", "revision": revision, "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_artifacts": copy.deepcopy(artifacts), "live_receipt": copy.deepcopy(receipt), "rollback_snapshot": {}, "status": "failed", "message": diagnosis._FAILURE_MESSAGE, "updated_at_epoch": 1790271038, "artifacts": copy.deepcopy(artifacts), "last_attempt": { "revision": diagnosis._FAILED_REVISION, "status": "failed", "message": diagnosis._FAILURE_MESSAGE, "updated_at_epoch": 1790271038, "metadata": {"operation": "delivery"}, }, } current_containers = copy.deepcopy(containers) current_containers["privatemuse-backend-1"]["container_id"] = adoption._OBSERVED_BACKEND_ID current_receipt = copy.deepcopy(receipt) current_receipt["containers"] = copy.deepcopy(current_containers) current_artifacts = { "containers": copy.deepcopy(current_containers), "compose_sha256": incident_contract._COMPOSE_SHA256, } return state, receipt, current_receipt, current_artifacts class FakeController: def __init__(self, current_receipt, current_artifacts): self.current_receipt = current_receipt self.current_artifacts = current_artifacts self.verified_image_maps = [] def repository_path(self, repository, _project): assert repository == diagnosis._REPOSITORY return Path("/unused/fixed/repository.git") def validate_recorded_receipt(self, _project, receipt, repository, revision): assert receipt["repository"] == repository assert receipt["revision"] == revision return receipt["build"] def verify_deployment(self, _project, _metadata, *, expected_image_ids): self.verified_image_maps.append(expected_image_ids) return copy.deepcopy(self.current_receipt) def deployment_artifacts(self, _project): return copy.deepcopy(self.current_artifacts) def _install_fixture(monkeypatch, tmp_path, evidence_sequence=None): state, old_receipt, current_receipt, current_artifacts = _fixture_state() state_path = tmp_path / "privatemuse.json" original_bytes = adoption._json_bytes(state) state_path.write_bytes(original_bytes) state_path.chmod(0o600) transaction_dir = tmp_path / "privatemuse-f84-adoption" controller = FakeController(current_receipt, current_artifacts) project = { "slug": "privatemuse", "compose_project": "privatemuse", "containers": list(diagnosis._CONTAINERS.values()), } sequence = list(evidence_sequence or [_incident_evidence(), _incident_evidence(), _post_adoption_evidence()]) @contextmanager def no_controller_locks(): yield monkeypatch.setattr(adoption, "_STATE_PATH", state_path) monkeypatch.setattr(adoption, "_INCIDENT_DIR", transaction_dir) monkeypatch.setattr(adoption, "_BACKUP_PATH", transaction_dir / "original-state.json") monkeypatch.setattr(adoption, "_RECEIPT_BACKUP_PATH", transaction_dir / "original-live-receipt.json") monkeypatch.setattr(adoption, "_AUDIT_PATH", transaction_dir / "approval-audit.json") monkeypatch.setattr(adoption, "_CANDIDATE_PATH", transaction_dir / "baseline-state.json") monkeypatch.setattr(adoption, "_MANIFEST_PATH", transaction_dir / "transaction-manifest.json") monkeypatch.setattr(adoption, "_TRANSACTION_FILES", { "original-state.json", "original-live-receipt.json", "approval-audit.json", "baseline-state.json", "transaction-manifest.json", }) monkeypatch.setattr(adoption, "exclusive_controller_locks", no_controller_locks) monkeypatch.setattr(adoption, "_protected_main_revision", lambda *_args: diagnosis._FAILED_REVISION) monkeypatch.setattr(diagnosis, "_load_controller_module", lambda: controller) monkeypatch.setattr(diagnosis, "_expected_project", lambda: copy.deepcopy(project)) monkeypatch.setattr( diagnosis, "collect_live_evidence_under_lock", lambda: copy.deepcopy(sequence.pop(0) if sequence else _post_adoption_evidence()), ) monkeypatch.setattr(adoption, "_fsync_directory", lambda _path: None) monkeypatch.setattr(adoption.os, "fchown", lambda *_args: None, raising=False) return state_path, transaction_dir, original_bytes, state, old_receipt, current_receipt, current_artifacts, controller def _run_adoption(): return adoption.adopt_current_runtime( approval_reference=adoption._APPROVAL_REFERENCE, acknowledge_unknown_historical_configuration=True, apply=True, ) def test_adoption_writes_an_audited_current_baseline_and_preserves_failure_history(monkeypatch, tmp_path): state_path, transaction_dir, original_bytes, original, _old_receipt, current_receipt, artifacts, controller = ( _install_fixture(monkeypatch, tmp_path) ) result = _run_adoption() state = json.loads(state_path.read_text(encoding="utf-8")) audit = json.loads((transaction_dir / "approval-audit.json").read_text(encoding="utf-8")) assert result["status"] == "adopted" assert (transaction_dir / "original-state.json").read_bytes() == original_bytes assert json.loads((transaction_dir / "original-live-receipt.json").read_text()) == original["live_receipt"] assert state["live_receipt"] == current_receipt assert state["live_artifacts"] == artifacts assert state["last_attempt"] == original["last_attempt"] assert state["status"] == "failed" assert state["live_revision"] == diagnosis._LIVE_REVISION assert state["f84_runtime_adoption"]["historical_effective_configuration"] == "unknown" assert state["f84_runtime_adoption"]["historical_equivalence_claimed"] is False assert audit["approval_reference"] == adoption._APPROVAL_REFERENCE assert audit["known_configuration_delta"] == { "key": diagnosis._HEALTH_PATH_KEY, "prior_file_value": diagnosis._OLD_HEALTH_PATH, "adopted_runtime_value": diagnosis._CURRENT_HEALTH_PATH, } assert "secret" not in (transaction_dir / "approval-audit.json").read_text().lower() assert controller.verified_image_maps[-1] == adoption._expected_image_ids_by_name() def test_adoption_is_idempotent_after_a_verified_commit(monkeypatch, tmp_path): state_path, _transaction_dir, *_rest = _install_fixture(monkeypatch, tmp_path) assert _run_adoption()["status"] == "adopted" adopted_bytes = state_path.read_bytes() assert _run_adoption()["status"] == "already_adopted" assert state_path.read_bytes() == adopted_bytes def test_prepared_partial_transaction_resumes_only_after_fresh_checks(monkeypatch, tmp_path): bad_second = _incident_evidence() bad_second["audio_health"] = "unhealthy" state_path, transaction_dir, original_bytes, *_rest = _install_fixture( monkeypatch, tmp_path, evidence_sequence=[_incident_evidence(), bad_second, _incident_evidence(), _incident_evidence(), _post_adoption_evidence()], ) with pytest.raises(adoption.AdoptionError, match="fresh live evidence"): _run_adoption() assert state_path.read_bytes() == original_bytes assert (transaction_dir / "approval-audit.json").is_file() resumed = _run_adoption() assert resumed["status"] == "adoption_resumed" assert json.loads(state_path.read_text())["f84_runtime_adoption"]["approval_reference"] == adoption._APPROVAL_REFERENCE def test_incomplete_transaction_fails_closed_without_replacing_state(monkeypatch, tmp_path): state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) transaction_dir.mkdir() (transaction_dir / "unexpected-partial-file").write_bytes(b"preserve") with pytest.raises(adoption.AdoptionError, match="incomplete adoption transaction"): _run_adoption() assert state_path.read_bytes() == original_bytes assert (transaction_dir / "unexpected-partial-file").read_bytes() == b"preserve" def test_post_write_verification_failure_restores_exact_original_state(monkeypatch, tmp_path): invalid_post = _post_adoption_evidence() invalid_post["backend_runtime_mount_network_port_and_env_match_current_compose"] = False state_path, _transaction_dir, original_bytes, *_rest = _install_fixture( monkeypatch, tmp_path, evidence_sequence=[_incident_evidence(), _incident_evidence(), invalid_post], ) with pytest.raises(adoption.AdoptionError, match="post-adoption runtime"): _run_adoption() assert state_path.read_bytes() == original_bytes def test_third_party_state_after_replace_is_preserved_without_rollback_overwrite( monkeypatch, tmp_path ): state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) original_collector = diagnosis.collect_live_evidence_under_lock calls = 0 third_party_bytes = b"third-party-controller-update" def diverge_after_state_replace(): nonlocal calls calls += 1 evidence = original_collector() if calls == 3: state_path.write_bytes(third_party_bytes) return evidence monkeypatch.setattr(diagnosis, "collect_live_evidence_under_lock", diverge_after_state_replace) with pytest.raises(adoption.AdoptionError, match="diverged after adoption failure"): _run_adoption() assert state_path.read_bytes() == third_party_bytes assert (transaction_dir / "original-state.json").read_bytes() == original_bytes def test_state_rename_failure_leaves_original_state_and_transaction_evidence( monkeypatch, tmp_path ): state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) real_replace = adoption.os.replace def fail_state_rename(source, destination): if Path(destination) == state_path: raise OSError("injected state rename failure") return real_replace(source, destination) monkeypatch.setattr(adoption.os, "replace", fail_state_rename) with pytest.raises(adoption.AdoptionError, match="state replacement did not complete"): _run_adoption() assert state_path.read_bytes() == original_bytes assert (transaction_dir / "original-state.json").read_bytes() == original_bytes @pytest.mark.skipif(sys.platform == "win32", reason="controller directory fsync requires Linux") def test_directory_fsync_failure_after_rename_restores_original_state(monkeypatch, tmp_path): real_fsync_directory = adoption._fsync_directory state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) real_replace = adoption.os.replace state_was_replaced = False injected = False def track_state_replace(source, destination): nonlocal state_was_replaced real_replace(source, destination) if Path(destination) == state_path: state_was_replaced = True def fail_first_post_replace_fsync(path): nonlocal injected if state_was_replaced and Path(path) == state_path.parent and not injected: injected = True raise OSError("injected directory fsync failure") return real_fsync_directory(path) monkeypatch.setattr(adoption.os, "replace", track_state_replace) monkeypatch.setattr(adoption, "_fsync_directory", fail_first_post_replace_fsync) with pytest.raises(adoption.AdoptionError, match="state replacement did not complete"): _run_adoption() assert injected is True assert state_path.read_bytes() == original_bytes @pytest.mark.parametrize( ("field", "replacement"), [ ("schema", "unexpected-schema"), ("action", "unbounded-adoption"), ("known_configuration_delta", {}), ("container_ids", {"audio": "tampered"}), ("image_ids", {"audio": "sha256:tampered"}), ], ) def test_candidate_verification_rejects_tampered_audit_fields( field, replacement ): original_state, original_receipt, current_receipt, current_artifacts = _fixture_state() original_bytes = adoption._json_bytes(original_state) candidate, candidate_bytes, audit, audit_bytes, _manifest, _manifest_bytes = ( adoption._prepare_transaction( original_bytes, original_state, original_receipt, receipt=current_receipt, artifacts=current_artifacts, ) ) tampered_audit = copy.deepcopy(audit) tampered_audit[field] = replacement with pytest.raises(adoption.AdoptionError, match="prepared adoption baseline"): adoption._verify_transaction_candidate( original_bytes=original_bytes, original_state=original_state, original_receipt=original_receipt, current_receipt=current_receipt, current_artifacts=current_artifacts, candidate_bytes=candidate_bytes, candidate=candidate, audit_bytes=audit_bytes, audit=tampered_audit, ) def test_state_race_immediately_before_replace_does_not_clobber_new_bytes(monkeypatch, tmp_path): state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) original_main_check = adoption._protected_main_revision calls = 0 def change_state_on_final_head_check(controller, project): nonlocal calls calls += 1 if calls == 3: state_path.write_bytes(b"concurrent-controller-update") return original_main_check(controller, project) monkeypatch.setattr(adoption, "_protected_main_revision", change_state_on_final_head_check) with pytest.raises(adoption.AdoptionError): _run_adoption() assert state_path.read_bytes() == b"concurrent-controller-update" assert (tmp_path / "privatemuse-f84-adoption" / "original-state.json").read_bytes() == original_bytes def test_mutation_requires_exact_approval_acknowledgement_and_apply(monkeypatch, tmp_path): state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) with pytest.raises(adoption.AdoptionError, match="approval reference"): adoption.adopt_current_runtime( approval_reference="other", acknowledge_unknown_historical_configuration=True, apply=True, ) with pytest.raises(adoption.AdoptionError, match="acknowledgement"): adoption.adopt_current_runtime( approval_reference=adoption._APPROVAL_REFERENCE, acknowledge_unknown_historical_configuration=False, apply=True, ) with pytest.raises(adoption.AdoptionError, match="--apply"): adoption.adopt_current_runtime( approval_reference=adoption._APPROVAL_REFERENCE, acknowledge_unknown_historical_configuration=True, apply=False, ) assert state_path.read_bytes() == original_bytes def test_fresh_runtime_drift_rejects_before_creating_adoption_files(monkeypatch, tmp_path): bad = _incident_evidence() bad["backend_networks"] = ["unexpected_network"] state_path, transaction_dir, original_bytes, *_rest = _install_fixture( monkeypatch, tmp_path, evidence_sequence=[bad], ) with pytest.raises(adoption.AdoptionError, match="fresh live evidence"): _run_adoption() assert state_path.read_bytes() == original_bytes assert not transaction_dir.exists() def test_protected_main_change_rejects_before_state_or_audit_mutation(monkeypatch, tmp_path): state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path) monkeypatch.setattr(adoption, "_protected_main_revision", lambda *_args: "0" * 40) with pytest.raises(adoption.AdoptionError, match="protected main"): _run_adoption() assert state_path.read_bytes() == original_bytes assert not transaction_dir.exists() @pytest.mark.skipif(sys.platform == "win32", reason="controller locks use Linux flock") def test_exclusive_controller_locks_are_ordered_and_refuse_contention(monkeypatch, tmp_path): import fcntl project_lock = tmp_path / "project.lock" global_lock = tmp_path / "global.lock" project_lock.write_bytes(b"") global_lock.write_bytes(b"") monkeypatch.setattr(diagnosis, "_PROJECT_LOCK", project_lock) monkeypatch.setattr(diagnosis, "_GLOBAL_LOCK", global_lock) real_open = os.open opened = [] def recording_open(path, *args, **kwargs): opened.append(Path(path)) return real_open(path, *args, **kwargs) monkeypatch.setattr(adoption.os, "open", recording_open) with adoption.exclusive_controller_locks(): assert opened == [project_lock, global_lock] held_descriptor = real_open(project_lock, os.O_RDWR) try: fcntl.flock(held_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) opened.clear() with pytest.raises(adoption.AdoptionError, match="deployment is active"), adoption.exclusive_controller_locks(): pytest.fail("contended project lock unexpectedly acquired") assert opened == [project_lock] finally: fcntl.flock(held_descriptor, fcntl.LOCK_UN) os.close(held_descriptor)