383 lines
15 KiB
Python
383 lines
15 KiB
Python
import ast
|
|
import copy
|
|
import re
|
|
|
|
from backend.app.services.media_codec_dashboard_deployment_reconciliation import (
|
|
_RECONCILE_SCRIPT,
|
|
MediaCodecDashboardDeploymentReconciliation,
|
|
)
|
|
from backend.tests.test_media_codec_dashboard_deployment_reconciliation import (
|
|
FakePortainer,
|
|
)
|
|
|
|
|
|
def _load_reconcile_script_function(name: str):
|
|
tree = ast.parse(_RECONCILE_SCRIPT)
|
|
function = next(
|
|
(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name),
|
|
None,
|
|
)
|
|
assert function is not None, f"reconciliation script function {name} is missing"
|
|
namespace = {"re": re}
|
|
exec(compile(ast.Module(body=[function], type_ignores=[]), "<reconcile-test>", "exec"), namespace) # noqa: S102
|
|
return namespace[name]
|
|
|
|
|
|
def test_full_receipt_snapshot_recovery_requires_exact_failed_state():
|
|
classify = _load_reconcile_script_function("_is_current_verified_receipt_snapshot_baseline")
|
|
live = "a" * 40
|
|
failed = "b" * 40
|
|
artifact = {"image_id": "sha256:" + "c" * 64}
|
|
state = {
|
|
"schema_version": 3,
|
|
"repository": "jens/media-codec-dashboard",
|
|
"revision": live,
|
|
"live_revision": live,
|
|
"status": "failed",
|
|
"live_status": "unknown",
|
|
"live_legacy_provenance": False,
|
|
"updated_at_epoch": 123,
|
|
"last_attempt": {
|
|
"revision": failed,
|
|
"status": "failed",
|
|
"message": "Existing production containers require a valid healthy live deployment state",
|
|
"updated_at_epoch": 123,
|
|
},
|
|
"live_artifacts": {
|
|
"compose_sha256": "d" * 64,
|
|
"containers": {"Media-Codec-Dashboard": artifact},
|
|
},
|
|
"live_receipt": {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": "jens/media-codec-dashboard",
|
|
"revision": live,
|
|
"live": None,
|
|
"containers": {"Media-Codec-Dashboard": artifact},
|
|
},
|
|
}
|
|
assert classify(state, "jens/media-codec-dashboard", "Media-Codec-Dashboard", live, failed)
|
|
for path, value in (
|
|
(("live_status",), "healthy"),
|
|
(("live_legacy_provenance",), True),
|
|
(("last_attempt", "message"), "unrelated failure"),
|
|
(("last_attempt", "updated_at_epoch"), 122),
|
|
(("live_artifacts", "compose_sha256"), ""),
|
|
(("live_receipt", "containers"), {}),
|
|
):
|
|
changed = copy.deepcopy(state)
|
|
target = changed
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
target[path[-1]] = value
|
|
assert not classify(changed, "jens/media-codec-dashboard", "Media-Codec-Dashboard", live, failed)
|
|
|
|
|
|
def test_reconciliation_accepts_only_the_receipt_proven_legacy_state_ahead_of_runtime():
|
|
classify = _load_reconcile_script_function("_is_exact_legacy_state_ahead_baseline")
|
|
recorded = "b" * 40
|
|
live = "9" * 40
|
|
attempt = "d" * 40
|
|
artifact = {
|
|
"container_id": "f" * 64,
|
|
"image_id": "sha256:" + "c" * 64,
|
|
"image_ref": "media-codec-dashboard-media-codec-dashboard",
|
|
"container_revision": live,
|
|
"image_revision": live,
|
|
"image_source": "",
|
|
"image_tree": "",
|
|
"image_version": "",
|
|
"image_created": "",
|
|
}
|
|
state = {
|
|
"schema_version": 3,
|
|
"repository": "jens/media-codec-dashboard",
|
|
"revision": recorded,
|
|
"live_revision": recorded,
|
|
"live_status": "unknown",
|
|
"live_legacy_provenance": False,
|
|
"status": "failed",
|
|
"updated_at_epoch": 30,
|
|
"last_attempt": {
|
|
"revision": attempt,
|
|
"status": "failed",
|
|
"message": "Existing production containers require a valid healthy live deployment state",
|
|
"updated_at_epoch": 30,
|
|
"metadata": {
|
|
"id": attempt[:12],
|
|
"repository": "jens/media-codec-dashboard",
|
|
"revision": attempt,
|
|
"tree": "1" * 40,
|
|
"treeDigest": "2" * 64,
|
|
"builtAt": "2026-09-20T11:17:29.051Z",
|
|
},
|
|
},
|
|
"live_artifacts": {
|
|
"compose_sha256": "3" * 64,
|
|
"containers": {"Media-Codec-Dashboard": artifact},
|
|
},
|
|
"live_receipt": {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": "jens/media-codec-dashboard",
|
|
"revision": recorded,
|
|
"tree": "4" * 40,
|
|
"treeDigest": "5" * 64,
|
|
"build": {
|
|
"id": recorded[:12],
|
|
"repository": "jens/media-codec-dashboard",
|
|
"revision": recorded,
|
|
"tree": "4" * 40,
|
|
"treeDigest": "5" * 64,
|
|
"builtAt": "2026-09-16T23:43:44.387Z",
|
|
},
|
|
"containers": {"Media-Codec-Dashboard": artifact},
|
|
"live": None,
|
|
},
|
|
}
|
|
git_facts = {
|
|
"recorded_to_live": 1,
|
|
"canonical_recorded_to_live": 1,
|
|
"live_to_recorded": 0,
|
|
"canonical_live_to_recorded": 0,
|
|
"mounted_recorded_main": 0,
|
|
"canonical_recorded_main": 0,
|
|
"canonical_live_main": 0,
|
|
"recorded_to_attempt": 0,
|
|
"canonical_recorded_to_attempt": 0,
|
|
"attempt_to_main": 0,
|
|
"canonical_attempt_to_main": 0,
|
|
}
|
|
|
|
assert classify(
|
|
state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
git_facts,
|
|
) is True
|
|
|
|
recreated_container_state = copy.deepcopy(state)
|
|
recreated_container_state["live_artifacts"]["containers"]["Media-Codec-Dashboard"][
|
|
"container_id"
|
|
] = "e" * 64
|
|
recreated_container_state["live_receipt"]["containers"]["Media-Codec-Dashboard"][
|
|
"container_id"
|
|
] = "e" * 64
|
|
assert classify(
|
|
recreated_container_state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
git_facts,
|
|
) is True
|
|
|
|
wrong_artifact_state = copy.deepcopy(state)
|
|
wrong_artifact_state["live_artifacts"]["containers"]["Media-Codec-Dashboard"] = {
|
|
**artifact,
|
|
"container_id": "e" * 64,
|
|
}
|
|
assert classify(
|
|
wrong_artifact_state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
git_facts,
|
|
) is False
|
|
|
|
for field, changed_value in {
|
|
"image_id": "sha256:" + "a" * 64,
|
|
"image_ref": "changed-image-reference",
|
|
"container_revision": "8" * 40,
|
|
"image_revision": "8" * 40,
|
|
"image_source": "changed-source",
|
|
"image_tree": "changed-tree",
|
|
"image_version": "changed-version",
|
|
"image_created": "changed-created",
|
|
}.items():
|
|
changed_immutable_state = copy.deepcopy(recreated_container_state)
|
|
changed_immutable_state["live_artifacts"]["containers"]["Media-Codec-Dashboard"][
|
|
field
|
|
] = changed_value
|
|
changed_immutable_state["live_receipt"]["containers"]["Media-Codec-Dashboard"][
|
|
field
|
|
] = changed_value
|
|
assert classify(
|
|
changed_immutable_state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
git_facts,
|
|
) is False
|
|
|
|
malformed_stored_container_id = copy.deepcopy(recreated_container_state)
|
|
malformed_stored_container_id["live_artifacts"]["containers"]["Media-Codec-Dashboard"][
|
|
"container_id"
|
|
] = "not-a-container-id"
|
|
malformed_stored_container_id["live_receipt"]["containers"]["Media-Codec-Dashboard"][
|
|
"container_id"
|
|
] = "not-a-container-id"
|
|
assert classify(
|
|
malformed_stored_container_id,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
git_facts,
|
|
) is False
|
|
|
|
malformed_live_artifact = {**artifact, "container_id": "not-a-container-id"}
|
|
assert classify(
|
|
recreated_container_state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
malformed_live_artifact,
|
|
git_facts,
|
|
) is False
|
|
|
|
wrong_failure_state = copy.deepcopy(state)
|
|
wrong_failure_state["last_attempt"]["message"] = "different failure"
|
|
assert classify(
|
|
wrong_failure_state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
git_facts,
|
|
) is False
|
|
|
|
wrong_ancestry = {**git_facts, "live_to_recorded": 1}
|
|
assert classify(
|
|
state,
|
|
"jens/media-codec-dashboard",
|
|
"Media-Codec-Dashboard",
|
|
recorded,
|
|
live,
|
|
artifact,
|
|
wrong_ancestry,
|
|
) is False
|
|
|
|
|
|
def test_reconciliation_reports_specific_state_drift_failure_boundaries():
|
|
expected = {
|
|
55: "controller state for the current live revision is internally inconsistent",
|
|
56: "failed deployment attempt is not a descendant of the live revision",
|
|
57: "failed deployment attempt is not an ancestor of main",
|
|
58: "recorded live deployment revision is invalid",
|
|
59: "stale controller state metadata is not a supported recoverable baseline",
|
|
60: "recorded live deployment revision is not an ancestor of the verified runtime",
|
|
61: "mounted deployment repository ancestry differs from canonical Gitea repository",
|
|
}
|
|
|
|
for exit_code, reason in expected.items():
|
|
result = MediaCodecDashboardDeploymentReconciliation(FakePortainer(exit_code=exit_code)).run()
|
|
assert result == {"status": "validation_failed", "reason": reason}
|
|
|
|
fake = FakePortainer()
|
|
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
|
script = fake.calls[0][2]["json"]["Cmd"][2]
|
|
for exit_code in expected:
|
|
assert f"raise SystemExit({exit_code})" in script
|
|
|
|
assert 'canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")' in script
|
|
assert "def canonical_git_run(*args):" in script
|
|
assert 'canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
|
|
|
|
|
def test_reconciliation_falls_back_only_to_positive_canonical_gitea_ancestry():
|
|
fake = FakePortainer()
|
|
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
|
script = fake.calls[0][2]["json"]["Cmd"][2]
|
|
|
|
assert "live_revision = image_revision" in script
|
|
assert 'recorded_live = str(state.get("live_revision") or state.get("revision") or "")' in script
|
|
|
|
assert 'recorded_to_live = git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
|
assert "if recorded_to_live.returncode != 0:" in script
|
|
|
|
assert 'canonical_recorded = canonical_git_run("rev-parse", f"{recorded_live}^{{commit}}")' in script
|
|
assert 'canonical_live = canonical_git_run("rev-parse", f"{live_revision}^{{commit}}")' in script
|
|
assert 'canonical_main = canonical_git_run("rev-parse", "refs/heads/main")' in script
|
|
assert "canonical_main.stdout.strip() == main_revision" in script
|
|
|
|
assert 'canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
|
assert 'canonical_recorded_main = canonical_git_run("merge-base", "--is-ancestor", recorded_live, main_revision)' in script
|
|
assert 'canonical_live_main = canonical_git_run("merge-base", "--is-ancestor", live_revision, main_revision)' in script
|
|
assert "canonical_ancestry_fallback = (" in script
|
|
assert "canonical_recorded_to_live.returncode == 0" in script
|
|
assert "canonical_recorded_main.returncode == 0" in script
|
|
assert "canonical_live_main.returncode == 0" in script
|
|
assert "if not canonical_ancestry_fallback:" in script
|
|
|
|
# Positive canonical ancestry remains the normal fallback. A genuine
|
|
# history transition requires the stricter cross-repository proof below.
|
|
assert "history_transition_baseline = (" in script
|
|
assert 'recorded_updated_at = int(state.get("updated_at_epoch") or 0)' in script
|
|
assert "canonical_merge_base_value == mounted_merge_base_value" in script
|
|
assert "recorded_commit_epoch < live_commit_epoch" in script
|
|
assert "0 <= live_distance < recorded_distance" in script
|
|
assert "971df2cc305c1b447b4feb255d4e9fda871315c3" not in script
|
|
assert "adf53604739d4247193a7b5bfeb8ad8981d9d359" not in script
|
|
|
|
|
|
def test_reconciliation_requires_cross_repository_proof_for_divergent_history_transition():
|
|
fake = FakePortainer()
|
|
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
|
script = fake.calls[0][2]["json"]["Cmd"][2]
|
|
|
|
compile(script, "<media-codec-dashboard-reconciliation>", "exec")
|
|
|
|
assert "history_transition_baseline = (" in script
|
|
assert "stale_success_baseline" in script
|
|
assert "recorded_to_live.returncode == 1" in script
|
|
assert "canonical_recorded_to_live.returncode == 1" in script
|
|
assert "live_to_recorded.returncode == 1" in script
|
|
assert "canonical_live_to_recorded.returncode == 1" in script
|
|
assert "mounted_recorded_main.returncode == 0" in script
|
|
assert "canonical_recorded_main.returncode == 0" in script
|
|
assert "canonical_live_main.returncode == 0" in script
|
|
assert "canonical_merge_base_value == mounted_merge_base_value" in script
|
|
assert "mounted_merge_base_value not in {recorded_live, live_revision}" in script
|
|
assert "canonical_recorded_commit_epoch == recorded_commit_epoch" in script
|
|
assert "canonical_live_commit_epoch == live_commit_epoch" in script
|
|
assert "recorded_commit_epoch <= recorded_updated_at" in script
|
|
assert "recorded_updated_at < live_commit_epoch" in script
|
|
assert "canonical_recorded_distance_value == recorded_distance" in script
|
|
assert "canonical_live_distance_value == live_distance" in script
|
|
assert "0 <= live_distance < recorded_distance" in script
|
|
|
|
|
|
def test_current_live_unknown_recovery_is_runtime_proven_not_error_text_bound():
|
|
script = _RECONCILE_SCRIPT
|
|
historical_message = (
|
|
'attempt.get("message") == '
|
|
'"Existing production containers require a valid healthy live deployment state"'
|
|
)
|
|
|
|
# The full-receipt path requires the exact snapshot error; the older
|
|
# legacy snapshot path remains based on immutable runtime evidence.
|
|
assert script.count(historical_message) == 2
|
|
current_live_block = script[
|
|
script.index("current_live_unknown_snapshot_baseline = ("):
|
|
script.index("current_live_failed_baseline = (")
|
|
]
|
|
assert historical_message not in current_live_block
|
|
|
|
# Recovery still requires exact immutable runtime evidence, exact candidate
|
|
# metadata and the same atomic state timestamp.
|
|
assert 'current_artifacts.get("compose_sha256") == ""' in current_live_block
|
|
assert "current_receipt == {}" in current_live_block
|
|
assert "current_stored_artifact.get(field) == container_artifact.get(field)" in current_live_block
|
|
assert 'set(current_attempt_metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}' in current_live_block
|
|
assert 'attempt.get("updated_at_epoch") == state.get("updated_at_epoch")' in current_live_block
|