Publish curated Unraid-AppOps-Gateway source from 25057d17d4995d4bd407ff0516ef69ddaec878c6

PUBLIC_SOURCE_EXPORT.md

@@ -1,3 +1,3 @@
 # Public source export
 
-This source snapshot was generated from the private canonical repository at revision `b6d6a3e1657000c6e0a482c8056a3fbebbafd711`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
+This source snapshot was generated from the private canonical repository at revision `25057d17d4995d4bd407ff0516ef69ddaec878c6`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.

PUBLIC_SOURCE_MANIFEST.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "sourceRevision": "b6d6a3e1657000c6e0a482c8056a3fbebbafd711",
+  "sourceRevision": "25057d17d4995d4bd407ff0516ef69ddaec878c6",
   "files": [
     {
       "path": ".dockerignore",
@@ -524,7 +524,7 @@
     },
     {
       "path": "backend/app/services/controller_artifact_helper_spec.py",
-      "sha256": "b6340734fcba86a341b2c29e7930ed30016d19352acd4a015d1e194c415655ac",
+      "sha256": "232048b69a06e5fe4edc1c5fab27c3032bdd829bbbe9589b7d47b1a288207ca8",
       "bytes": 6029
     },
     {
@@ -914,8 +914,8 @@
     },
     {
       "path": "backend/tests/test_appops_self_upgrade_recovery_boundaries.py",
-      "sha256": "676a465700296b081cc07eaeab3d2669f3047001b28b614e8a8a82088010f3f0",
-      "bytes": 29130
+      "sha256": "0377b7e921002dd30231d038b6f0cb2834bbc4df834d7f189e9d68e1da90af03",
+      "bytes": 40107
     },
     {
       "path": "backend/tests/test_appops_self_upgrade_runtime_model.py",
@@ -949,8 +949,8 @@
     },
     {
       "path": "backend/tests/test_controller_artifact_bootstrap.py",
-      "sha256": "d9acae805eedcf50e90222ce52f3a43ad101e558d7c1b49d4f55968b208e957f",
-      "bytes": 82161
+      "sha256": "46ceb18a80b1f413da98325bea69c80422057354fb54569cc5563fccfffdda64",
+      "bytes": 82644
     },
     {
       "path": "backend/tests/test_controller_artifact_bootstrap_created_child_recovery.py",
@@ -964,7 +964,7 @@
     },
     {
       "path": "backend/tests/test_controller_artifact_promoter.py",
-      "sha256": "e1f987217790868228daba95ab564b4fb9e44798a80070e6c1d76d0b1ab6fe0a",
+      "sha256": "284389dc63da8305927e80bf4ed4120e1acac6b0eb86b2b29f5cbbc360fa3778",
       "bytes": 7570
     },
     {

backend/app/services/controller_artifact_helper_spec.py

@@ -5,7 +5,7 @@ from typing import Any
 
 SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
 HELPER_NAME = "itworx-controller-artifact-promoter-modern"
-CONTROLLER_VERSION = "2.1.17"
+CONTROLLER_VERSION = "2.1.18"
 HELPER_LABEL = "itworx.controller.promoter"
 _REVISION = re.compile(r"^[0-9a-f]{40}$")
 _IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")

backend/tests/test_appops_self_upgrade_recovery_boundaries.py

@@ -332,8 +332,9 @@ def test_private_recovery_journal_is_0600_and_hmac_tampering_fails_closed(
     assert controller._self_upgrade_recovery_journal(recovery_id) is None
 
 
+@pytest.mark.parametrize("failed_revalidation", [False, True])
 def test_recovery_request_is_immutable_and_idempotent_for_one_key(
-    controller, monkeypatch, tmp_path,
+    controller, monkeypatch, tmp_path, failed_revalidation,
 ):
     if os.name == "nt" or os.geteuid() != 0:
         pytest.skip("The controller recovery store requires root-owned POSIX files")
@@ -353,6 +354,10 @@ def test_recovery_request_is_immutable_and_idempotent_for_one_key(
     original_raw = b'{"immutable":"original failed delivery"}\n'
     original_path.write_bytes(original_raw)
     original = {"revision": revision}
+    if failed_revalidation:
+        original["status"] = "validation_timeout"
+        monkeypatch.setattr(controller, "_self_upgrade_failed_revalidation_sha256", lambda *_args: "e" * 64)
+        monkeypatch.setattr(controller, "_self_upgrade_read_legacy_cache_json", lambda *_a, **_k: (original_raw, original))
     original_info = (original_path, original_raw, original)
     monkeypatch.setattr(
         controller,
@@ -453,6 +458,13 @@ def test_recovery_request_is_immutable_and_idempotent_for_one_key(
     assert b"journal-secret-sentinel" not in accepted_bytes[
         f"requests/{idempotency_key_sha}.json"
     ]
+    journal = json.loads(journal_bytes)["payload"]
+    request = json.loads(accepted_bytes[f"requests/{idempotency_key_sha}.json"])
+    if failed_revalidation:
+        assert journal["failedRevalidationRawSha256"] == request["failedRevalidationRawSha256"] == "e" * 64
+    else:
+        assert "failedRevalidationRawSha256" not in journal
+        assert "failedRevalidationRawSha256" not in request
 
     same_key = controller._self_upgrade_recovery_request_status(
         semantic,
@@ -479,6 +491,14 @@ def test_recovery_request_is_immutable_and_idempotent_for_one_key(
     assert different_key == ("conflict", semantic)
     assert store_snapshot() == accepted_bytes
     assert original_path.read_bytes() == original_raw
+    if failed_revalidation:
+        monkeypatch.setattr(controller, "_self_upgrade_failed_revalidation_sha256", lambda *_args: "f" * 64)
+        changed = controller._self_upgrade_recovery_request_status(
+            semantic, original_sha, idempotency_key_sha, expected_preflight,
+            expected_current_revision, reason_sha, policy,
+        )
+        assert changed == ("conflict", semantic)
+        assert store_snapshot() == accepted_bytes
 
 
 @pytest.mark.parametrize(
@@ -719,3 +739,169 @@ def test_source_root_allows_only_legacy_or_root_owned_appdata_anchor(
     assert controller._self_upgrade_source_root_safe(root) is valid
     monkeypatch.setattr(Path, "is_symlink", lambda path: path == root.parent)
     assert controller._self_upgrade_source_root_safe(root) is False
+
+
+def _failed_revalidation_pair(controller, monkeypatch):
+    revision = "c" * 40
+    semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
+    policy = {
+        "repository": controller.SELF_UPGRADE_REPOSITORY,
+        "validation_gate_version": 1, "contexts": ["Managed validation / full (push)"],
+        "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30,
+    }
+    original = {
+        "schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": semantic,
+        "gitea_repository_id": 9, "ref": "refs/heads/main",
+        "repository": controller.SELF_UPGRADE_REPOSITORY, "revision": revision,
+        "status": "validation_timeout", "reason": "validation_timeout",
+        "source": "gitea_webhook", "body_sha256": "1" * 64,
+        "delivery": "12345678-1234-4234-8234-123456789abc",
+        "validation": controller._validation_snapshot(policy, 1),
+        "audit": [{"stage": "validation_timeout", "reason": "validation_timeout"}],
+    }
+    original["validation"]["last_result"] = "timeout"
+    validation = controller._validation_snapshot(policy, 2)
+    validation.update(last_result="pass", observed=[{
+        "context": "Managed validation / full (push)", "status": "success",
+    }])
+    sibling = {
+        **original, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA,
+        "revalidation_of": semantic, "status": "failed", "reason": "deployment_failed",
+        "validation": validation,
+        "audit": [{"stage": stage} for stage in (
+            "revalidation_requested", "validation_passed", "deployment_started",
+        )] + [{"stage": "deployment_failed", "reason": "deployment_failed"}],
+    }
+    sibling.pop("source")
+    def read(path, **_kwargs):
+        if path == controller.WEBHOOK_INBOX_ROOT / f"{semantic}.json":
+            record = original
+        elif path == controller.WEBHOOK_INBOX_ROOT / f"{semantic}.revalidation.json":
+            record = sibling
+        else:
+            raise AssertionError("Only the exact immutable original and fixed sibling may be read")
+        return controller._self_upgrade_canonical_json(record), record
+    monkeypatch.setattr(controller, "_self_upgrade_read_legacy_cache_json", read)
+    return semantic, revision, policy, original, sibling
+
+
+def test_failed_revalidation_origin_binds_both_records_and_raw_sibling_bytes(controller, monkeypatch):
+    semantic, revision, policy, original, sibling = _failed_revalidation_pair(controller, monkeypatch)
+    selected = controller._self_upgrade_original_failed_delivery(semantic, revision, policy)
+    assert selected is not None and selected[2] == original
+    assert selected[1] == controller._self_upgrade_canonical_json(original)
+    sibling_hash = controller.hashlib.sha256(controller._self_upgrade_canonical_json(sibling)).hexdigest()
+    binding = {"failedRevalidationRawSha256": sibling_hash}
+    assert controller._self_upgrade_origin_binding_matches(semantic, original, policy, binding)
+    assert not controller._self_upgrade_origin_binding_matches(semantic, original, policy, {})
+    sibling["updated_at"] = 42
+    # Valid semantic data with different raw bytes must still fail the signed CAS.
+    assert controller._self_upgrade_original_failed_delivery(semantic, revision, policy) is not None
+    assert not controller._self_upgrade_origin_binding_matches(semantic, original, policy, binding)
+    original["revision"] = "d" * 40
+    assert controller._self_upgrade_original_failed_delivery(semantic, revision, policy) is None
+
+
+@pytest.mark.parametrize("mutation", [
+    "schema", "semantic_sha256", "revalidation_of", "gitea_repository_id", "ref",
+    "repository", "revision", "body_sha256", "delivery", "active", "interrupted",
+    "missing_gate", "wrong_gate", "wrong_policy", "missing_audit", "unordered_audit", "null_audit", "source",
+    "registered_legacy_request_id", "projectbrain_delivery_revalidation", "hub_delivery_revalidation",
+    "original_not_timeout", "original_wrong_reason", "original_wrong_source", "original_bad_delivery",
+    "original_missing_audit", "original_passed", "original_started", "original_null_status",
+])
+def test_failed_revalidation_origin_rejects_foreign_or_unproven_pair(controller, monkeypatch, mutation):
+    semantic, revision, policy, original, sibling = _failed_revalidation_pair(controller, monkeypatch)
+    if mutation == "active":
+        sibling["status"] = "deploying"
+    elif mutation == "interrupted":
+        sibling["reason"] = "deployment_interrupted"
+    elif mutation == "missing_gate":
+        sibling["validation"]["observed"] = []
+    elif mutation == "wrong_gate":
+        sibling["validation"]["observed"][0]["status"] = "pending"
+    elif mutation == "wrong_policy":
+        sibling["validation"]["required_contexts"] = ["Other gate"]
+    elif mutation == "null_audit":
+        sibling["audit"] = None
+    elif mutation == "original_missing_audit":
+        original["audit"] = []
+    elif mutation == "original_passed":
+        original["validation"]["last_result"] = "pass"
+    elif mutation == "original_started":
+        original["audit"].append({"stage": "deployment_started"})
+    elif mutation == "original_null_status":
+        original["status"] = []
+    elif mutation == "missing_audit":
+        sibling["audit"].pop(0)
+    elif mutation == "unordered_audit":
+        sibling["audit"].reverse()
+    elif mutation == "original_not_timeout":
+        original["status"] = "validation_failed"
+    elif mutation == "original_wrong_reason":
+        original["reason"] = "other"
+    elif mutation == "original_wrong_source":
+        original["source"] = "foreign"
+    elif mutation == "original_bad_delivery":
+        original["delivery"] = sibling["delivery"] = "invalid"
+    else:
+        sibling[mutation] = "foreign"
+    assert controller._self_upgrade_original_failed_delivery(semantic, revision, policy) is None
+
+
+@pytest.mark.parametrize("boundary", ["preimage", "authorization"])
+def test_failed_revalidation_rechecks_pair_at_execution_boundaries(controller, monkeypatch, boundary):
+    semantic, revision, policy, original, sibling = _failed_revalidation_pair(controller, monkeypatch)
+    recovery_id = "12345678-1234-4234-8234-123456789abc"
+    origin_raw = controller._self_upgrade_canonical_json(original)
+    journal = _private_recovery_payload(controller, recovery_id)
+    journal.update(
+        expectedCurrentRevision=controller.SELF_UPGRADE_OLD_REVISION,
+        originalRawSha256=controller.hashlib.sha256(origin_raw).hexdigest(),
+        failedRevalidationRawSha256=controller.hashlib.sha256(controller._self_upgrade_canonical_json(sibling)).hexdigest(),
+    )
+    monkeypatch.setattr(controller, "_self_upgrade_recovery_journal", lambda _id: journal)
+    monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {}})
+    monkeypatch.setattr(controller, "_validation_policy", lambda *_args: policy)
+    fresh = {"status": "preflight_ok", **{key: journal[key] for key in (
+        "preflightSha256", "currentContainerId", "currentImageId", "currentSourceTree",
+    )}, "currentRevision": journal["expectedCurrentRevision"], "currentRuntimeFingerprint": journal["runtimeFingerprint"]}
+    monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: fresh)
+    bindings = {key: journal[key] for key in (
+        "currentContainerId", "currentImageId", "currentSourceTree", "runtimeFingerprint",
+        "registryFingerprint", "sourceFingerprint", "stateSha256", "runtimeEnvironment",
+    )}
+    bindings["currentRevision"] = journal["expectedCurrentRevision"]
+    monkeypatch.setattr(controller, "_self_upgrade_recovery_current_bindings", lambda: bindings)
+    delivery = {"semantic_sha256": semantic, "revision": revision, "recovery_id": recovery_id,
+                "status": "deploying", "validation": {"last_result": "pass"},
+                "audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"}]}
+    request = {
+        "schema": controller._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA, "revision": revision,
+        "preflightSha256": journal["preflightSha256"], "operationId": semantic,
+        "reasonSha256": "f" * 64, "idempotencyKeySha256": journal["idempotencyKeySha256"],
+        "recoveryId": recovery_id, "recoveryOfSha256": journal["originalRawSha256"],
+        "failedRevalidationRawSha256": journal["failedRevalidationRawSha256"], "createdAt": 1,
+    }
+    secret = b"fixed-recovery-boundary-key-0123456789"
+    monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret)
+    signed_request = {**request, "hmac": controller._self_upgrade_mac(
+        secret, "recovery-request", controller._self_upgrade_canonical_json(request))}
+    monkeypatch.setattr(controller, "_self_upgrade_request_root", lambda **_kwargs: controller.SELF_UPGRADE_RECOVERY_REQUEST_ROOT)
+    def read_private(path, **_kwargs):
+        record = delivery if path.name.endswith(".delivery.json") else signed_request
+        return controller._self_upgrade_canonical_json(record), record
+    monkeypatch.setattr(controller, "_self_upgrade_read_private_json", read_private)
+    # The separate HMAC admission is covered by journal/request tests; this isolates the immutable pair CAS.
+    monkeypatch.setattr(controller, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True)
+    def check():
+        if boundary == "preimage":
+            return controller._self_upgrade_recovery_preimage_matches(delivery)
+        return controller._self_upgrade_recovery_authorization(recovery_id) is not None
+    assert check()
+    sibling["updated_at"] = 99
+    assert not check()
+    sibling.pop("updated_at")
+    assert check()
+    original["updated_at"] = 99
+    assert not check()

backend/tests/test_controller_artifact_bootstrap.py

@@ -589,7 +589,7 @@ def _health(
     *,
     revision: str = REVISION,
     tree: str = TREE,
-    version: str = "2.1.17",
+    version: str = "2.1.18",
     artifact_sha256: str = "1" * 64,
 ) -> dict[str, Any]:
     return {
@@ -1347,7 +1347,7 @@ def test_current_214_baseline_is_verified_and_bound_to_new_helper():
         "itworx.controller.promoter": "true",
         "itworx.source.repository": "jens/unraid-appops-gateway",
         "itworx.source.revision": REVISION,
-        "itworx.controller.version": "2.1.17",
+        "itworx.controller.version": "2.1.18",
         "itworx.controller.bootstrap.target.id": MODERN_CONTROLLER_CONTAINER_ID,
         "itworx.controller.bootstrap.target.image": CONTROLLER_IMAGE_ID,
         "itworx.controller.bootstrap.target.revision": MODERN_CONTROLLER_REVISION,
@@ -1826,7 +1826,7 @@ def test_successful_child_requires_exact_source_tree_version_and_health():
 
     assert result["status"] == "promoted"
     assert result["revision"] == REVISION
-    assert result["controllerVersion"] == "2.1.17"
+    assert result["controllerVersion"] == "2.1.18"
     assert result["artifactSha256"] == "1" * 64
     assert docker.started == [MODERN_HELPER_ID]
     assert docker.created[0]["Image"] == CONTROLLER_IMAGE_ID
@@ -2086,24 +2086,32 @@ def test_modern_upgrade_path_still_requires_private_archive_and_image_id(
 def test_current_helper_version_is_distinct_from_fixed_controller_baselines():
     source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes()
     namespace = bootstrap._safe_modern_spec_namespace(source)
-    assert namespace["CONTROLLER_VERSION"] == "2.1.17"
-    assert bootstrap._verified_historical_helper_version(source) == "2.1.17"
+    assert namespace["CONTROLLER_VERSION"] == "2.1.18"
+    assert bootstrap._verified_historical_helper_version(source) == "2.1.18"
     assert [baseline.version for baseline in bootstrap.CONTROLLER_BASELINES] == ["2.1.12", "2.1.14"]
     with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
         bootstrap._safe_modern_spec_namespace(RETIRED_MODERN_HELPER_SOURCE.encode("utf-8"))
     assert bootstrap._verified_historical_helper_version(RETIRED_MODERN_HELPER_SOURCE.encode("utf-8")) == "2.1.14"
 
 
-@pytest.mark.parametrize("version", ["2.1.16", "2.1.18", "9.9.9"])
+@pytest.mark.parametrize("version", ["2.1.16", "2.1.19", "9.9.9"])
 def test_current_and_historical_helper_reject_unreviewed_target_versions(version):
     source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes()
-    foreign = source.replace(b'CONTROLLER_VERSION = "2.1.17"', f'CONTROLLER_VERSION = "{version}"'.encode())
+    foreign = source.replace(b'CONTROLLER_VERSION = "2.1.18"', f'CONTROLLER_VERSION = "{version}"'.encode())
     with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
         bootstrap._safe_modern_spec_namespace(foreign)
     with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
         bootstrap._verified_historical_helper_version(foreign)
 
 
-@pytest.mark.parametrize("version", ["2.1.14", "2.1.16", "2.1.18"])
+@pytest.mark.parametrize("version", ["2.1.14", "2.1.16", "2.1.19"])
 def test_promoted_health_cannot_substitute_a_baseline_or_unknown_version(version):
     assert not bootstrap._health_matches(_health(version=version), REVISION, TREE)
+
+
+def test_previous_reviewed_helper_is_historical_only():
+    source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes()
+    prior = source.replace(b'CONTROLLER_VERSION = "2.1.18"', b'CONTROLLER_VERSION = "2.1.17"')
+    with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
+        bootstrap._safe_modern_spec_namespace(prior)
+    assert bootstrap._verified_historical_helper_version(prior) == "2.1.17"

backend/tests/test_controller_artifact_promoter.py

@@ -46,7 +46,7 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded():
     spec = helper_container_spec(revision=revision, image_id=image_id)
 
     assert HELPER_NAME == "itworx-controller-artifact-promoter-modern"
-    assert CONTROLLER_VERSION == "2.1.17"
+    assert CONTROLLER_VERSION == "2.1.18"
     assert re.fullmatch(r"2\.1\.\d+", CONTROLLER_VERSION)
     assert spec["Image"] == image_id
     assert spec["HostConfig"]["Privileged"] is False
@@ -79,7 +79,7 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded():
     assert revision in command
     assert "--image-id" in command
     assert CONTROLLER_VERSION in command
-    assert "version='2.1.17'" in command
+    assert "version='2.1.18'" in command
     assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command
     archive_start = command.index('git --git-dir="$repo" archive')
     archive_end = command.index('> "$work/controller.tar"', archive_start)
@@ -110,7 +110,7 @@ def test_controller_promoter_bootstrap_child_spec_binds_legacy_target_identity()
         "itworx.controller.promoter": "true",
         "itworx.source.repository": "jens/unraid-appops-gateway",
         "itworx.source.revision": revision,
-        "itworx.controller.version": "2.1.17",
+        "itworx.controller.version": "2.1.18",
         "itworx.controller.bootstrap.target.id": target_id,
         "itworx.controller.bootstrap.target.image": image_id,
         "itworx.controller.bootstrap.target.revision": target_revision,