Publish curated Unraid-AppOps-Gateway source from 945a919d1a3bbd2bb5bd21eb221fc9b58d795d80
This commit is contained in:
1 parent
aa9f7d178c
commit
fd4b962bdd
10 files changed
+884
-37
No files matched your search
@@ -1,3 +1,3 @@
|
||||
# Public source export
|
||||
|
||||
This source snapshot was generated from the private canonical repository at revision `ae6b891c48da412de613ce0ed52c0bc1e9908fbf`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
This source snapshot was generated from the private canonical repository at revision `945a919d1a3bbd2bb5bd21eb221fc9b58d795d80`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
+20
-15
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"sourceRevision": "ae6b891c48da412de613ce0ed52c0bc1e9908fbf",
|
||||
"sourceRevision": "945a919d1a3bbd2bb5bd21eb221fc9b58d795d80",
|
||||
"files": [
|
||||
{
|
||||
"path": ".dockerignore",
|
||||
@@ -1164,8 +1164,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_forge_secret_delivery_controller.py",
|
||||
"sha256": "1695383ed99777000bec2df4ae97d252dc242b690430f102284fa101679465ed",
|
||||
"bytes": 31582
|
||||
"sha256": "d5b35a6f9d188849171fed6f3cfcc813a0e999a9a821855520c3fc6a7502764b",
|
||||
"bytes": 32266
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_frontend_file_boundary.py",
|
||||
@@ -1204,8 +1204,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_hub_candidate_import.py",
|
||||
"sha256": "c5eb8845b6fba09cb8d49edd06b8fad1420a8a14eac0e732198fdce0b6a0a333",
|
||||
"bytes": 45629
|
||||
"sha256": "87762293c6088b5e3e71adfa4c109ec468d7931f3e10b8be6e04ce1d19f8d849",
|
||||
"bytes": 47846
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_hub_candidate_transport.py",
|
||||
@@ -1259,13 +1259,13 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_managed_validation_checkout.py",
|
||||
"sha256": "92239d5c8760034c145a917534dd4622d507af334eaa0c10db3a310b53bae482",
|
||||
"bytes": 462
|
||||
"sha256": "ada4d91ef0e22e94cf2c5f7af63f7ace8e2db1820a6b4ace7ae174ff065300fb",
|
||||
"bytes": 541
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_managed_validation_parallel.py",
|
||||
"sha256": "5654211d4ab1ab2749c269c148497f9448b0257c70779ebac063ef55780e3c11",
|
||||
"bytes": 2355
|
||||
"sha256": "8b3354bd6353831a081459b7398eac2ce4f8626251ff38422f6d217caf516ec3",
|
||||
"bytes": 2367
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_mcp_hub_candidate_preparation.py",
|
||||
@@ -1604,14 +1604,19 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_admission_controller.py",
|
||||
"sha256": "dcfb8e7a1d1ee86d6bfadef71bbd8be629a7f70a26baafb37bb134c61e78491d",
|
||||
"bytes": 57459
|
||||
"sha256": "c05470d633c3f8c745522019724de85f470df926235fe7155f9aa754ea3e4e23",
|
||||
"bytes": 69519
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_admission_policy.py",
|
||||
"sha256": "e4bdf5b454b5e17d1785b2db82819c1e43150fe84de1bf0fd242288352f4ee26",
|
||||
"bytes": 5951
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_build_acceptance_workflow.py",
|
||||
"sha256": "618c65909b89a7c52e85c6c190bf63a4c6dc202917d60d48eb71b069e5cf30c2",
|
||||
"bytes": 9336
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_capacity_controller.py",
|
||||
"sha256": "f1c12a7799f11b7dcb78e79820303a5dce6828dded26b258898e4c80f80a3ed8",
|
||||
@@ -1659,8 +1664,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_fleet_specialize.py",
|
||||
"sha256": "5bd85d16db6ef824a8c83bdafb4e1a70cbb736064611929e538df7224748f885",
|
||||
"bytes": 205798
|
||||
"sha256": "c2dd0ed7a893df8a73868e8efd27baa4ae83949dfa03f75816a3c59e66389895",
|
||||
"bytes": 220641
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_fleet_v2_execution_surface.py",
|
||||
@@ -1724,8 +1729,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_runner_task_image_producer.py",
|
||||
"sha256": "c55f54bf58fa81649ddb31a8953abadf299c2fe32d1b4411dd59889981cc7376",
|
||||
"bytes": 12112
|
||||
"sha256": "025307274b0a93038ebe2e4ceb85579d1bf012ee855a392418f5ed7d6daab617",
|
||||
"bytes": 13787
|
||||
},
|
||||
{
|
||||
"path": "backend/tests/test_secret_storage.py",
|
||||
|
||||
@@ -129,8 +129,8 @@ def _signed_headers(status_module, body: bytes, *, nonce: str | None = None):
|
||||
}
|
||||
|
||||
|
||||
def _post(address, path: str, body: bytes, headers: dict[str, str]):
|
||||
connection = http.client.HTTPConnection(*address, timeout=3)
|
||||
def _post(address, path: str, body: bytes, headers: dict[str, str], *, timeout: int = 3):
|
||||
connection = http.client.HTTPConnection(*address, timeout=timeout)
|
||||
try:
|
||||
connection.request("POST", path, body=body, headers=headers)
|
||||
response = connection.getresponse()
|
||||
@@ -497,12 +497,13 @@ def test_new_nonce_retries_same_bundle_and_changed_input_is_rejected(
|
||||
assert READ_TOKEN.decode() not in response.decode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fsync_delay_seconds", [0.0, 0.04])
|
||||
@pytest.mark.skipif(
|
||||
os.name != "posix" or getattr(os, "geteuid", lambda: 1)() != 0,
|
||||
reason="the real bundle helper requires POSIX root ownership",
|
||||
)
|
||||
def test_signed_http_delivery_uses_real_bundle_helper_and_never_rotates(
|
||||
status_module, endpoint, monkeypatch, tmp_path
|
||||
status_module, endpoint, monkeypatch, tmp_path, fsync_delay_seconds
|
||||
):
|
||||
_install_hmac_test(monkeypatch, status_module)
|
||||
import importlib
|
||||
@@ -511,6 +512,19 @@ def test_signed_http_delivery_uses_real_bundle_helper_and_never_rotates(
|
||||
assert Path(bundle_module.__file__).resolve() == (
|
||||
BROKER / "forge_secret_bundle.py"
|
||||
).resolve()
|
||||
if fsync_delay_seconds:
|
||||
real_os = bundle_module.os
|
||||
|
||||
class SlowFsyncOs:
|
||||
def __getattr__(self, name):
|
||||
return getattr(real_os, name)
|
||||
|
||||
def fsync(self, descriptor):
|
||||
time.sleep(fsync_delay_seconds)
|
||||
return real_os.fsync(descriptor)
|
||||
|
||||
# Keep actual durable writes and all ownership checks; model valid slow I/O.
|
||||
monkeypatch.setattr(bundle_module, "os", SlowFsyncOs())
|
||||
root = tmp_path / "real-bundle-root"
|
||||
root.mkdir(mode=0o700)
|
||||
root.chmod(0o700)
|
||||
@@ -541,6 +555,8 @@ def test_signed_http_delivery_uses_real_bundle_helper_and_never_rotates(
|
||||
status_module.FORGE_SECRET_DELIVERY_PATH,
|
||||
body,
|
||||
_signed_headers(status_module, body),
|
||||
# Match the existing bounded production HTTP budget for real fsync I/O.
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
status, response = send(_payload(status_module))
|
||||
|
||||
@@ -5,6 +5,7 @@ import importlib
|
||||
import io
|
||||
import json
|
||||
import contextlib
|
||||
import os
|
||||
import stat
|
||||
import struct
|
||||
import sys
|
||||
@@ -119,6 +120,20 @@ def _private_file(path: Path, data: bytes) -> None:
|
||||
path.chmod(0o600)
|
||||
|
||||
|
||||
def _candidate_directory_link_count(monkeypatch, directory: Path, count: int) -> None:
|
||||
original_lstat = Path.lstat
|
||||
|
||||
def candidate_lstat(path: Path, *args, **kwargs):
|
||||
info = original_lstat(path, *args, **kwargs)
|
||||
if path == directory:
|
||||
values = list(info)
|
||||
values[3] = count
|
||||
return os.stat_result(values)
|
||||
return info
|
||||
|
||||
monkeypatch.setattr(Path, "lstat", candidate_lstat)
|
||||
|
||||
|
||||
def _synthetic_scan_reports(
|
||||
services, images, *, created_at: str, finding_service: str | None = None
|
||||
) -> dict[str, bytes]:
|
||||
@@ -590,8 +605,10 @@ def test_unfixable_vulnerability_finding_is_bound_to_fixed_image_reference(modul
|
||||
|
||||
@pytest.mark.parametrize("finding_present", [False, True])
|
||||
@pytest.mark.parametrize("rerun_during_load", [False, True])
|
||||
@pytest.mark.parametrize("directory_link_count", [1, 2])
|
||||
def test_fake_local_import_publishes_candidate_and_requires_review_for_findings(
|
||||
modules, monkeypatch, tmp_path, finding_present, rerun_during_load
|
||||
modules, monkeypatch, tmp_path, finding_present, rerun_during_load,
|
||||
directory_link_count,
|
||||
):
|
||||
importer = modules.importer
|
||||
prepare = modules.prepare
|
||||
@@ -889,6 +906,7 @@ def test_fake_local_import_publishes_candidate_and_requires_review_for_findings(
|
||||
|
||||
monkeypatch.setattr(importer, "_publish_candidate", fake_no_overwrite_publish)
|
||||
candidate_directory = state_root / "mcp-hub-release-candidates" / revision
|
||||
_candidate_directory_link_count(monkeypatch, candidate_directory, directory_link_count)
|
||||
if rerun_during_load:
|
||||
with pytest.raises(importer.CandidateRejected, match="run changed"):
|
||||
importer.prepare_candidate(
|
||||
@@ -970,6 +988,39 @@ def test_fake_local_import_publishes_candidate_and_requires_review_for_findings(
|
||||
assert (candidate_directory / "import-receipt.json").read_bytes() == receipt_before_review
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix", reason="POSIX directory inode metadata")
|
||||
@pytest.mark.parametrize("directory_issue", ["zero_links", "public_mode", "foreign_owner", "symlink", "file"])
|
||||
def test_cached_candidate_rejects_unsafe_directory(
|
||||
modules, monkeypatch, tmp_path, directory_issue,
|
||||
):
|
||||
importer, controller = modules.importer, modules.controller
|
||||
revision = "a" * 40
|
||||
state_root = tmp_path / "state"
|
||||
directory = state_root / "mcp-hub-release-candidates" / revision
|
||||
directory.mkdir(mode=0o700, parents=True)
|
||||
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
|
||||
monkeypatch.setattr(controller, "_root_owned", lambda *_args: directory_issue != "foreign_owner")
|
||||
_candidate_directory_link_count(monkeypatch, directory, 0 if directory_issue == "zero_links" else 1)
|
||||
if directory_issue == "public_mode":
|
||||
directory.chmod(0o755)
|
||||
elif directory_issue == "symlink":
|
||||
target = directory.with_name("owned-target")
|
||||
directory.rename(target)
|
||||
directory.symlink_to(target, target_is_directory=True)
|
||||
elif directory_issue == "file":
|
||||
directory.rmdir()
|
||||
_private_file(directory, b"{}")
|
||||
|
||||
def reject_late_read(*_args):
|
||||
raise AssertionError("Directory rejection must precede candidate evidence reads")
|
||||
|
||||
monkeypatch.setattr(controller, "_mcp_hub_candidate_directory", reject_late_read)
|
||||
with pytest.raises(importer.CandidateRejected, match="not.*private"):
|
||||
importer._cached_candidate_result(
|
||||
revision, "b" * 40, {}, "", lambda *_args: {}, lambda *_args: None,
|
||||
)
|
||||
|
||||
|
||||
def test_candidate_artifact_is_exact_bounded_and_unexpired(modules):
|
||||
importer = modules.importer
|
||||
revision = "c" * 40
|
||||
|
||||
@@ -6,8 +6,9 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml"
|
||||
|
||||
|
||||
def test_managed_validation_uses_internal_gitea_checkout_path() -> None:
|
||||
def test_managed_validation_uses_private_https_gitea_checkout_path() -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
assert "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" in workflow
|
||||
assert "github-server-url: http://gitea-runner-internal:3000" in workflow
|
||||
assert "github-server-url: https://gitea.example.com" in workflow
|
||||
assert "github-server-url: http://gitea-runner-internal:3000" not in workflow
|
||||
@@ -56,7 +56,7 @@ def test_backend_budget_is_bounded_without_weakening_validation():
|
||||
line.strip() for line in backend_body.splitlines() if "-m pytest " in line
|
||||
]
|
||||
assert pytest_commands == [
|
||||
'"${py}" -m pytest backend/tests -q --durations=20'
|
||||
'"${py}" -m pytest backend/tests -q --durations=20 --maxfail=1'
|
||||
]
|
||||
assert "continue-on-error" not in backend_body
|
||||
assert "--ignore" not in backend_body
|
||||
|
||||
@@ -44,6 +44,89 @@ def controller_module(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
def _host_gate_sample(module, monkeypatch):
|
||||
now = [100.0]
|
||||
host = {"loadRatio": 0.62, "availableMemoryBytes": 19 * 1024**3,
|
||||
"cacheFreeBytes": 140 * 1024**3, "swapCritical": False,
|
||||
"swapSampleAt": now[0]}
|
||||
capacity = {"host": host, "decision": {"pressure": "headroom"}}
|
||||
controller = module.RunnerAdmissionController(clock=lambda: now[0])
|
||||
monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, capacity))
|
||||
return controller, now, host, capacity
|
||||
|
||||
|
||||
def test_safe_live_metric_fluctuations_do_not_restart_stable_headroom(controller_module, monkeypatch):
|
||||
module = controller_module
|
||||
controller, now, host, _ = _host_gate_sample(module, monkeypatch)
|
||||
state = {}
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable"
|
||||
for at, load, memory, cache in [(130, 0.58, 18.5, 139.4), (160, 0.53, 18.2, 138.9)]:
|
||||
now[0] = at
|
||||
host.update(loadRatio=load, availableMemoryBytes=int(memory * 1024**3),
|
||||
cacheFreeBytes=int(cache * 1024**3), swapSampleAt=at)
|
||||
reason, _ = controller._fresh_mutation_gate("backend", ["backend"], state)
|
||||
assert reason is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"])
|
||||
def test_unsafe_sample_restarts_the_entire_stability_window(controller_module, monkeypatch, change):
|
||||
module = controller_module
|
||||
controller, now, host, capacity = _host_gate_sample(module, monkeypatch)
|
||||
state = {}
|
||||
controller._fresh_mutation_gate("backend", ["backend"], state)
|
||||
now[0] = 130
|
||||
host["swapSampleAt"] = 130
|
||||
original = copy.deepcopy(capacity)
|
||||
if change == "load":
|
||||
host["loadRatio"] = 0.71
|
||||
elif change == "memory":
|
||||
host["availableMemoryBytes"] = 14 * 1024**3
|
||||
elif change == "cache":
|
||||
host["cacheFreeBytes"] = 29 * 1024**3
|
||||
elif change == "swap":
|
||||
host["swapCritical"] = True
|
||||
elif change == "stale":
|
||||
host["swapSampleAt"] = 1
|
||||
elif change == "future":
|
||||
host["swapSampleAt"] = 131
|
||||
elif change == "pressure":
|
||||
capacity["decision"]["pressure"] = "pressure"
|
||||
else:
|
||||
monkeypatch.setattr(controller, "_host_evidence", lambda: ("blocked", "host_guard_paused", capacity))
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is not None
|
||||
capacity.clear()
|
||||
capacity.update(original)
|
||||
host = capacity["host"]
|
||||
now[0] = 160
|
||||
host["swapSampleAt"] = 160
|
||||
monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, capacity))
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable"
|
||||
now[0] = 220
|
||||
host["swapSampleAt"] = 220
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is None
|
||||
|
||||
|
||||
def test_missing_observation_interval_cannot_count_as_stable_headroom(controller_module, monkeypatch):
|
||||
module = controller_module
|
||||
controller, now, host, _ = _host_gate_sample(module, monkeypatch)
|
||||
state = {}
|
||||
controller._fresh_mutation_gate("backend", ["backend"], state)
|
||||
now[0] = 200
|
||||
host["swapSampleAt"] = 200
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("profile,claims", [("frontend", ["frontend"]), ("backend", ["backend", "checks"])])
|
||||
def test_profile_or_reservation_change_requires_new_stability_window(controller_module, monkeypatch, profile, claims):
|
||||
module = controller_module
|
||||
controller, now, host, _ = _host_gate_sample(module, monkeypatch)
|
||||
state = {}
|
||||
controller._fresh_mutation_gate("backend", ["backend"], state)
|
||||
now[0] = 160
|
||||
host["swapSampleAt"] = 160
|
||||
assert controller._fresh_mutation_gate(profile, claims, state)[0] == "runner_admission_host_headroom_not_stable"
|
||||
|
||||
|
||||
def _public_observation(module, now: float):
|
||||
rows = {}
|
||||
containers = {}
|
||||
@@ -257,6 +340,173 @@ def _lease_fixture(module, *, enabled=("backend", "frontend"), busy=(), queue=()
|
||||
return controller, state, observation, inventory_rows, containers
|
||||
|
||||
|
||||
def _retired_security_inventory(module):
|
||||
rows = [
|
||||
{"name": profile.display_name, "id": 101 + index, "status": "offline",
|
||||
"busy": False, "disabled": True,
|
||||
"labels": [profile.runner_label, *profile.legacy_labels]}
|
||||
for index, profile in enumerate(module.RUNNER_PROFILES.values())
|
||||
]
|
||||
rows.append({
|
||||
"name": "itworx-security-runner", "id": 9, "status": "offline",
|
||||
"busy": False, "disabled": True,
|
||||
"labels": ["linux-validation", "docker-validation", "ubuntu-latest",
|
||||
"ubuntu-24.04", "ubuntu-22.04", "mcp-validation", "unraid"],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def test_exact_retired_v1_security_labels_do_not_block_new_profile_inventory(controller_module):
|
||||
module = controller_module
|
||||
rows, ids = module.RunnerAdmissionController._normalized_runner_rows(
|
||||
_retired_security_inventory(module)
|
||||
)
|
||||
claims, profile_ids, _ = module.RunnerAdmissionController._active_claims(rows, ids, {})
|
||||
assert claims == []
|
||||
assert set(profile_ids) == set(module.PROFILE_IDS)
|
||||
assert rows["itworx-security-runner"]["disabled"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["enabled", "busy", "missing", "foreign", "replaced_id"])
|
||||
def test_retired_v1_security_variant_refuses_live_or_changed_registration(controller_module, change):
|
||||
module = controller_module
|
||||
inventory = _retired_security_inventory(module)
|
||||
old = inventory[-1]
|
||||
if change == "enabled":
|
||||
old["disabled"] = False
|
||||
elif change == "busy":
|
||||
old["busy"] = True
|
||||
elif change == "missing":
|
||||
old["labels"].remove("mcp-validation")
|
||||
elif change == "replaced_id":
|
||||
old["id"] = 99
|
||||
else:
|
||||
old["labels"].append("foreign-capability")
|
||||
with pytest.raises(module.RunnerAdmissionError, match="legacy_runner_labels_invalid"):
|
||||
module.RunnerAdmissionController._normalized_runner_rows(inventory)
|
||||
|
||||
|
||||
def test_retired_v1_security_active_job_still_refuses_admission(controller_module):
|
||||
module = controller_module
|
||||
rows, ids = module.RunnerAdmissionController._normalized_runner_rows(
|
||||
_retired_security_inventory(module)
|
||||
)
|
||||
with pytest.raises(module.RunnerAdmissionError, match="legacy_claim_unmapped"):
|
||||
module.RunnerAdmissionController._active_claims(rows, ids, {9: [{"id": "foreign-active"}]})
|
||||
|
||||
|
||||
def test_original_security_label_contract_is_preserved(controller_module):
|
||||
module = controller_module
|
||||
inventory = _retired_security_inventory(module)
|
||||
inventory[-1]["labels"] = sorted(module.LEGACY_REQUIRED_LABELS["itworx-security-runner"])
|
||||
rows, _ = module.RunnerAdmissionController._normalized_runner_rows(inventory)
|
||||
assert rows["itworx-security-runner"]["labels"] == set(inventory[-1]["labels"])
|
||||
|
||||
|
||||
def _migrated_tower_inventory(module):
|
||||
return [*_retired_security_inventory(module),
|
||||
{"name": "itworx-unraid-deploy", "id": 13, "status": "online", "busy": False,
|
||||
"disabled": False, "labels": ["unraid-deploy"]},
|
||||
{"name": "itworx-windows11-native", "id": 14, "status": "online", "busy": False,
|
||||
"disabled": False, "labels": ["windows-native"]}]
|
||||
|
||||
|
||||
def test_migrated_tower_preserves_idle_native_registrations_without_family_claims(controller_module):
|
||||
module = controller_module
|
||||
rows, ids = module.RunnerAdmissionController._normalized_runner_rows(_migrated_tower_inventory(module))
|
||||
claims, profiles, assigned = module.RunnerAdmissionController._active_claims(rows, ids, {})
|
||||
assert claims == []
|
||||
assert set(profiles) == set(module.PROFILE_IDS)
|
||||
assert 13 not in assigned and 14 not in assigned
|
||||
|
||||
|
||||
@pytest.mark.parametrize("index", [-2, -1])
|
||||
@pytest.mark.parametrize("change", ["id", "labels", "name"])
|
||||
def test_preserved_native_identity_or_capability_drift_refuses(controller_module, index, change):
|
||||
module = controller_module
|
||||
inventory = _migrated_tower_inventory(module)
|
||||
if change == "id":
|
||||
inventory[index]["id"] = 500
|
||||
elif change == "labels":
|
||||
inventory[index]["labels"].append("linux-validation")
|
||||
else:
|
||||
inventory[index]["name"] = "foreign-native"
|
||||
with pytest.raises(module.RunnerAdmissionError, match="preserved_native_identity_invalid"):
|
||||
module.RunnerAdmissionController._normalized_runner_rows(inventory)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("index", [-2, -1])
|
||||
@pytest.mark.parametrize("busy", [True, False])
|
||||
def test_active_preserved_native_blocks_family_admission_until_it_drains(controller_module, index, busy):
|
||||
module = controller_module
|
||||
inventory = _migrated_tower_inventory(module)
|
||||
row = inventory[index]
|
||||
row["busy"] = busy
|
||||
rows, ids = module.RunnerAdmissionController._normalized_runner_rows(inventory)
|
||||
active = {} if busy else {row["id"]: [{"id": "native-active"}]}
|
||||
with pytest.raises(module.RunnerAdmissionError, match="preserved_native_active"):
|
||||
module.RunnerAdmissionController._active_claims(rows, ids, active)
|
||||
|
||||
|
||||
def test_foreign_enabled_registration_still_refuses_with_preserved_natives(controller_module):
|
||||
module = controller_module
|
||||
inventory = _migrated_tower_inventory(module)
|
||||
inventory.append({"name": "foreign", "id": 500, "status": "online", "busy": False,
|
||||
"disabled": False, "labels": ["linux-validation"]})
|
||||
rows, ids = module.RunnerAdmissionController._normalized_runner_rows(inventory)
|
||||
with pytest.raises(module.RunnerAdmissionError, match="enabled_registration_unmapped"):
|
||||
module.RunnerAdmissionController._active_claims(rows, ids, {})
|
||||
|
||||
|
||||
def test_absent_retired_container_is_confirmed_by_successful_exact_inventory(controller_module, monkeypatch):
|
||||
module = controller_module
|
||||
calls = []
|
||||
def run(args, **_kwargs):
|
||||
calls.append(args)
|
||||
if args[1] == "inspect":
|
||||
return SimpleNamespace(returncode=1, stdout="[]\n", stderr="opaque failure")
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
monkeypatch.setattr(module.subprocess, "run", run)
|
||||
assert module.RunnerAdmissionController()._container("itworx-security-runner") is None
|
||||
assert all(args[1] != "inspect" for args in calls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stdout,exit_code", [
|
||||
("", 1), ("foreign", 0), ("a" * 64 + " wrong-name\n", 0),
|
||||
(("a" * 64 + " itworx-security-runner\n") * 2, 0),
|
||||
])
|
||||
def test_retired_container_inventory_failure_or_ambiguity_refuses(controller_module, monkeypatch, stdout, exit_code):
|
||||
module = controller_module
|
||||
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs:
|
||||
SimpleNamespace(returncode=exit_code, stdout=stdout, stderr="opaque failure"))
|
||||
with pytest.raises(module.RunnerAdmissionError):
|
||||
module.RunnerAdmissionController()._container("itworx-security-runner")
|
||||
|
||||
|
||||
def test_present_retired_container_inspect_failure_is_not_absence(controller_module, monkeypatch):
|
||||
module = controller_module
|
||||
def run(args, **_kwargs):
|
||||
if args[1] == "inspect":
|
||||
return SimpleNamespace(returncode=1, stdout="[]\n", stderr="opaque failure")
|
||||
return SimpleNamespace(returncode=0, stdout="a" * 64 + " itworx-security-runner\n", stderr="")
|
||||
monkeypatch.setattr(module.subprocess, "run", run)
|
||||
with pytest.raises(module.RunnerAdmissionError, match="container_inspect_failed"):
|
||||
module.RunnerAdmissionController()._container("itworx-security-runner")
|
||||
|
||||
|
||||
def test_present_retired_container_replacement_refuses(controller_module, monkeypatch):
|
||||
module = controller_module
|
||||
def run(args, **_kwargs):
|
||||
if args[1] == "inspect":
|
||||
row = {"Id": "b" * 64, "Name": "/itworx-security-runner", "State": {"Running": False},
|
||||
"Config": {"Labels": {"com.docker.compose.project": module.COMPOSE_PROJECT}}}
|
||||
return SimpleNamespace(returncode=0, stdout=json.dumps([row]), stderr="")
|
||||
return SimpleNamespace(returncode=0, stdout="a" * 64 + " itworx-security-runner\n", stderr="")
|
||||
monkeypatch.setattr(module.subprocess, "run", run)
|
||||
with pytest.raises(module.RunnerAdmissionError, match="container_identity_invalid"):
|
||||
module.RunnerAdmissionController()._container("itworx-security-runner")
|
||||
|
||||
|
||||
def _raw_job(module, job, *, runner_id=None, active=False):
|
||||
row = {
|
||||
"id": job["id"],
|
||||
@@ -514,7 +764,7 @@ def test_runner_inventory_maps_exact_gitea_display_names_to_fixed_containers(
|
||||
"disabled": True,
|
||||
"labels": [profile.runner_label, *profile.legacy_labels],
|
||||
}
|
||||
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=10)
|
||||
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=100)
|
||||
]
|
||||
|
||||
by_name, by_id = module.RunnerAdmissionController._normalized_runner_rows(rows)
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Exercise the real workflow's receipt publisher without Docker or network."""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = ROOT / ".gitea/workflows/runner-build-acceptance.yml"
|
||||
REVISION = "a" * 40
|
||||
TREE = "b" * 40
|
||||
MARKER = "ITWORX_CI_JOB_DOCKER_PROBE_RESULT="
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workflow():
|
||||
return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_only_manual_fixed_build_profile_and_pinned_actions(workflow):
|
||||
# PyYAML's YAML 1.1 parser treats the Actions `on` key as boolean true.
|
||||
events = workflow.get("on", workflow.get(True))
|
||||
assert set(events) == {"workflow_dispatch"}
|
||||
assert set(events["workflow_dispatch"]["inputs"]) == {"source_sha"}
|
||||
assert workflow["permissions"] == {"contents": "read"}
|
||||
assert workflow["concurrency"]["cancel-in-progress"] is False
|
||||
assert set(workflow["jobs"]) == {"build-runner-acceptance"}
|
||||
job = workflow["jobs"]["build-runner-acceptance"]
|
||||
assert job["runs-on"] == "itworx-ci-build"
|
||||
assert "services" not in job and "container" not in job
|
||||
checkout, probe, upload = job["steps"]
|
||||
assert checkout["uses"] == "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
|
||||
assert checkout["with"]["persist-credentials"] is False
|
||||
assert checkout["with"]["github-server-url"] == "https://gitea.example.com"
|
||||
assert checkout["with"]["ref"] == "${{ gitea.sha }}"
|
||||
assert "secrets." not in WORKFLOW.read_text(encoding="utf-8")
|
||||
assert probe["shell"] == "bash"
|
||||
assert upload["uses"] == "ChristopherHX/gitea-upload-artifact@62ac910c5d3dfa85c7cb2df15afe2e342b2407c2"
|
||||
assert upload.get("if") is None # Actions default success; never always().
|
||||
assert upload["with"]["path"] == "${{ runner.temp }}/runner-build-acceptance/receipt.json"
|
||||
assert upload["with"]["if-no-files-found"] == "error"
|
||||
assert upload["with"]["overwrite"] is False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def publisher(workflow, monkeypatch, tmp_path):
|
||||
script = workflow["jobs"]["build-runner-acceptance"]["steps"][1]["run"]
|
||||
script = script.split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0]
|
||||
environment = {
|
||||
"EXPECTED_SOURCE_SHA": REVISION,
|
||||
"DISPATCH_SHA": REVISION,
|
||||
"DISPATCH_REF": "refs/heads/main",
|
||||
"DISPATCH_EVENT": "workflow_dispatch",
|
||||
"RUN_REPOSITORY": "Jens/Unraid-AppOps-Gateway",
|
||||
"RUN_ID": "22840",
|
||||
"RUN_NUMBER": "1261",
|
||||
"RUN_ATTEMPT": "1",
|
||||
"RUN_JOB": "build-runner-acceptance",
|
||||
"RUNNER_TEMP": str(tmp_path),
|
||||
}
|
||||
probe = {
|
||||
"schema": "itworx.ci.job-docker-probe/v1", "status": "passed",
|
||||
"sourceRevision": REVISION, "sourceTree": TREE,
|
||||
"engineVersion": "29.8.2", "engineRootless": True,
|
||||
"engineStorageDriver": "overlay2", "engineCgroupDriver": "none",
|
||||
"engineSocket": "unix:///run/itworx-job-docker/docker.sock",
|
||||
"engineSocketUid": 1000, "composeVersion": "5.5.1",
|
||||
"buildRun": True, "composeHealthy": True, "composeDnsResolved": True,
|
||||
"archiveNormalized": True, "archiveLoaded": True, "holdSeconds": 30,
|
||||
"imageLabels": {
|
||||
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
|
||||
"org.opencontainers.image.revision": REVISION,
|
||||
"tech.itworx.source-tree": TREE, "tech.itworx.ci.profile": "build",
|
||||
},
|
||||
"imageId": "sha256:" + "c" * 64,
|
||||
"archiveSha256": "sha256:" + "d" * 64,
|
||||
}
|
||||
state = {"calls": [], "probe": probe, "exit": 0, "dirty": "", "tree": TREE}
|
||||
|
||||
def run(args, **kwargs):
|
||||
state["calls"].append(args)
|
||||
assert kwargs["stdin"] == subprocess.DEVNULL
|
||||
if args[0] == "git":
|
||||
assert kwargs["stderr"] == subprocess.DEVNULL
|
||||
command = args[1:]
|
||||
values = {
|
||||
("rev-parse", "HEAD"): REVISION,
|
||||
("cat-file", "-t", REVISION): "commit",
|
||||
("rev-parse", "HEAD^{tree}"): state["tree"],
|
||||
("cat-file", "-t", TREE): "tree",
|
||||
("rev-parse", REVISION + "^{tree}"): TREE,
|
||||
("status", "--porcelain", "--untracked-files=no"): state["dirty"],
|
||||
}
|
||||
return subprocess.CompletedProcess(args, 0, values[tuple(command)] + "\n")
|
||||
assert args == ["/usr/local/bin/itworx-ci-job-docker-probe", REVISION,
|
||||
TREE, "--hold-seconds", "30"]
|
||||
assert kwargs["stderr"] == subprocess.DEVNULL
|
||||
assert kwargs["timeout"] == 1000
|
||||
if state.get("timeout"):
|
||||
raise subprocess.TimeoutExpired(args, 1000, stderr="PRIVATE_TEST_SECRET")
|
||||
output = state.get("raw", MARKER + json.dumps(state["probe"]) + "\n")
|
||||
kwargs["stdout"].write(output.encode())
|
||||
return subprocess.CompletedProcess(args, state["exit"])
|
||||
|
||||
def execute():
|
||||
with monkeypatch.context() as patch:
|
||||
for key, value in environment.items():
|
||||
patch.setenv(key, value)
|
||||
patch.setattr(subprocess, "run", run)
|
||||
# Execute repository-owned workflow code; every subprocess is intercepted.
|
||||
previous_umask = os.umask(0o077)
|
||||
try:
|
||||
exec(compile(script, str(WORKFLOW), "exec"), {}) # noqa: S102
|
||||
finally:
|
||||
os.umask(previous_umask)
|
||||
|
||||
state["environment"] = environment
|
||||
state["execute"] = execute
|
||||
state["receipt"] = tmp_path / "runner-build-acceptance/receipt.json"
|
||||
return state
|
||||
|
||||
|
||||
def test_success_publishes_only_finite_post_cleanup_receipt(publisher):
|
||||
publisher["execute"]()
|
||||
receipt = json.loads(publisher["receipt"].read_text())
|
||||
assert receipt["cleanupVerified"] is True
|
||||
assert receipt["probeExitCode"] == 0
|
||||
assert receipt["probe"] == publisher["probe"]
|
||||
assert receipt["gitea"] == {
|
||||
"repository": "Jens/Unraid-AppOps-Gateway",
|
||||
"workflow": "runner-build-acceptance.yml", "jobKey": "build-runner-acceptance",
|
||||
"runId": 22840, "runNumber": 1261, "runAttempt": 1,
|
||||
"nativeCorrelationRequired": True,
|
||||
}
|
||||
|
||||
|
||||
def test_missing_attempt_is_honest_null_not_invented_one(publisher):
|
||||
publisher["environment"]["RUN_ATTEMPT"] = ""
|
||||
publisher["execute"]()
|
||||
run = json.loads(publisher["receipt"].read_text())["gitea"]
|
||||
assert run["runAttempt"] is None
|
||||
assert run["nativeCorrelationRequired"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_probe", [False, True])
|
||||
def test_publisher_fixture_restores_process_umask(publisher, failed_probe, monkeypatch):
|
||||
mask = [0o022]
|
||||
|
||||
def umask(value):
|
||||
previous, mask[0] = mask[0], value
|
||||
return previous
|
||||
|
||||
monkeypatch.setattr(os, "umask", umask)
|
||||
publisher["exit"] = 70 if failed_probe else 0
|
||||
if failed_probe:
|
||||
with pytest.raises(SystemExit):
|
||||
publisher["execute"]()
|
||||
else:
|
||||
publisher["execute"]()
|
||||
assert mask[0] == 0o022
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), [
|
||||
("EXPECTED_SOURCE_SHA", "$(echo PRIVATE_TEST_SECRET)"),
|
||||
("EXPECTED_SOURCE_SHA", "e" * 40),
|
||||
("DISPATCH_SHA", "e" * 40),
|
||||
("DISPATCH_REF", "refs/heads/other"),
|
||||
("DISPATCH_EVENT", "push"),
|
||||
("RUN_REPOSITORY", "Jens/Other"),
|
||||
("RUN_ID", ""), ("RUN_ID", "0"),
|
||||
("RUN_ATTEMPT", "garbage"),
|
||||
("RUN_JOB", "other"),
|
||||
])
|
||||
def test_invalid_identity_refuses_before_probe(publisher, key, value):
|
||||
publisher["environment"][key] = value
|
||||
with pytest.raises(SystemExit, match="no success receipt published"):
|
||||
publisher["execute"]()
|
||||
assert all(args[0] == "git" for args in publisher["calls"])
|
||||
assert not publisher["receipt"].exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), [("dirty", " M tracked.txt"), ("tree", "invalid")])
|
||||
def test_bad_checkout_tree_refuses_before_probe(publisher, key, value):
|
||||
publisher[key] = value
|
||||
with pytest.raises(SystemExit, match="no success receipt published"):
|
||||
publisher["execute"]()
|
||||
assert all(args[0] == "git" for args in publisher["calls"])
|
||||
assert not publisher["receipt"].exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["cleanup_failure", "hold_failure", "timeout", "secret_output"])
|
||||
def test_marker_does_not_authorize_success_before_exit_zero(publisher, mode):
|
||||
if mode in {"cleanup_failure", "hold_failure"}:
|
||||
publisher["exit"] = 70
|
||||
elif mode == "timeout":
|
||||
publisher["timeout"] = True
|
||||
else:
|
||||
publisher["raw"] = "PRIVATE_TEST_SECRET\n" + MARKER + json.dumps(publisher["probe"])
|
||||
with pytest.raises(SystemExit) as failure:
|
||||
publisher["execute"]()
|
||||
assert "PRIVATE_TEST_SECRET" not in str(failure.value)
|
||||
assert not publisher["receipt"].exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), [
|
||||
("engineRootless", False), ("engineRootless", 1),
|
||||
("archiveLoaded", False), ("composeHealthy", False),
|
||||
("sourceRevision", "e" * 40), ("sourceTree", "e" * 40),
|
||||
("imageId", "invalid"), ("unexpected", "PRIVATE_TEST_SECRET"),
|
||||
])
|
||||
def test_false_or_foreign_probe_receipt_is_never_published(publisher, key, value):
|
||||
publisher["probe"] = copy.deepcopy(publisher["probe"])
|
||||
publisher["probe"][key] = value
|
||||
with pytest.raises(SystemExit) as failure:
|
||||
publisher["execute"]()
|
||||
assert "PRIVATE_TEST_SECRET" not in str(failure.value)
|
||||
assert not publisher["receipt"].exists()
|
||||
@@ -4308,16 +4308,18 @@ def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specia
|
||||
specialize._check_legacy_queued_jobs()
|
||||
|
||||
|
||||
def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url():
|
||||
def test_image_workflow_builds_from_protected_appops_main_with_private_https_checkout_url():
|
||||
workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8")
|
||||
build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8")
|
||||
frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8")
|
||||
|
||||
assert "github-server-url: http://gitea-runner-internal:3000" in workflow
|
||||
assert "runs-on: linux-validation" in workflow
|
||||
assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output"' in workflow
|
||||
assert "runs-on: itworx-ci-build" not in workflow
|
||||
assert "--job-local" not in workflow
|
||||
assert "github-server-url: https://gitea.example.com" in workflow
|
||||
for name in ("managed-validation.yml", "runner-build-acceptance.yml", "ci-task-images.yml"):
|
||||
content = (ROOT / ".gitea" / "workflows" / name).read_text(encoding="utf-8")
|
||||
assert "gitea-runner-internal" not in content
|
||||
assert "runs-on: itworx-ci-build" in workflow
|
||||
assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output" --job-local' in workflow
|
||||
assert "runs-on: linux-validation" not in workflow
|
||||
assert "jens/unraid-appops-gateway" in build
|
||||
assert "Jens/Unraid-AppOps-Gateway" in build
|
||||
assert "corepack prepare pnpm@11.15.0 --activate" in frontend
|
||||
@@ -4375,6 +4377,9 @@ def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity(
|
||||
|
||||
def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True):
|
||||
rows, containers, dind, journal, model = _active_recovery_fixture(specialize)
|
||||
dind.update({"Id": "a" * 64, "Image": specialize._DIND_IMAGE_ID})
|
||||
for profile in specialize.RUNNER_PROFILES.values():
|
||||
containers[profile.container_name]["State"]["Pid"] = 0
|
||||
events = []
|
||||
|
||||
class Volume:
|
||||
@@ -4425,6 +4430,10 @@ def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True):
|
||||
)
|
||||
monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy"))
|
||||
monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True)
|
||||
monkeypatch.setattr(specialize, "_bind_active_dind_family",
|
||||
lambda *_args: events.append("owned-populated-family"))
|
||||
monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "")
|
||||
monkeypatch.setattr(specialize, "_run", lambda args, **_kwargs: events.append(("docker", args)) or "")
|
||||
monkeypatch.setattr(
|
||||
specialize,
|
||||
"_resource_proof",
|
||||
@@ -4433,7 +4442,8 @@ def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True):
|
||||
|
||||
def inspect(name):
|
||||
if name == specialize.DIND_NAME:
|
||||
dind["State"] = {"Running": True, "Health": {"Status": "healthy"}}
|
||||
dind["State"] = {"Running": True, "Pid": 77,
|
||||
"StartedAt": "2026-10-12T00:00:00Z", "Health": {"Status": "healthy"}}
|
||||
return dind
|
||||
return None
|
||||
|
||||
@@ -4461,11 +4471,270 @@ def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates(
|
||||
assert compose_bytes == b"captured-compose"
|
||||
assert args == ("up", "-d", "--no-deps", "runner-dind")
|
||||
assert kwargs["timeout"] <= 30
|
||||
assert events.count("fresh-host-gate") == 2
|
||||
assert events.count("fresh-host-gate") == 3
|
||||
assert events.index("fresh-host-gate") < events.index("healthy")
|
||||
|
||||
|
||||
def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind(
|
||||
def test_active_recovery_survives_disappearing_empty_family(specialize, monkeypatch):
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(
|
||||
specialize, monkeypatch, family_exists=False
|
||||
)
|
||||
monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {
|
||||
"exists": False,
|
||||
"safeToStartDind": False,
|
||||
"safeToStartDindReason": "resource_cgroup_family_missing",
|
||||
"rootSubtreeControllers": ["cpu", "memory", "pids"],
|
||||
})
|
||||
monkeypatch.setattr(specialize, "_bind_active_dind_family",
|
||||
lambda *_args: events.append("owned-populated-family"), raising=False)
|
||||
|
||||
result = specialize.ensure_active_fleet_ready()
|
||||
|
||||
assert result == {"ready": True, "reason": None}
|
||||
assert "cgroup-create" not in events
|
||||
assert events.index("healthy") < events.index("owned-populated-family")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["health", "inspect-after-binding", "binding", "images", "envelope", "guard"])
|
||||
def test_active_recovery_stops_only_its_new_empty_start_on_failed_acceptance(
|
||||
specialize, monkeypatch, failure,
|
||||
):
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
if failure == "health":
|
||||
def fail_health(*_args, **_kwargs):
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_dind_recovery_unverified")
|
||||
monkeypatch.setattr(specialize, "_wait_healthy", fail_health)
|
||||
elif failure == "inspect-after-binding":
|
||||
original_inspect = specialize._inspect
|
||||
count = [0]
|
||||
def fail_inspect(name):
|
||||
count[0] += 1
|
||||
if count[0] == 2:
|
||||
raise RuntimeError("sentinel-secret")
|
||||
return original_inspect(name)
|
||||
monkeypatch.setattr(specialize, "_inspect", fail_inspect)
|
||||
elif failure == "binding":
|
||||
def fail_binding(*_args):
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable")
|
||||
monkeypatch.setattr(specialize, "_bind_active_dind_family", fail_binding)
|
||||
elif failure == "images":
|
||||
monkeypatch.setattr(specialize, "verify_profile_task_images", lambda *_args: False)
|
||||
elif failure == "envelope":
|
||||
def fail_proof(*_args):
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_resource_envelope_unready")
|
||||
monkeypatch.setattr(specialize, "_resource_proof", fail_proof)
|
||||
else:
|
||||
count = [0]
|
||||
def fail_guard():
|
||||
count[0] += 1
|
||||
if count[0] == 3:
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_host_guard_paused")
|
||||
monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fail_guard)
|
||||
|
||||
result = specialize.ensure_active_fleet_ready()
|
||||
|
||||
assert result["ready"] is False
|
||||
assert "sentinel-secret" not in str(result)
|
||||
assert ("docker", ["docker", "stop", "--time", "30", "a" * 64]) in events
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drift", ["id", "pid", "started", "job", "profile", "profile-pid", "registration", "busy", "enabled", "volume"])
|
||||
def test_active_recovery_rollback_refuses_changed_or_foreign_state(specialize, monkeypatch, drift):
|
||||
events, rows, containers, dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
inspected = specialize._inspect(specialize.DIND_NAME)
|
||||
binding = specialize._active_dind_binding(inspected, {})
|
||||
marker, journal, model, compose = specialize._active_release_bundle()
|
||||
identity = specialize._active_recovery_identity(marker, journal, compose)
|
||||
if drift == "id":
|
||||
dind["Id"] = "b" * 64
|
||||
elif drift in {"pid", "started"}:
|
||||
key, value = ("Pid", 88) if drift == "pid" else ("StartedAt", "2026-10-12T00:01:00Z")
|
||||
def changed_inspect(_name):
|
||||
changed = copy.deepcopy(dind)
|
||||
changed["State"][key] = value
|
||||
return changed
|
||||
monkeypatch.setattr(specialize, "_inspect", changed_inspect)
|
||||
elif drift == "job":
|
||||
monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
|
||||
elif drift == "profile":
|
||||
containers[specialize.RUNNER_PROFILES["backend"].container_name]["State"]["Running"] = True
|
||||
elif drift == "profile-pid":
|
||||
containers[specialize.RUNNER_PROFILES["backend"].container_name]["State"]["Pid"] = 88
|
||||
elif drift == "registration":
|
||||
rows[0]["id"] += 1000
|
||||
elif drift == "busy":
|
||||
rows[0]["busy"] = True
|
||||
elif drift == "enabled":
|
||||
rows[0]["disabled"] = False
|
||||
else:
|
||||
def changed_bundle():
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_active_release_untrusted")
|
||||
monkeypatch.setattr(specialize, "_active_release_bundle", changed_bundle)
|
||||
try:
|
||||
specialize._stop_new_active_dind(binding, model, identity)
|
||||
except specialize.RunnerFleetSpecializeError:
|
||||
pass
|
||||
assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["missing-original", "replaced", "same-cid-restarted", "uninspectable", "inspect-once", "nonempty"])
|
||||
def test_active_recovery_failed_start_never_infers_cleanup_ownership(specialize, monkeypatch, case):
|
||||
events, _rows, containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
original_inspect = specialize._inspect
|
||||
if case == "missing-original":
|
||||
containers.pop(specialize.DIND_NAME)
|
||||
elif case == "replaced":
|
||||
def replaced(name):
|
||||
item = copy.deepcopy(original_inspect(name))
|
||||
item["Id"] = "b" * 64
|
||||
return item
|
||||
monkeypatch.setattr(specialize, "_inspect", replaced)
|
||||
elif case == "uninspectable":
|
||||
def fail_inspect(_name):
|
||||
raise RuntimeError("sentinel-secret")
|
||||
monkeypatch.setattr(specialize, "_inspect", fail_inspect)
|
||||
elif case == "inspect-once":
|
||||
count = [0]
|
||||
def fail_once(name):
|
||||
count[0] += 1
|
||||
if count[0] == 1:
|
||||
raise RuntimeError("sentinel-secret")
|
||||
return original_inspect(name)
|
||||
monkeypatch.setattr(specialize, "_inspect", fail_once)
|
||||
elif case == "same-cid-restarted":
|
||||
def restarted(name):
|
||||
item = copy.deepcopy(original_inspect(name))
|
||||
item["State"].update({"Pid": 88, "StartedAt": "2026-10-12T00:01:00Z"})
|
||||
return item
|
||||
monkeypatch.setattr(specialize, "_inspect", restarted)
|
||||
else:
|
||||
monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
|
||||
def fail_start(*_args, **_kwargs):
|
||||
raise RuntimeError("sentinel-secret")
|
||||
if case != "inspect-once":
|
||||
monkeypatch.setattr(specialize, "_run_compose", fail_start)
|
||||
|
||||
result = specialize.ensure_active_fleet_ready()
|
||||
|
||||
assert result["ready"] is False
|
||||
assert "sentinel-secret" not in str(result)
|
||||
assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
|
||||
|
||||
|
||||
def test_active_binder_uses_exact_owned_population_without_migration_journal_writes(specialize, monkeypatch):
|
||||
binder = specialize._bind_active_dind_family
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
# Exercise the real binder rather than the high-level lifecycle stub.
|
||||
item = specialize._inspect(specialize.DIND_NAME)
|
||||
calls = []
|
||||
monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\n77\n78\n")
|
||||
monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", lambda **kwargs: calls.append(kwargs))
|
||||
monkeypatch.setattr(specialize, "_write_journal", lambda *_args: pytest.fail("migration evidence must remain intact"))
|
||||
marker, journal, model, compose = specialize._active_release_bundle()
|
||||
binder(item, model, specialize._active_recovery_identity(marker, journal, compose))
|
||||
|
||||
assert calls == [{"expected_container_id": "a" * 64, "expected_host_pid": 77,
|
||||
"expected_process_ids": [77, 78], "allow_owned_partial": True}]
|
||||
assert "cgroup-create" not in events
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["job", "job-after", "duplicate", "zero", "missing-init", "unknown-pid", "foreign-family", "replaced", "replaced-before", "restarted-before"])
|
||||
def test_active_binder_rejects_untrusted_or_nonempty_population(specialize, monkeypatch, case):
|
||||
binder = specialize._bind_active_dind_family
|
||||
_events, _rows, _containers, dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
item = copy.deepcopy(specialize._inspect(specialize.DIND_NAME))
|
||||
if case == "replaced-before":
|
||||
dind["Id"] = "b" * 64
|
||||
elif case == "restarted-before":
|
||||
original_inspect = specialize._inspect
|
||||
def restarted(name):
|
||||
changed = copy.deepcopy(original_inspect(name))
|
||||
changed["State"].update({"Pid": 88, "StartedAt": "2026-10-12T00:01:00Z"})
|
||||
return changed
|
||||
monkeypatch.setattr(specialize, "_inspect", restarted)
|
||||
text = {"duplicate": "PID\n77\n77\n", "zero": "PID\n77\n0\n",
|
||||
"missing-init": "PID\n78\n", "unknown-pid": "PID\nsentinel-secret\n"}.get(case, "PID\n77\n78\n")
|
||||
monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: text)
|
||||
if case == "job":
|
||||
monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
|
||||
calls = []
|
||||
def bind(**kwargs):
|
||||
calls.append(kwargs)
|
||||
if case == "foreign-family":
|
||||
raise RuntimeError("sentinel-secret")
|
||||
if case == "replaced":
|
||||
dind["Id"] = "b" * 64
|
||||
elif case == "job-after":
|
||||
monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
|
||||
monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", bind)
|
||||
marker, journal, model, compose = specialize._active_release_bundle()
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError) as error:
|
||||
binder(item, model, specialize._active_recovery_identity(marker, journal, compose))
|
||||
assert "sentinel-secret" not in str(error.value)
|
||||
if case not in {"foreign-family", "replaced", "job-after"}:
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_active_recovery_can_finish_existing_empty_bounded_start_without_stopping_it(specialize, monkeypatch):
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
specialize._inspect(specialize.DIND_NAME)
|
||||
ready = iter([{"ready": False}, {"ready": True}])
|
||||
monkeypatch.setattr(specialize, "verified_topology", lambda: next(ready))
|
||||
assert specialize.ensure_active_fleet_ready() == {"ready": True, "reason": None}
|
||||
assert "owned-populated-family" in events
|
||||
assert not any(isinstance(item, tuple) and item[0] in {"compose", "docker"} for item in events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drift", ["sourceScriptSha256", "imageSourceRevision", "imageSourceTree", "imageManifestSha256", "compose", "registration"])
|
||||
def test_active_binder_and_cleanup_refuse_valid_same_model_release_drift(specialize, monkeypatch, drift):
|
||||
binder = specialize._bind_active_dind_family
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
item = specialize._inspect(specialize.DIND_NAME)
|
||||
binding = specialize._active_dind_binding(item, {})
|
||||
marker, journal, model, compose = specialize._active_release_bundle()
|
||||
identity = specialize._active_recovery_identity(marker, journal, compose)
|
||||
changed_marker, changed_journal, changed_compose = copy.deepcopy(marker), copy.deepcopy(journal), compose
|
||||
if drift == "compose":
|
||||
changed_compose += b" "
|
||||
elif drift == "registration":
|
||||
changed_journal["newRegistrationIds"]["backend"] += 1000
|
||||
else:
|
||||
changed_marker[drift] = "f" * (40 if drift in {"imageSourceRevision", "imageSourceTree"} else 64)
|
||||
monkeypatch.setattr(specialize, "_active_release_bundle", lambda: (changed_marker, changed_journal, model, changed_compose))
|
||||
monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\n77\n78\n")
|
||||
monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", lambda **_kwargs: pytest.fail("changed source cannot bind"))
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError, match="active_release_changed"):
|
||||
binder(item, model, identity)
|
||||
monkeypatch.setattr(specialize, "_run", lambda args, **_kwargs: events.append(("docker", args)) or "")
|
||||
with pytest.raises(specialize.RunnerFleetSpecializeError, match="active_release_changed"):
|
||||
specialize._stop_new_active_dind(binding, model, identity)
|
||||
assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["binding", "proof"])
|
||||
def test_active_recovery_preserves_preexisting_running_daemon_on_failure(specialize, monkeypatch, failure):
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
specialize._inspect(specialize.DIND_NAME)
|
||||
monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False})
|
||||
def fail_bind(*_args):
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable")
|
||||
if failure == "binding":
|
||||
monkeypatch.setattr(specialize, "_bind_active_dind_family", fail_bind)
|
||||
assert specialize.ensure_active_fleet_ready()["ready"] is False
|
||||
assert not any(isinstance(item, tuple) and item[0] in {"compose", "docker"} for item in events)
|
||||
|
||||
|
||||
def test_active_recovery_does_not_stop_new_daemon_with_work_appearing_after_binding(specialize, monkeypatch):
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch)
|
||||
def failed_binding(*_args):
|
||||
monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job")
|
||||
raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_dind_recovery_unverified")
|
||||
monkeypatch.setattr(specialize, "_bind_active_dind_family", failed_binding)
|
||||
assert specialize.ensure_active_fleet_ready()["ready"] is False
|
||||
assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events)
|
||||
|
||||
|
||||
def test_active_recovery_makes_no_cgroup_or_start_write_if_host_guard_pauses_before_dind(
|
||||
specialize, monkeypatch
|
||||
):
|
||||
events, _rows, _containers, _dind = _stub_active_recovery(
|
||||
@@ -4495,9 +4764,8 @@ def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_din
|
||||
"ready": False,
|
||||
"reason": "runner_fleet_specialization_host_guard_paused",
|
||||
}
|
||||
assert rollback == [
|
||||
{"created": True, "initial_controllers": ["cpu", "memory", "pids"]}
|
||||
]
|
||||
assert rollback == []
|
||||
assert "cgroup-create" not in events
|
||||
assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events)
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
PROFILES = ("checks", "backend", "frontend", "security", "browser", "build")
|
||||
@@ -30,6 +30,39 @@ BASE_LABELS = {
|
||||
}
|
||||
|
||||
|
||||
def test_post_activation_producer_uses_fixed_job_local_build_route_and_preserves_artifact_contract():
|
||||
workflow = yaml.safe_load((ROOT / ".gitea/workflows/ci-task-images.yml").read_text(encoding="utf-8"))
|
||||
events = workflow.get("on", workflow.get(True))
|
||||
assert events == {"push": {"branches": ["main"]}}
|
||||
assert workflow["permissions"] == {"contents": "read"}
|
||||
assert workflow["concurrency"]["cancel-in-progress"] is False
|
||||
assert set(workflow["jobs"]) == {"build"}
|
||||
job = workflow["jobs"]["build"]
|
||||
assert job["runs-on"] == "itworx-ci-build"
|
||||
assert job["timeout-minutes"] == 90
|
||||
assert "services" not in job and "container" not in job
|
||||
checkout, build, upload = job["steps"]
|
||||
assert checkout["uses"] == "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
|
||||
assert checkout["with"] == {
|
||||
"fetch-depth": 1, "persist-credentials": False,
|
||||
"github-server-url": "https://gitea.example.com",
|
||||
}
|
||||
assert build["run"].strip() == (
|
||||
'set -euo pipefail\n'
|
||||
'output="${RUNNER_TEMP}/ci-task-images-${GITHUB_SHA}"\n'
|
||||
'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output" --job-local'
|
||||
)
|
||||
assert "env" not in build
|
||||
assert upload["uses"] == "ChristopherHX/gitea-upload-artifact@62ac910c5d3dfa85c7cb2df15afe2e342b2407c2"
|
||||
assert upload.get("if") is None
|
||||
assert upload["with"] == {
|
||||
"name": "ci-task-images-${{ github.sha }}",
|
||||
"path": "${{ runner.temp }}/ci-task-images-${{ github.sha }}/*",
|
||||
"if-no-files-found": "error", "retention-days": 90,
|
||||
"compression-level": 6, "overwrite": False,
|
||||
}
|
||||
|
||||
|
||||
def _run_producer(tmp_path: Path, mutation: str = ""):
|
||||
repo = tmp_path / "repository"
|
||||
repo.mkdir()
|
||||
@@ -121,6 +154,7 @@ def _run_producer(tmp_path: Path, mutation: str = ""):
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
return result, output, revision, tree
|
||||
|
||||
|
||||
Reference in new issue
Block a user