Publish curated Unraid-AppOps-Gateway source from 1a5159c66078983299a1c2acb8837e4b5a8a607a
This commit is contained in:
Public Source Publisher committed
1 parent
75269a32eb
commit
5a3941a286
8 files changed
+443
-23
No files matched your search
@@ -1,11 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -15,7 +18,7 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py"
|
||||
|
||||
|
||||
def _module():
|
||||
def _module(*, real_lock=False):
|
||||
broker_dir = str(ENVELOPE_PATH.parent)
|
||||
if broker_dir not in sys.path:
|
||||
sys.path.insert(0, broker_dir)
|
||||
@@ -24,9 +27,281 @@ def _module():
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
if not real_lock:
|
||||
# Existing Docker fixtures exercise identity/cleanup independently of
|
||||
# the protected volume. Native lifecycle tests below retain the lock.
|
||||
@contextmanager
|
||||
def fixture_lock(**_kwargs):
|
||||
yield lambda: None
|
||||
module._host_probe_lifecycle_lock = fixture_lock
|
||||
return module
|
||||
|
||||
|
||||
class _ProbeLockStore:
|
||||
"""Two independently opened fixture views, with real descriptor/inode checks."""
|
||||
|
||||
def __init__(self, logical: Path, host: Path | None = None):
|
||||
self.paths = (logical, host or logical)
|
||||
self.logical = SimpleNamespace(fd=os.open(logical, os.O_RDONLY | os.O_DIRECTORY))
|
||||
self.host = SimpleNamespace(fd=os.open(host or logical, os.O_RDONLY | os.O_DIRECTORY))
|
||||
|
||||
def assert_current(self):
|
||||
identities = []
|
||||
for view, path in zip((self.logical, self.host), self.paths, strict=True):
|
||||
opened, current = os.fstat(view.fd), path.stat(follow_symlinks=False)
|
||||
assert (opened.st_dev, opened.st_ino) == (current.st_dev, current.st_ino)
|
||||
identities.append((opened.st_dev, opened.st_ino))
|
||||
if identities[0] != identities[1]:
|
||||
raise RuntimeError("private-canary: mismatched volume views")
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
os.close(self.host.fd)
|
||||
os.close(self.logical.fd)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", [None, "remote", "options", "labels", "root", "malformed"])
|
||||
@pytest.mark.parametrize("native_cleanup", [False, True])
|
||||
def test_probe_lock_store_uses_only_attested_fixed_local_volume(monkeypatch, mutation, native_cleanup):
|
||||
envelope = _module(real_lock=True)
|
||||
import mcp_hub_release_store as store
|
||||
document = [{"Name": store.VOLUME_NAME, "Driver": "local", "Scope": "local",
|
||||
"Mountpoint": store.VOLUME_HOST_ROOT.as_posix(), "Options": {},
|
||||
"Labels": dict(store.EXPECTED_LABELS)}]
|
||||
root = store.DOCKER_ROOT.as_posix()
|
||||
if mutation == "remote":
|
||||
document[0]["Driver"] = "nfs"
|
||||
elif mutation == "options":
|
||||
document[0]["Options"] = {"device": "private-canary"}
|
||||
elif mutation == "labels":
|
||||
document[0]["Labels"] = {}
|
||||
elif mutation == "root":
|
||||
root = "/srv/example/private-canary"
|
||||
calls = []
|
||||
def run(args, **kwargs):
|
||||
calls.append(args)
|
||||
assert kwargs == {"timeout": 15}
|
||||
if args == ["docker", "volume", "inspect", store.VOLUME_NAME]:
|
||||
return "private-canary" if mutation == "malformed" else json.dumps(document)
|
||||
assert args == ["docker", "info", "--format", "{{.DockerRootDir}}"]
|
||||
return root
|
||||
sentinel = object()
|
||||
def verify(metadata, docker_root):
|
||||
store.validate_volume_metadata(metadata, docker_root)
|
||||
return sentinel
|
||||
monkeypatch.setattr(envelope, "_run", run)
|
||||
monkeypatch.setattr(envelope, "open_fixed_release_store", verify)
|
||||
class PinnedFixture:
|
||||
@staticmethod
|
||||
def open(path):
|
||||
assert path == store.VOLUME_HOST_ROOT
|
||||
return sentinel
|
||||
monkeypatch.setattr(envelope, "PinnedDirectory", PinnedFixture)
|
||||
if mutation is None:
|
||||
result = envelope._open_host_probe_lock_store(native_cleanup=native_cleanup)
|
||||
if native_cleanup:
|
||||
assert result.host is result.logical is sentinel
|
||||
else:
|
||||
assert result is sentinel
|
||||
assert len(calls) == 2
|
||||
else:
|
||||
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_store_untrusted$") as error:
|
||||
envelope._open_host_probe_lock_store(native_cleanup=native_cleanup)
|
||||
assert error.value.__cause__ is None
|
||||
|
||||
|
||||
def test_native_cleanup_store_rechecks_only_fixed_physical_volume_and_closes_once():
|
||||
envelope = _module(real_lock=True)
|
||||
calls = []
|
||||
directory = SimpleNamespace(assert_current=lambda path: calls.append(path), close=lambda: calls.append("close"))
|
||||
with envelope._NativeProbeLockStore(directory) as store:
|
||||
assert store.host is store.logical is directory
|
||||
store.assert_current()
|
||||
assert calls == [envelope.VOLUME_HOST_ROOT, "close"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_probe_lock(tmp_path, monkeypatch):
|
||||
if os.name != "posix" or os.geteuid() != 0:
|
||||
pytest.skip("real protected flock contract requires root/POSIX")
|
||||
envelope = _module(real_lock=True)
|
||||
root = tmp_path / "private-lock-root"
|
||||
root.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(envelope, "_open_host_probe_lock_store", lambda **_kwargs: _ProbeLockStore(root))
|
||||
return envelope, root
|
||||
|
||||
|
||||
def test_probe_lock_is_private_stable_and_released_after_failure(native_probe_lock):
|
||||
envelope, root = native_probe_lock
|
||||
with pytest.raises(ValueError, match="fixture failure"), envelope._host_probe_lifecycle_lock() as guard:
|
||||
guard()
|
||||
first = (root / envelope._HOST_PROBE_LOCK_NAME).stat()
|
||||
assert first.st_mode & 0o777 == 0o600
|
||||
raise ValueError("fixture failure")
|
||||
with envelope._host_probe_lifecycle_lock() as guard:
|
||||
guard()
|
||||
second = (root / envelope._HOST_PROBE_LOCK_NAME).stat()
|
||||
assert (second.st_dev, second.st_ino) == (first.st_dev, first.st_ino)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["mode", "owner", "symlink", "hardlink", "nonempty", "directory"])
|
||||
def test_probe_lock_refuses_untrusted_leaf_without_adopting(native_probe_lock, mutation):
|
||||
envelope, root = native_probe_lock
|
||||
path = root / envelope._HOST_PROBE_LOCK_NAME
|
||||
if mutation == "directory":
|
||||
path.mkdir(mode=0o700)
|
||||
elif mutation == "symlink":
|
||||
target = root / "foreign"
|
||||
target.write_bytes(b"")
|
||||
target.chmod(0o600)
|
||||
path.symlink_to(target)
|
||||
else:
|
||||
path.write_bytes(b"private-canary" if mutation == "nonempty" else b"")
|
||||
path.chmod(0o644 if mutation == "mode" else 0o600)
|
||||
if mutation == "owner":
|
||||
os.chown(path, 1000, 1000)
|
||||
elif mutation == "hardlink":
|
||||
os.link(path, root / "foreign-link")
|
||||
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_untrusted$"), envelope._host_probe_lifecycle_lock():
|
||||
pytest.fail("unsafe lock adopted")
|
||||
|
||||
|
||||
def test_probe_lock_refuses_replacement_and_different_volume_alias(native_probe_lock, monkeypatch, tmp_path):
|
||||
envelope, root = native_probe_lock
|
||||
with envelope._host_probe_lifecycle_lock() as guard:
|
||||
path = root / envelope._HOST_PROBE_LOCK_NAME
|
||||
path.rename(root / "original-owned-lock")
|
||||
path.touch(mode=0o600)
|
||||
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_untrusted$"):
|
||||
guard()
|
||||
other = tmp_path / "foreign-volume"
|
||||
other.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(envelope, "_open_host_probe_lock_store", lambda **_kwargs: _ProbeLockStore(root, other))
|
||||
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_store_untrusted$"), envelope._host_probe_lifecycle_lock():
|
||||
pytest.fail("unrelated aliases accepted")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wait", [0.05, 3])
|
||||
def test_probe_lock_excludes_independent_process_through_independent_view(native_probe_lock, wait):
|
||||
envelope, root = native_probe_lock
|
||||
# Independent interpreter and opens: no inherited Python lock or lock FD.
|
||||
child_source = (
|
||||
"import importlib.util,json,os,sys\n"
|
||||
"from pathlib import Path\n"
|
||||
"from types import SimpleNamespace\n"
|
||||
"sys.path.insert(0,sys.argv[1])\n"
|
||||
"spec=importlib.util.spec_from_file_location('probe_lock_child',Path(sys.argv[1])/'runner_resource_envelope.py')\n"
|
||||
"m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)\n"
|
||||
"class Store:\n"
|
||||
" def __enter__(self):\n"
|
||||
" self.logical=SimpleNamespace(fd=os.open(sys.argv[2],os.O_RDONLY|os.O_DIRECTORY))\n"
|
||||
" self.host=SimpleNamespace(fd=os.open(sys.argv[2],os.O_RDONLY|os.O_DIRECTORY));return self\n"
|
||||
" def assert_current(self):\n"
|
||||
" assert os.fstat(self.logical.fd).st_ino==os.fstat(self.host.fd).st_ino\n"
|
||||
" def __exit__(self,*args):os.close(self.logical.fd);os.close(self.host.fd)\n"
|
||||
"m._open_host_probe_lock_store=lambda **kw:Store();m._HOST_PROBE_LOCK_WAIT_SECONDS=float(sys.argv[3])\n"
|
||||
"print('started',flush=True)\n"
|
||||
"try:\n"
|
||||
" with m._host_probe_lifecycle_lock(native_cleanup=True) as guard:guard();print('acquired',flush=True)\n"
|
||||
"except RuntimeError as error:print(str(error),flush=True)\n"
|
||||
)
|
||||
with envelope._host_probe_lifecycle_lock():
|
||||
child = subprocess.Popen([sys.executable, "-c", child_source, str(ENVELOPE_PATH.parent), str(root), str(wait)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
try:
|
||||
assert child.stdout.readline().strip() == "started"
|
||||
if wait < 1:
|
||||
output, error = child.communicate(timeout=3)
|
||||
assert output.strip() == "resource_cgroup_probe_lock_busy"
|
||||
assert error == "" and child.returncode == 0
|
||||
else:
|
||||
assert child.poll() is None
|
||||
finally:
|
||||
if wait < 1 and child.poll() is None:
|
||||
child.kill()
|
||||
child.communicate()
|
||||
if wait >= 1:
|
||||
output, error = child.communicate(timeout=5)
|
||||
assert output.strip() == "acquired"
|
||||
assert error == "" and child.returncode == 0
|
||||
|
||||
|
||||
def test_concurrent_probe_lifecycles_both_complete_without_busy_or_stolen_cleanup(native_probe_lock, monkeypatch):
|
||||
envelope, _root = native_probe_lock
|
||||
probe, image_id, code_source = _host_probe_fixture(envelope)
|
||||
started, release = threading.Event(), threading.Event()
|
||||
state = {"exists": False, "starts": 0, "removals": 0}
|
||||
monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source))
|
||||
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {})
|
||||
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: copy.deepcopy(probe) if state["exists"] else None)
|
||||
|
||||
def run(args, **_kwargs):
|
||||
if args[:2] == ["docker", "create"]:
|
||||
assert not state["exists"]
|
||||
state["exists"] = True
|
||||
return probe["Id"]
|
||||
if args[:2] == ["docker", "inspect"]:
|
||||
assert state["exists"]
|
||||
return json.dumps([probe])
|
||||
assert args == ["docker", "rm", probe["Id"]]
|
||||
assert state["exists"] and probe["State"]["Running"] is False
|
||||
state["exists"] = False
|
||||
state["removals"] += 1
|
||||
return ""
|
||||
|
||||
def start(args, **_kwargs):
|
||||
assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]]
|
||||
state["starts"] += 1
|
||||
probe["State"]["Running"] = True
|
||||
if state["starts"] == 1:
|
||||
started.set()
|
||||
assert release.wait(3)
|
||||
probe["State"]["Running"] = False
|
||||
return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}')
|
||||
|
||||
monkeypatch.setattr(envelope, "_run", run)
|
||||
monkeypatch.setattr(envelope.subprocess, "run", start)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
first = pool.submit(envelope._run_host_probe, "snapshot")
|
||||
assert started.wait(3)
|
||||
second = pool.submit(envelope._run_host_probe, "snapshot")
|
||||
try:
|
||||
assert state["starts"] == 1 and not second.done()
|
||||
finally:
|
||||
release.set()
|
||||
assert first.result(timeout=5) == second.result(timeout=5) == {"verified": True}
|
||||
assert state["removals"] == 2 and state["exists"] is False
|
||||
|
||||
|
||||
def test_normal_probe_and_promotion_cleanup_share_same_complete_lock(monkeypatch):
|
||||
envelope = _module(real_lock=True)
|
||||
stages = []
|
||||
@contextmanager
|
||||
def lock(**kwargs):
|
||||
assert kwargs in ({}, {"native_cleanup": True})
|
||||
stages.append("acquire")
|
||||
yield lambda: stages.append("guard")
|
||||
stages.append("release")
|
||||
monkeypatch.setattr(envelope, "_host_probe_lifecycle_lock", lock)
|
||||
def normal(mode, payload, guard):
|
||||
assert mode == "family" and payload is None
|
||||
guard()
|
||||
stages.append("normal")
|
||||
return {"verified": True}
|
||||
def cleanup(**kwargs):
|
||||
kwargs["assert_lock"]()
|
||||
stages.append("cleanup")
|
||||
return True
|
||||
monkeypatch.setattr(envelope, "_run_host_probe_locked", normal)
|
||||
monkeypatch.setattr(envelope, "_cleanup_stopped_host_probe_locked", cleanup)
|
||||
assert envelope.inspect_host_cgroup_family() == {"verified": True}
|
||||
assert envelope.cleanup_stopped_host_probe(image_id="sha256:" + "a" * 64,
|
||||
code_source=f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'b' * 40}")
|
||||
assert stages == ["acquire", "guard", "normal", "release", "acquire", "guard", "cleanup", "release"]
|
||||
|
||||
|
||||
def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]:
|
||||
return {
|
||||
"path": path,
|
||||
@@ -747,9 +1022,17 @@ def test_real_probe_cleanup_cli_exposes_only_finite_child_failure(tmp_path, fail
|
||||
"elif a==['rm',v['probe']['Id']]: print(v['failure'],file=sys.stderr); sys.exit(1)\n" +
|
||||
"else: sys.exit(99)\n")
|
||||
shim.chmod(0o700)
|
||||
result = subprocess.run([sys.executable, str(ENVELOPE_PATH), "--cleanup-stopped-host-probe",
|
||||
image_id, code_source], capture_output=True, timeout=15,
|
||||
env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", "")})
|
||||
# Keep the actual CLI branch/diagnostics, isolating only its volume lock.
|
||||
entry = tmp_path / "cleanup-entry.py"
|
||||
source = ENVELOPE_PATH.read_text(encoding="utf-8")
|
||||
source = source.replace('if __name__ == "__main__":',
|
||||
'@contextmanager\ndef _host_probe_lifecycle_lock(**_kwargs):\n'
|
||||
' yield lambda: None\n\nif __name__ == "__main__":')
|
||||
entry.write_text(source, encoding="utf-8")
|
||||
result = subprocess.run([sys.executable, str(entry), "--cleanup-stopped-host-probe",
|
||||
image_id, code_source], capture_output=True, timeout=15, check=False,
|
||||
env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", ""),
|
||||
"PYTHONPATH": str(ENVELOPE_PATH.parent)})
|
||||
assert result.returncode == 1
|
||||
assert result.stdout == b""
|
||||
assert result.stderr == ("resource_cgroup_probe_cleanup_" + condition + "\n").encode()
|
||||
|
||||
Reference in new issue
Block a user