2082 lines
78 KiB
Python
2082 lines
78 KiB
Python
from __future__ import annotations
|
|
|
|
import ast
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from scripts import bootstrap_controller_artifact_promoter as bootstrap
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
LEGACY_REVISION = "5553c04ce497c08a35ed301052dc5e0b02e5d9b4"
|
|
LEGACY_TREE = "1624e1748053f998d8427c8c8a4203277d22f13c"
|
|
APPOPS_CONTAINER_ID = "72daa9f638defc4f8897d8aad23c4181054b2a1bc04c993078a5ac9597eeb0ec"
|
|
APPOPS_IMAGE_ID = "sha256:9b5eddf32c8eb5361106fc35cc0e056c49d287a20d1fb26490662cc8447f09d4"
|
|
CONTROLLER_CONTAINER_ID = "e4e7550405aa2a9a91901faf1177d46a35b0fc43c304976ec8971865d43e30eb"
|
|
CONTROLLER_IMAGE_ID = "sha256:c474d435a4f55ac5205570da0caa47e103f89a6c2ef96ea45861a23eb7b005dc"
|
|
CONTROLLER_IMAGE_REF = "docker.gitea.com/runner-images:ubuntu-latest"
|
|
CONTROLLER_REVISION = "034c9f844d2455850ba37b31379b7edaa786583e"
|
|
CONTROLLER_TREE = "6b8e92942e6e5eca8a9aa9da0a1509b2019d7b7b"
|
|
MODERN_CONTROLLER_CONTAINER_ID = "9093c8e29d9064f9948918e5884f9adc5db6b740991e2934902f93c6ea813e01"
|
|
MODERN_CONTROLLER_REVISION = "e854334923cca574482d22b8424ba3ce46ec6aa1"
|
|
MODERN_CONTROLLER_TREE = "cf2a117f9222b952deb9902c646116612f3a7ea0"
|
|
MODERN_CONTROLLER_ARTIFACT_SHA256 = "e88739bf091c8bac77ea0db5b97300666a1baa8ba984bbf6ae91e3bf25719e08"
|
|
REVISION = "a" * 40
|
|
TREE = "b" * 40
|
|
CALLER_ID = "c" * 64
|
|
MODERN_HELPER_ID = "f" * 64
|
|
RETIRED_HELPER_ID = bootstrap.RETIRED_FAILED_MODERN_HELPER_ID
|
|
HOST_HOSTNAME = "Tower"
|
|
|
|
|
|
LEGACY_HELPER_SOURCE_FIXTURE = '''
|
|
def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]:
|
|
if not _REVISION.fullmatch(revision):
|
|
raise ValueError("controller_promoter_revision_invalid")
|
|
if not _IMAGE_ID.fullmatch(image_id):
|
|
raise ValueError("controller_promoter_image_invalid")
|
|
|
|
command = f"""set -Eeuo pipefail
|
|
revision='{revision}'
|
|
version='{CONTROLLER_VERSION}'
|
|
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
|
|
work='/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{revision[:12]}'
|
|
rm -rf -- "$work"
|
|
install -d -m 0700 "$work"
|
|
cleanup() {{ rm -rf -- "$work"; }}
|
|
trap cleanup EXIT
|
|
command -v git >/dev/null
|
|
command -v docker >/dev/null
|
|
docker compose version >/dev/null
|
|
command -v python3 >/dev/null
|
|
command -v tar >/dev/null
|
|
command -v curl >/dev/null
|
|
git --git-dir="$repo" cat-file -e "$revision^{{commit}}"
|
|
tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")"
|
|
git --git-dir="$repo" archive --format=tar "$revision" \\
|
|
ops/gitea-deploy-broker \\
|
|
scripts/prepare_controller_artifact.py \\
|
|
scripts/reconcile_controller_state.py \\
|
|
scripts/run_repopulse_adoption.py > "$work/controller.tar"
|
|
archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)"
|
|
git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh"
|
|
chmod 0700 "$work/upgrade.sh"
|
|
bash "$work/upgrade.sh" \\
|
|
--archive "$work/controller.tar" \\
|
|
--archive-sha256 "$archive_sha" \\
|
|
--source-repository '{SOURCE_REPOSITORY}' \\
|
|
--source-revision "$revision" \\
|
|
--source-tree "$tree" \\
|
|
--controller-version "$version" \\
|
|
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
|
|
"""
|
|
|
|
return {
|
|
"Image": image_id,
|
|
"User": "0:0",
|
|
"Cmd": ["/bin/bash", "-lc", command],
|
|
"Labels": {
|
|
HELPER_LABEL: "true",
|
|
"itworx.source.repository": SOURCE_REPOSITORY,
|
|
"itworx.source.revision": revision,
|
|
"itworx.controller.version": CONTROLLER_VERSION,
|
|
},
|
|
"HostConfig": {
|
|
"Binds": [
|
|
"/mnt/cache:/mnt/cache",
|
|
"/mnt/user:/mnt/user",
|
|
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
|
|
"/var/run/docker.sock:/var/run/docker.sock",
|
|
],
|
|
"NetworkMode": "host",
|
|
"Privileged": False,
|
|
"SecurityOpt": ["no-new-privileges:true"],
|
|
"RestartPolicy": {"Name": "no"},
|
|
},
|
|
}
|
|
'''
|
|
|
|
|
|
def _legacy_helper_spec(revision: str = REVISION) -> dict[str, Any]:
|
|
tree = ast.parse(LEGACY_HELPER_SOURCE_FIXTURE)
|
|
function = next(
|
|
node
|
|
for node in tree.body
|
|
if isinstance(node, ast.FunctionDef)
|
|
and node.name == "helper_container_spec"
|
|
)
|
|
namespace: dict[str, Any] = {
|
|
"re": re,
|
|
"Any": Any,
|
|
"_REVISION": re.compile(r"^[0-9a-f]{40}$"),
|
|
"_IMAGE_ID": re.compile(r"^sha256:[0-9a-f]{64}$"),
|
|
"SOURCE_REPOSITORY": "jens/unraid-appops-gateway",
|
|
"HELPER_NAME": "itworx-controller-artifact-promoter",
|
|
"CONTROLLER_VERSION": "2.1.5",
|
|
"HELPER_LABEL": "itworx.controller.promoter",
|
|
}
|
|
exec(
|
|
compile(ast.Module(body=[function], type_ignores=[]), "legacy-promoter", "exec"),
|
|
namespace,
|
|
)
|
|
return namespace["helper_container_spec"](revision=revision, image_id=CONTROLLER_IMAGE_ID)
|
|
|
|
|
|
def _image(labels: dict[str, str]) -> dict[str, Any]:
|
|
return {"Id": APPOPS_IMAGE_ID, "Config": {"Labels": dict(labels)}}
|
|
|
|
|
|
def _appops_labels() -> dict[str, str]:
|
|
return {
|
|
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
|
|
"org.opencontainers.image.revision": LEGACY_REVISION,
|
|
"org.opencontainers.image.source-tree": LEGACY_TREE,
|
|
"org.opencontainers.image.version": "1.8.0-platform-5553c04ce497",
|
|
"org.opencontainers.image.created": "2026-10-01T16:32:01Z",
|
|
"com.docker.compose.project": "unraid-appops-gateway",
|
|
"com.docker.compose.service": "appops-gateway",
|
|
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
|
|
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
|
|
}
|
|
|
|
|
|
def _controller_labels() -> dict[str, str]:
|
|
return {
|
|
"io.buildah.version": "1.35.0",
|
|
"org.opencontainers.image.authors": "Gitea",
|
|
"org.opencontainers.image.created": "2026-10-01T00:01:57Z",
|
|
"org.opencontainers.image.description": "Official docker images used by Gitea runner to run workflows.",
|
|
"org.opencontainers.image.documentation": "https://gitea.com/gitea/runner-images/src/branch/main/README.md",
|
|
"org.opencontainers.image.licenses": "MIT",
|
|
"org.opencontainers.image.revision": "317f8e3c510e5ec773a103f5d9a9e5769105c59b",
|
|
"org.opencontainers.image.source": "https://gitea.com/gitea/runner-images",
|
|
"org.opencontainers.image.title": "Official Gitea Actions runner images",
|
|
"org.opencontainers.image.url": "https://gitea.com/gitea/runner-images",
|
|
"org.opencontainers.image.vendor": "Gitea",
|
|
"org.opencontainers.image.version": "v26.10.01",
|
|
}
|
|
|
|
|
|
def _controller_container_labels() -> dict[str, str]:
|
|
return {
|
|
**_controller_labels(),
|
|
"com.docker.compose.config-hash": "577d77247471213db8a5c465ee2450765ba52cb4048f983c42083ac45a16ddb7",
|
|
"com.docker.compose.container-number": "1",
|
|
"com.docker.compose.depends_on": "",
|
|
"com.docker.compose.image": "sha256:adbb51b2873bf05546019c341caf6fd4368159546c36a5aea81b4d8bf088804e",
|
|
"com.docker.compose.oneoff": "False",
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.project.config_files": "/mnt/cache/appdata/gitea-deploy-runners/docker-compose.yml",
|
|
"com.docker.compose.project.working_dir": "/mnt/cache/appdata/gitea-deploy-runners",
|
|
"com.docker.compose.replace": "gitea-deploy-control",
|
|
"com.docker.compose.service": "deploy-control",
|
|
"com.docker.compose.version": "2.40.3",
|
|
"tech.itworx.deployment.target": "unraid",
|
|
"tech.itworx.role": "gitea-deploy-controller",
|
|
"tech.itworx.gitea.scope": "Jens",
|
|
}
|
|
|
|
|
|
def _modern_controller_labels() -> dict[str, str]:
|
|
return {
|
|
**_controller_labels(),
|
|
"com.docker.compose.config-hash": "a" * 64,
|
|
"com.docker.compose.container-number": "1",
|
|
"com.docker.compose.depends_on": "",
|
|
"com.docker.compose.image": CONTROLLER_IMAGE_ID,
|
|
"com.docker.compose.oneoff": "False",
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.project.config_files": "-",
|
|
"com.docker.compose.project.environment_file": "/dev/null",
|
|
"com.docker.compose.project.working_dir": "/var/lib/docker/volumes/itworx-controller-hub-releases/_data",
|
|
"com.docker.compose.replace": "gitea-deploy-control",
|
|
"com.docker.compose.service": "deploy-control",
|
|
"com.docker.compose.version": "2.40.3",
|
|
"tech.itworx.deployment.target": "unraid",
|
|
"tech.itworx.gitea.scope": "Jens",
|
|
"tech.itworx.role": "gitea-deploy-controller",
|
|
}
|
|
|
|
|
|
def _modern_controller_mounts(source_revision: str) -> list[dict[str, Any]]:
|
|
artifact_source = (
|
|
"/var/lib/docker/volumes/itworx-controller-hub-releases/_data/"
|
|
f"controller-artifacts/{source_revision}"
|
|
)
|
|
return [
|
|
{"Type": "bind", "Source": "/boot", "Destination": "/boot", "RW": True, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/var/lib/docker", "Destination": "/var/lib/docker", "RW": False, "Propagation": "rslave"},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/srv/example/appdata/gitea/git/repositories",
|
|
"Destination": "/gitea-repositories",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{"Type": "bind", "Source": artifact_source, "Destination": "/opt/gitea-deploy", "RW": False, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/var/run/docker.sock", "Destination": "/var/run/docker.sock", "RW": True, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/mnt/user", "Destination": "/mnt/user", "RW": True, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/mnt/cache", "Destination": "/mnt/cache", "RW": True, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/mnt/disks", "Destination": "/mnt/disks", "RW": False, "Propagation": "rprivate"},
|
|
{
|
|
"Type": "volume",
|
|
"Name": "itworx-controller-hub-releases",
|
|
"Source": "/var/lib/docker/volumes/itworx-controller-hub-releases/_data",
|
|
"Destination": "/controller-hub-releases",
|
|
"RW": True,
|
|
"Propagation": "",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/var/lib/docker/volumes/itworx-controller-hub-releases/_data/controller-config",
|
|
"Destination": "/controller-config",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "volume",
|
|
"Name": "itworx-forge-master-key-backup",
|
|
"Source": "/var/lib/docker/volumes/itworx-forge-master-key-backup/_data",
|
|
"Destination": "/forge-master-key-backup",
|
|
"RW": True,
|
|
"Propagation": "",
|
|
},
|
|
]
|
|
|
|
|
|
def _modern_controller_source_mounts(source_revision: str) -> list[dict[str, Any]]:
|
|
mounts = [dict(mount) for mount in _modern_controller_mounts(source_revision)]
|
|
mounts = [mount for mount in mounts if mount["Destination"] != "/forge-master-key-backup"]
|
|
for mount in mounts:
|
|
if mount["Destination"] in {"/opt/gitea-deploy", "/controller-config"}:
|
|
mount["Propagation"] = "rslave"
|
|
return mounts
|
|
|
|
|
|
def _modern_controller_inspect(
|
|
source_revision: str,
|
|
*,
|
|
identifier: str = MODERN_CONTROLLER_CONTAINER_ID,
|
|
) -> dict[str, Any]:
|
|
controller = _controller_inspect()
|
|
controller.update({"Id": identifier, "Image": CONTROLLER_IMAGE_ID})
|
|
controller["State"].update({"Dead": False, "Paused": False, "Restarting": False})
|
|
controller["Config"]["Image"] = CONTROLLER_IMAGE_ID
|
|
controller["Config"]["Labels"] = _modern_controller_labels()
|
|
controller["Mounts"] = _modern_controller_mounts(source_revision)
|
|
return controller
|
|
|
|
|
|
def _modern_controller_source_inspect(
|
|
source_revision: str,
|
|
*,
|
|
identifier: str = MODERN_CONTROLLER_CONTAINER_ID,
|
|
) -> dict[str, Any]:
|
|
controller = _modern_controller_inspect(source_revision, identifier=identifier)
|
|
controller["Mounts"] = _modern_controller_source_mounts(source_revision)
|
|
return controller
|
|
|
|
|
|
def _appops_inspect() -> dict[str, Any]:
|
|
labels = _appops_labels()
|
|
return {
|
|
"Id": APPOPS_CONTAINER_ID,
|
|
"Image": APPOPS_IMAGE_ID,
|
|
"Name": "/Unraid-AppOps-Gateway",
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
"Config": {
|
|
"Hostname": "unraid-appops-gateway",
|
|
"Image": "unraid-appops-gateway:platform-5553c04ce497",
|
|
"Labels": labels,
|
|
},
|
|
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/srv/example/appdata/unraid-appops-gateway-1216",
|
|
"Destination": "/data",
|
|
"RW": True,
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/srv/example/appdata/comfyui-nvidia/basedir/models",
|
|
"Destination": "/mnt/privatemuse-comfyui-models",
|
|
"RW": True,
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def _controller_inspect() -> dict[str, Any]:
|
|
return {
|
|
"Id": CONTROLLER_CONTAINER_ID,
|
|
"Image": CONTROLLER_IMAGE_ID,
|
|
"Name": "/gitea-deploy-control",
|
|
"State": {"Running": True},
|
|
"Config": {"Image": CONTROLLER_IMAGE_REF, "Labels": _controller_container_labels()},
|
|
"HostConfig": {
|
|
"Binds": [
|
|
"/mnt/user:/mnt/user:rw",
|
|
"/mnt/cache:/mnt/cache:rw",
|
|
"/mnt/disks:/mnt/disks:ro",
|
|
"/boot:/boot:rw",
|
|
"/var/lib/docker:/var/lib/docker:ro",
|
|
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
|
|
"/mnt/cache/appdata/gitea-deploy-runners/controller:/opt/gitea-deploy:ro",
|
|
"/var/run/docker.sock:/var/run/docker.sock:rw",
|
|
],
|
|
"Mounts": None,
|
|
},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/boot",
|
|
"Destination": "/boot",
|
|
"RW": True,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/var/lib/docker",
|
|
"Destination": "/var/lib/docker",
|
|
"RW": False,
|
|
"Propagation": "rslave",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/srv/example/appdata/gitea/git/repositories",
|
|
"Destination": "/gitea-repositories",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/mnt/cache/appdata/gitea-deploy-runners/controller",
|
|
"Destination": "/opt/gitea-deploy",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/var/run/docker.sock",
|
|
"Destination": "/var/run/docker.sock",
|
|
"RW": True,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/mnt/user",
|
|
"Destination": "/mnt/user",
|
|
"RW": True,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/mnt/cache",
|
|
"Destination": "/mnt/cache",
|
|
"RW": True,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/mnt/disks",
|
|
"Destination": "/mnt/disks",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def _inspect_from_spec(
|
|
name: str,
|
|
identifier: str,
|
|
image_id: str,
|
|
spec: dict[str, Any],
|
|
*,
|
|
state: dict[str, Any],
|
|
hostname: str | None = None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"Id": identifier,
|
|
"Image": image_id,
|
|
"Name": f"/{name}",
|
|
"State": dict(state),
|
|
"Config": {
|
|
"Image": spec["Image"],
|
|
"User": spec["User"],
|
|
"Cmd": spec["Cmd"],
|
|
"Labels": {**_controller_labels(), **spec["Labels"]},
|
|
"Hostname": hostname if hostname is not None else identifier[:12],
|
|
},
|
|
"HostConfig": {"ReadonlyRootfs": False, **spec["HostConfig"]},
|
|
}
|
|
|
|
|
|
class FakeDocker:
|
|
def __init__(self, caller_spec: dict[str, Any], *, existing_child: dict[str, Any] | None = None):
|
|
legacy_caller = _inspect_from_spec(
|
|
"itworx-controller-artifact-promoter",
|
|
CALLER_ID,
|
|
CONTROLLER_IMAGE_ID,
|
|
caller_spec,
|
|
state={"Running": True},
|
|
hostname=HOST_HOSTNAME,
|
|
)
|
|
self.containers = {
|
|
APPOPS_CONTAINER_ID: _appops_inspect(),
|
|
"gitea-deploy-control": _controller_inspect(),
|
|
bootstrap.LEGACY_HELPER_NAME: legacy_caller,
|
|
CALLER_ID[:12]: legacy_caller,
|
|
}
|
|
self.images = {
|
|
APPOPS_IMAGE_ID: _image(_appops_labels()),
|
|
CONTROLLER_IMAGE_ID: {
|
|
"Id": CONTROLLER_IMAGE_ID,
|
|
"Config": {"Labels": _controller_labels()},
|
|
},
|
|
}
|
|
self.existing_child = existing_child
|
|
self.created: list[dict[str, Any]] = []
|
|
self.started: list[str] = []
|
|
self.removed: list[str] = []
|
|
self.inspected: list[str] = []
|
|
self.host_name = HOST_HOSTNAME
|
|
self.log_bytes = b"controller_v2_ready revision=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
|
|
self.named_inspect_override: dict[str, Any] | None = None
|
|
self.named_inspect_override_at = 0
|
|
self.retired_id_inspect_override: dict[str, Any] | None = None
|
|
self.appops_inspect_count = 0
|
|
self.appops_inspect_override: dict[str, Any] | None = None
|
|
self.appops_inspect_override_at = 0
|
|
self._inspect_counts: dict[str, int] = {}
|
|
|
|
def inspect_container(self, reference: str) -> dict[str, Any] | None:
|
|
self.inspected.append(reference)
|
|
self._inspect_counts[reference] = self._inspect_counts.get(reference, 0) + 1
|
|
if reference == "itworx-controller-artifact-promoter-modern":
|
|
if (
|
|
self.named_inspect_override is not None
|
|
and self._inspect_counts[reference] == self.named_inspect_override_at
|
|
):
|
|
return self.named_inspect_override
|
|
return self.existing_child
|
|
if (
|
|
reference == RETIRED_HELPER_ID
|
|
and self.retired_id_inspect_override is not None
|
|
):
|
|
return self.retired_id_inspect_override
|
|
if (
|
|
reference == RETIRED_HELPER_ID
|
|
and self.existing_child is not None
|
|
and self.existing_child.get("Id") == RETIRED_HELPER_ID
|
|
):
|
|
return self.existing_child
|
|
if reference == APPOPS_CONTAINER_ID:
|
|
self.appops_inspect_count += 1
|
|
if self.appops_inspect_count == self.appops_inspect_override_at:
|
|
return self.appops_inspect_override
|
|
return self.containers.get(reference)
|
|
|
|
def daemon_name(self) -> str:
|
|
return self.host_name
|
|
|
|
def inspect_image(self, image_id: str) -> dict[str, Any] | None:
|
|
return self.images.get(image_id)
|
|
|
|
def create_container(self, name: str, spec: dict[str, Any]) -> str:
|
|
assert name == "itworx-controller-artifact-promoter-modern"
|
|
self.created.append(spec)
|
|
container = _inspect_from_spec(
|
|
name,
|
|
MODERN_HELPER_ID,
|
|
spec["Image"],
|
|
spec,
|
|
state={"Running": False, "ExitCode": 0, "Status": "exited"},
|
|
)
|
|
self.containers[name] = container
|
|
self.containers[MODERN_HELPER_ID] = container
|
|
return MODERN_HELPER_ID
|
|
|
|
def start_container(self, identifier: str) -> None:
|
|
self.started.append(identifier)
|
|
source_revision = self.created[-1]["Labels"]["itworx.source.revision"]
|
|
self.containers["gitea-deploy-control"] = _modern_controller_inspect(
|
|
source_revision, identifier="e" * 64
|
|
)
|
|
|
|
def remove_retired_failed_container(self, identifier: str) -> None:
|
|
assert identifier == RETIRED_HELPER_ID
|
|
assert self.existing_child is not None
|
|
assert self.existing_child["Id"] == RETIRED_HELPER_ID
|
|
self.removed.append(identifier)
|
|
self.existing_child = None
|
|
|
|
def container_logs(self, identifier: str, *, tail: int, max_bytes: int) -> bytes:
|
|
assert identifier in {MODERN_HELPER_ID, RETIRED_HELPER_ID}
|
|
assert tail <= 40
|
|
assert max_bytes <= 8192
|
|
return self.log_bytes[:max_bytes]
|
|
|
|
|
|
class FakeProtectedGit:
|
|
def __init__(self, helper_source: str):
|
|
self.revision = REVISION
|
|
self.tree = TREE
|
|
self.helper_source = helper_source
|
|
self.blobs_read: list[tuple[str, str]] = []
|
|
self.main_checks = 0
|
|
self.fail_current_main_at: int | None = None
|
|
self.retired_tree_valid = True
|
|
self.controller_release_checks: list[bootstrap.ControllerBaseline] = []
|
|
self.invalid_controller_release: bootstrap.ControllerBaseline | None = None
|
|
|
|
def verify_current_main(self, revision: str, tree: str) -> None:
|
|
self.main_checks += 1
|
|
if self.fail_current_main_at == self.main_checks:
|
|
raise bootstrap.BootstrapError("controller_promoter_source_not_current_main")
|
|
if (revision, tree) != (self.revision, self.tree):
|
|
raise bootstrap.BootstrapError("controller_promoter_source_not_current_main")
|
|
|
|
def verify_legacy_revision(self, revision: str, tree: str) -> None:
|
|
if (revision, tree) != (LEGACY_REVISION, LEGACY_TREE):
|
|
raise bootstrap.BootstrapError("controller_promoter_legacy_provenance_invalid")
|
|
|
|
def verify_controller_release(self, baseline: bootstrap.ControllerBaseline) -> None:
|
|
self.controller_release_checks.append(baseline)
|
|
if baseline not in bootstrap.CONTROLLER_BASELINES or baseline == self.invalid_controller_release:
|
|
raise bootstrap.BootstrapError("controller_promoter_controller_provenance_invalid")
|
|
|
|
def verify_retired_modern_revision(self, revision: str, tree: str) -> None:
|
|
if (revision, tree) != (
|
|
bootstrap.RETIRED_FAILED_MODERN_REVISION,
|
|
bootstrap.RETIRED_FAILED_MODERN_TREE,
|
|
) or not self.retired_tree_valid:
|
|
raise bootstrap.BootstrapError(
|
|
"controller_promoter_retired_helper_provenance_invalid"
|
|
)
|
|
|
|
def read_blob(self, revision: str, path: str) -> bytes:
|
|
self.blobs_read.append((revision, path))
|
|
if revision not in {self.revision, bootstrap.RETIRED_FAILED_MODERN_REVISION} or path != "backend/app/services/controller_artifact_helper_spec.py":
|
|
raise bootstrap.BootstrapError("controller_promoter_source_blob_invalid")
|
|
return self.helper_source.encode("utf-8")
|
|
|
|
def read_legacy_blob(self, revision: str, path: str) -> bytes:
|
|
self.blobs_read.append((revision, path))
|
|
if (revision, path) != (
|
|
LEGACY_REVISION,
|
|
"backend/app/services/controller_artifact_promoter.py",
|
|
):
|
|
raise bootstrap.BootstrapError("controller_promoter_legacy_blob_invalid")
|
|
return LEGACY_HELPER_SOURCE_FIXTURE.encode("utf-8")
|
|
|
|
|
|
def _health(
|
|
*,
|
|
revision: str = REVISION,
|
|
tree: str = TREE,
|
|
version: str = "2.1.14",
|
|
artifact_sha256: str = "1" * 64,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"status": "ok",
|
|
"protocolVersion": "itworx-controller-request/v2",
|
|
"artifact": {
|
|
"apiVersion": "itworx.controller-artifact/v1",
|
|
"sourceRepository": "jens/unraid-appops-gateway",
|
|
"sourceRevision": revision,
|
|
"sourceTree": tree,
|
|
"controllerVersion": version,
|
|
"artifactSha256": artifact_sha256,
|
|
},
|
|
"delivery": {
|
|
"health": {
|
|
"configuration": "healthy",
|
|
"inbox": "healthy",
|
|
"artifact": "healthy",
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _modern_baseline_health() -> dict[str, Any]:
|
|
return _health(
|
|
revision=MODERN_CONTROLLER_REVISION,
|
|
tree=MODERN_CONTROLLER_TREE,
|
|
version="2.1.14",
|
|
artifact_sha256=MODERN_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
|
|
|
|
def _run_bootstrap(
|
|
docker: FakeDocker,
|
|
git: FakeProtectedGit,
|
|
*,
|
|
health: dict[str, Any] | None = None,
|
|
initial_health: dict[str, Any] | None = None,
|
|
health_sequence: list[dict[str, Any]] | None = None,
|
|
hostname: str = HOST_HOSTNAME,
|
|
source_revision: str = REVISION,
|
|
source_tree: str = TREE,
|
|
):
|
|
source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
git.helper_source = source
|
|
initial = initial_health or _health(
|
|
revision=CONTROLLER_REVISION,
|
|
tree=CONTROLLER_TREE,
|
|
version="2.1.12",
|
|
artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
health_reads = health_sequence or [initial, health or _health()]
|
|
|
|
def read_health() -> dict[str, Any]:
|
|
return health_reads.pop(0) if health_reads else health or _health()
|
|
|
|
return bootstrap.run_bootstrap(
|
|
"jens/unraid-appops-gateway",
|
|
source_revision,
|
|
source_tree,
|
|
docker=docker,
|
|
protected_git=git,
|
|
hostname=hostname,
|
|
health_reader=read_health,
|
|
monotonic=lambda: 0.0,
|
|
sleeper=lambda _seconds: None,
|
|
child_timeout_seconds=0.0,
|
|
)
|
|
|
|
|
|
def test_protected_git_reproduces_pinned_release_with_exact_handoff_crlf_conversion(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
):
|
|
git_executable = shutil.which("git")
|
|
if git_executable is None:
|
|
pytest.skip("git is required to validate the pinned source archives")
|
|
|
|
probe = subprocess.run(
|
|
[git_executable, "cat-file", "-e", f"{CONTROLLER_REVISION}^{{commit}}"],
|
|
cwd=ROOT,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
if probe.returncode != 0:
|
|
pytest.skip("the checkout does not expose the retained historical commits to Git")
|
|
|
|
def local_git_output(arguments: list[str], *, limit: int = bootstrap.MAX_GIT_BLOB_BYTES) -> bytes:
|
|
result = subprocess.run(
|
|
[git_executable, "-c", f"safe.directory={ROOT}", *arguments],
|
|
cwd=ROOT,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise bootstrap.BootstrapError("controller_promoter_protected_git_invalid")
|
|
if len(result.stdout) > limit:
|
|
raise bootstrap.BootstrapError("controller_promoter_protected_git_invalid")
|
|
return result.stdout
|
|
|
|
monkeypatch.setattr(bootstrap, "_git_output", local_git_output)
|
|
repository = bootstrap.ProtectedGitRepository()
|
|
for baseline in (
|
|
bootstrap.LEGACY_CONTROLLER_BASELINE,
|
|
bootstrap.MODERN_CONTROLLER_BASELINE,
|
|
):
|
|
repository.verify_controller_release(baseline)
|
|
with pytest.raises(
|
|
bootstrap.BootstrapError,
|
|
match="controller_promoter_controller_provenance_invalid",
|
|
):
|
|
repository.verify_controller_release(
|
|
replace(baseline, tree="f" * 40)
|
|
)
|
|
archive_paths = [
|
|
"scripts/run_repopulse_adoption.py"
|
|
if name == "run_repopulse_adoption.py"
|
|
else f"ops/gitea-deploy-broker/{name}"
|
|
for name in baseline.artifact_files
|
|
]
|
|
archive = local_git_output(
|
|
["archive", "--format=tar", baseline.revision, "--", *archive_paths],
|
|
limit=bootstrap.MAX_CONTROLLER_ARTIFACT_ARCHIVE_BYTES,
|
|
)
|
|
assert bootstrap._controller_artifact_archive_digest(
|
|
archive, baseline
|
|
) == baseline.artifact_sha256
|
|
|
|
with pytest.raises(
|
|
bootstrap.BootstrapError,
|
|
match="controller_promoter_controller_provenance_invalid",
|
|
):
|
|
bootstrap._controller_artifact_source_bytes(
|
|
replace(baseline, revision="f" * 40),
|
|
bootstrap.CONTROLLER_CRLF_HANDOFF,
|
|
b"source",
|
|
)
|
|
|
|
corrupted_archive = _rewrite_controller_archive(
|
|
archive,
|
|
mutate_path="ops/gitea-deploy-broker/deploy.py",
|
|
mutate=lambda content: content + b"# changed\n",
|
|
)
|
|
assert bootstrap._controller_artifact_archive_digest(
|
|
corrupted_archive, baseline
|
|
) != baseline.artifact_sha256
|
|
|
|
renamed_archive = _rewrite_controller_archive(
|
|
archive,
|
|
rename_path="scripts/run_repopulse_adoption.py",
|
|
new_path="scripts/unreviewed.py",
|
|
)
|
|
with pytest.raises(
|
|
bootstrap.BootstrapError,
|
|
match="controller_promoter_controller_provenance_invalid",
|
|
):
|
|
bootstrap._controller_artifact_archive_digest(renamed_archive, baseline)
|
|
|
|
handoff_path = "scripts/run_repopulse_adoption.py"
|
|
corrupted_handoff = _rewrite_controller_archive(
|
|
archive,
|
|
mutate_path=handoff_path,
|
|
mutate=lambda content: content + b"# changed\n",
|
|
)
|
|
with pytest.raises(
|
|
bootstrap.BootstrapError,
|
|
match="controller_promoter_controller_provenance_invalid",
|
|
):
|
|
bootstrap._controller_artifact_archive_digest(corrupted_handoff, baseline)
|
|
|
|
crlf_source_archive = _rewrite_controller_archive(
|
|
archive,
|
|
mutate_path=handoff_path,
|
|
mutate=lambda content: content.replace(b"\n", b"\r\r\n"),
|
|
)
|
|
with pytest.raises(
|
|
bootstrap.BootstrapError,
|
|
match="controller_promoter_controller_provenance_invalid",
|
|
):
|
|
bootstrap._controller_artifact_archive_digest(crlf_source_archive, baseline)
|
|
|
|
|
|
def _rewrite_controller_archive(
|
|
archive: bytes,
|
|
*,
|
|
rename_path: str | None = None,
|
|
new_path: str | None = None,
|
|
mutate_path: str | None = None,
|
|
mutate: Any | None = None,
|
|
) -> bytes:
|
|
output = io.BytesIO()
|
|
with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as original:
|
|
with tarfile.open(fileobj=output, mode="w:") as rewritten:
|
|
for member in original.getmembers():
|
|
if member.isdir():
|
|
rewritten.addfile(member)
|
|
continue
|
|
body = original.extractfile(member)
|
|
assert body is not None
|
|
content = body.read()
|
|
if member.name == rename_path:
|
|
assert new_path is not None
|
|
member.name = new_path
|
|
if member.name == mutate_path:
|
|
assert mutate is not None
|
|
content = mutate(content)
|
|
member.size = len(content)
|
|
rewritten.addfile(member, io.BytesIO(content))
|
|
return output.getvalue()
|
|
|
|
|
|
def test_controller_source_provenance_failure_precedes_health_and_side_effects():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
git = FakeProtectedGit(helper_source)
|
|
git.invalid_controller_release = bootstrap.MODERN_CONTROLLER_BASELINE
|
|
|
|
with pytest.raises(
|
|
bootstrap.BootstrapError,
|
|
match="controller_promoter_controller_provenance_invalid",
|
|
):
|
|
_run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[_modern_baseline_health()],
|
|
)
|
|
|
|
assert git.controller_release_checks == [bootstrap.MODERN_CONTROLLER_BASELINE]
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
assert docker.started == []
|
|
|
|
|
|
def _retired_failed_child(helper_source: str) -> dict[str, Any]:
|
|
spec = bootstrap.render_modern_helper_spec(
|
|
helper_source,
|
|
revision=bootstrap.RETIRED_FAILED_MODERN_REVISION,
|
|
image_id=CONTROLLER_IMAGE_ID,
|
|
current_controller_id=CONTROLLER_CONTAINER_ID,
|
|
current_controller_revision=CONTROLLER_REVISION,
|
|
current_controller_tree=CONTROLLER_TREE,
|
|
current_controller_version="2.1.12",
|
|
)
|
|
child = _inspect_from_spec(
|
|
"itworx-controller-artifact-promoter-modern",
|
|
RETIRED_HELPER_ID,
|
|
CONTROLLER_IMAGE_ID,
|
|
spec,
|
|
state={"Running": False, "ExitCode": 1, "Status": "exited"},
|
|
)
|
|
child["Config"]["Volumes"] = None
|
|
child["Mounts"] = [
|
|
{"Type": "bind", "Source": "/mnt/cache", "Destination": "/mnt/cache", "RW": True, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/mnt/user", "Destination": "/mnt/user", "RW": True, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/var/lib/docker", "Destination": "/var/lib/docker", "RW": False, "Propagation": "rslave"},
|
|
{"Type": "bind", "Source": "/srv/example/appdata/gitea/git/repositories", "Destination": "/gitea-repositories", "RW": False, "Propagation": "rprivate"},
|
|
{"Type": "bind", "Source": "/var/run/docker.sock", "Destination": "/var/run/docker.sock", "RW": True, "Propagation": "rprivate"},
|
|
]
|
|
child["HostConfig"].update(
|
|
{
|
|
"Mounts": None,
|
|
"CapAdd": None,
|
|
"CapDrop": None,
|
|
"Devices": None,
|
|
"DeviceRequests": None,
|
|
"PortBindings": None,
|
|
"VolumesFrom": None,
|
|
"PidMode": "",
|
|
"UTSMode": "",
|
|
"UsernsMode": "",
|
|
"CgroupParent": "",
|
|
"IpcMode": "private",
|
|
"CgroupnsMode": "private",
|
|
"Runtime": "runc",
|
|
"AutoRemove": False,
|
|
"PublishAllPorts": False,
|
|
}
|
|
)
|
|
return child
|
|
|
|
|
|
def test_initial_controller_runtime_accepts_exact_current_compose_container():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
base_image_labels = bootstrap._inspect_controller_base_image(docker)
|
|
health = _health(
|
|
revision=CONTROLLER_REVISION,
|
|
tree=CONTROLLER_TREE,
|
|
version="2.1.12",
|
|
artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
|
|
assert (
|
|
bootstrap._verify_controller_runtime(
|
|
docker,
|
|
health_payload=health,
|
|
base_image_labels=base_image_labels,
|
|
)
|
|
== base_image_labels
|
|
)
|
|
|
|
|
|
def test_modern_source_baseline_accepts_its_fixed_source_mount_profile():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.containers["gitea-deploy-control"] = _modern_controller_source_inspect(
|
|
MODERN_CONTROLLER_REVISION
|
|
)
|
|
base_image_labels = bootstrap._inspect_controller_base_image(docker)
|
|
|
|
assert (
|
|
bootstrap._verify_controller_runtime(
|
|
docker,
|
|
health_payload=_modern_baseline_health(),
|
|
base_image_labels=base_image_labels,
|
|
)
|
|
== base_image_labels
|
|
)
|
|
|
|
|
|
def test_modern_source_mount_profile_is_distinct_from_postpromotion_topology():
|
|
source_mounts = _modern_controller_source_mounts(MODERN_CONTROLLER_REVISION)
|
|
desired_mounts = _modern_controller_mounts(MODERN_CONTROLLER_REVISION)
|
|
|
|
assert len(source_mounts) == 10
|
|
assert len(desired_mounts) == 11
|
|
assert bootstrap._modern_controller_source_mounts_match(
|
|
source_mounts, MODERN_CONTROLLER_REVISION
|
|
)
|
|
assert not bootstrap._modern_controller_source_mounts_match(
|
|
desired_mounts, MODERN_CONTROLLER_REVISION
|
|
)
|
|
assert not bootstrap._modern_controller_source_mounts_match(
|
|
source_mounts, "9" * 40
|
|
)
|
|
assert bootstrap._modern_controller_mounts_match(
|
|
desired_mounts, MODERN_CONTROLLER_REVISION
|
|
)
|
|
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.containers["gitea-deploy-control"] = _modern_controller_source_inspect(
|
|
MODERN_CONTROLLER_REVISION
|
|
)
|
|
assert not bootstrap._verified_live_controller(
|
|
docker, source_revision=MODERN_CONTROLLER_REVISION
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"missing_mount",
|
|
"extra_backup_mount",
|
|
"artifact_propagation",
|
|
"config_propagation",
|
|
"integer_rw",
|
|
"duplicate_mount",
|
|
],
|
|
)
|
|
def test_modern_source_mount_profile_rejects_topology_drift(mutation: str):
|
|
mounts = _modern_controller_source_mounts(MODERN_CONTROLLER_REVISION)
|
|
if mutation == "missing_mount":
|
|
mounts.pop()
|
|
elif mutation == "extra_backup_mount":
|
|
backup = next(
|
|
mount
|
|
for mount in _modern_controller_mounts(MODERN_CONTROLLER_REVISION)
|
|
if mount["Destination"] == "/forge-master-key-backup"
|
|
)
|
|
mounts.append(backup)
|
|
elif mutation == "artifact_propagation":
|
|
mounts[3]["Propagation"] = "rprivate"
|
|
elif mutation == "config_propagation":
|
|
mounts[9]["Propagation"] = "rprivate"
|
|
elif mutation == "integer_rw":
|
|
mounts[1]["RW"] = 0
|
|
elif mutation == "duplicate_mount":
|
|
mounts.append(dict(mounts[0]))
|
|
|
|
assert not bootstrap._modern_controller_source_mounts_match(
|
|
mounts, MODERN_CONTROLLER_REVISION
|
|
)
|
|
|
|
|
|
def test_immutable_legacy_helper_source_fixture_passes_bootstrap_identity_guard():
|
|
legacy_spec = _legacy_helper_spec()
|
|
docker = FakeDocker(legacy_spec)
|
|
git = FakeProtectedGit("")
|
|
|
|
result = _run_bootstrap(docker, git)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert docker.created
|
|
assert git.blobs_read == [
|
|
(REVISION, "backend/app/services/controller_artifact_helper_spec.py"),
|
|
(LEGACY_REVISION, "backend/app/services/controller_artifact_promoter.py"),
|
|
]
|
|
|
|
|
|
def test_host_network_legacy_caller_is_resolved_by_fixed_name_not_host_hostname():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
git = FakeProtectedGit("")
|
|
|
|
result = _run_bootstrap(docker, git, hostname=HOST_HOSTNAME)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert bootstrap.LEGACY_HELPER_NAME in docker.inspected
|
|
assert HOST_HOSTNAME not in docker.inspected
|
|
assert docker.created
|
|
|
|
|
|
def test_docker_info_hostname_reader_returns_only_bounded_name(monkeypatch: pytest.MonkeyPatch):
|
|
client = object.__new__(bootstrap.DockerSocketClient)
|
|
requests: list[tuple[str, str, int]] = []
|
|
|
|
def fake_request(method: str, path: str, *, max_bytes: int) -> tuple[int, bytes]:
|
|
requests.append((method, path, max_bytes))
|
|
return 200, json.dumps({"Name": HOST_HOSTNAME, "ID": "must-not-be-returned"}).encode()
|
|
|
|
monkeypatch.setattr(client, "_request", fake_request)
|
|
|
|
assert client.daemon_name() == HOST_HOSTNAME
|
|
assert requests == [("GET", "/info", bootstrap.MAX_DOCKER_JSON_BYTES)]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name",
|
|
["", "Tower/other", "Tower\n", "Töwer", "bad..name", "-tower", "tower-", "a" * 254],
|
|
)
|
|
def test_docker_info_hostname_reader_rejects_malformed_or_non_ascii_names(
|
|
monkeypatch: pytest.MonkeyPatch, name: str
|
|
):
|
|
client = object.__new__(bootstrap.DockerSocketClient)
|
|
monkeypatch.setattr(
|
|
client,
|
|
"_request",
|
|
lambda *_args, **_kwargs: (200, json.dumps({"Name": name}).encode()),
|
|
)
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
client.daemon_name()
|
|
|
|
|
|
def test_docker_info_hostname_reader_rejects_oversized_response(monkeypatch: pytest.MonkeyPatch):
|
|
read_limits: list[int] = []
|
|
|
|
class Response:
|
|
status = 200
|
|
|
|
def read(self, limit: int) -> bytes:
|
|
read_limits.append(limit)
|
|
return b"x" * limit
|
|
|
|
class Connection:
|
|
def __init__(self, _socket_path: str):
|
|
pass
|
|
|
|
def request(self, *_args: Any, **_kwargs: Any) -> None:
|
|
pass
|
|
|
|
def getresponse(self) -> Response:
|
|
return Response()
|
|
|
|
def close(self) -> None:
|
|
pass
|
|
|
|
monkeypatch.setattr(bootstrap, "_UnixHTTPConnection", Connection)
|
|
|
|
with pytest.raises(bootstrap.BootstrapError, match="response_too_large"):
|
|
bootstrap.DockerSocketClient().daemon_name()
|
|
|
|
assert read_limits == [bootstrap.MAX_DOCKER_JSON_BYTES + 1]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("daemon_name", "process_hostname", "caller_hostname"),
|
|
[
|
|
("Tower", "tower", "Tower"),
|
|
("OtherHost", "Tower", "Tower"),
|
|
("Tower", "Tower", "OtherHost"),
|
|
("bad/name", "bad/name", "bad/name"),
|
|
("Töwer", "Töwer", "Töwer"),
|
|
],
|
|
)
|
|
def test_legacy_caller_rejects_hostname_source_disagreement_without_creating_child(
|
|
daemon_name: str, process_hostname: str, caller_hostname: str
|
|
):
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.host_name = daemon_name
|
|
docker.containers[bootstrap.LEGACY_HELPER_NAME]["Config"]["Hostname"] = caller_hostname
|
|
git = FakeProtectedGit("")
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(docker, git, hostname=process_hostname)
|
|
|
|
assert bootstrap.LEGACY_HELPER_NAME in docker.inspected
|
|
assert process_hostname not in docker.inspected
|
|
assert docker.created == []
|
|
|
|
|
|
def test_legacy_caller_rejects_malformed_container_id_without_creating_child():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.containers[bootstrap.LEGACY_HELPER_NAME]["Id"] = "not-a-container-id"
|
|
git = FakeProtectedGit("")
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(docker, git)
|
|
|
|
assert docker.created == []
|
|
|
|
|
|
def test_missing_fixed_legacy_name_does_not_fall_back_to_process_hostname():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.containers[HOST_HOSTNAME] = docker.containers.pop(bootstrap.LEGACY_HELPER_NAME)
|
|
git = FakeProtectedGit("")
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(docker, git, hostname=HOST_HOSTNAME)
|
|
|
|
assert bootstrap.LEGACY_HELPER_NAME in docker.inspected
|
|
assert HOST_HOSTNAME not in docker.inspected
|
|
assert docker.created == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"caller-image",
|
|
"caller-label",
|
|
"caller-revision",
|
|
"caller-command",
|
|
"caller-bind",
|
|
"appops-image",
|
|
"appops-provenance",
|
|
"target-id",
|
|
"target-source",
|
|
"target-tree",
|
|
"target-version",
|
|
"old-target-id",
|
|
"old-target-image",
|
|
"old-target-version",
|
|
"target-config-image",
|
|
"target-compose-hash",
|
|
"target-compose-missing",
|
|
"target-compose-extra",
|
|
"target-compose-image",
|
|
"target-tech-label",
|
|
"target-tech-extra",
|
|
"target-path",
|
|
"target-extra-mount",
|
|
"target-bind-mode",
|
|
"base-image-installed-provenance",
|
|
],
|
|
)
|
|
def test_bootstrap_rejects_unknown_runtime_identity_without_creating_child(mutation: str):
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
if mutation == "caller-image":
|
|
docker.containers[CALLER_ID[:12]]["Image"] = "sha256:" + "9" * 64
|
|
elif mutation == "caller-label":
|
|
docker.containers[CALLER_ID[:12]]["Config"]["Labels"]["itworx.controller.version"] = "2.1.4"
|
|
elif mutation == "caller-revision":
|
|
docker.containers[CALLER_ID[:12]]["Config"]["Labels"]["itworx.source.revision"] = "d" * 40
|
|
elif mutation == "caller-command":
|
|
docker.containers[CALLER_ID[:12]]["Config"]["Cmd"].append("--image-id")
|
|
elif mutation == "caller-bind":
|
|
docker.containers[CALLER_ID[:12]]["HostConfig"]["Binds"].append("/tmp:/payload")
|
|
elif mutation == "appops-image":
|
|
docker.containers[APPOPS_CONTAINER_ID]["Image"] = "sha256:" + "8" * 64
|
|
elif mutation == "appops-provenance":
|
|
docker.images[APPOPS_IMAGE_ID]["Config"]["Labels"]["org.opencontainers.image.revision"] = "f" * 40
|
|
elif mutation == "target-id":
|
|
docker.containers["gitea-deploy-control"]["Id"] = "0" * 64
|
|
elif mutation == "old-target-id":
|
|
docker.containers["gitea-deploy-control"]["Id"] = "5539b672e66890e657d0f3b0e6e94f9c9ca5b47c245e120d330f6f1d5be73fae"
|
|
elif mutation == "old-target-image":
|
|
docker.containers["gitea-deploy-control"]["Image"] = "sha256:adbb51b2873bf05546019c341caf6fd4368159546c36a5aea81b4d8bf088804e"
|
|
elif mutation == "target-source":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.revision"] = "f" * 40
|
|
elif mutation == "target-tree":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.source-tree"] = "f" * 40
|
|
elif mutation == "target-version":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.version"] = "2.1.11"
|
|
elif mutation == "old-target-version":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.version"] = "v26.09.29"
|
|
elif mutation == "target-config-image":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Image"] = "docker.gitea.com/runner-images:ubuntu-22.04"
|
|
elif mutation == "target-compose-hash":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["com.docker.compose.config-hash"] = "0" * 64
|
|
elif mutation == "target-compose-missing":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"].pop("com.docker.compose.service")
|
|
elif mutation == "target-compose-extra":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["com.docker.compose.container_name"] = "gitea-deploy-control"
|
|
elif mutation == "target-compose-image":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["com.docker.compose.image"] = CONTROLLER_IMAGE_ID
|
|
elif mutation == "target-tech-label":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["tech.itworx.role"] = "other"
|
|
elif mutation == "target-tech-extra":
|
|
docker.containers["gitea-deploy-control"]["Config"]["Labels"]["tech.itworx.extra"] = "unexpected"
|
|
elif mutation == "target-path":
|
|
docker.containers["gitea-deploy-control"]["Mounts"][3]["Destination"] = "/controller-config"
|
|
elif mutation == "target-extra-mount":
|
|
docker.containers["gitea-deploy-control"]["Mounts"].append(
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/tmp",
|
|
"Destination": "/tmp",
|
|
"RW": True,
|
|
"Propagation": "rprivate",
|
|
}
|
|
)
|
|
docker.containers["gitea-deploy-control"]["HostConfig"]["Binds"].append(
|
|
"/tmp:/tmp:rw"
|
|
)
|
|
elif mutation == "target-bind-mode":
|
|
docker.containers["gitea-deploy-control"]["HostConfig"]["Binds"][6] = (
|
|
"/mnt/cache/appdata/gitea-deploy-runners/controller:/opt/gitea-deploy:rw"
|
|
)
|
|
docker.containers["gitea-deploy-control"]["Mounts"][3]["RW"] = True
|
|
elif mutation == "base-image-installed-provenance":
|
|
labels = docker.images[CONTROLLER_IMAGE_ID]["Config"]["Labels"]
|
|
labels["org.opencontainers.image.revision"] = CONTROLLER_REVISION
|
|
labels["org.opencontainers.image.source-tree"] = CONTROLLER_TREE
|
|
git = FakeProtectedGit("")
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(docker, git)
|
|
|
|
assert docker.created == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
["revision", "tree", "version", "artifact-sha", "delivery-health"],
|
|
)
|
|
def test_bootstrap_requires_exact_mounted_controller_artifact_health(mutation: str):
|
|
initial = _health(
|
|
revision=CONTROLLER_REVISION,
|
|
tree=CONTROLLER_TREE,
|
|
version="2.1.12",
|
|
artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
artifact = initial["artifact"]
|
|
if mutation == "revision":
|
|
artifact["sourceRevision"] = "9" * 40
|
|
elif mutation == "tree":
|
|
artifact["sourceTree"] = "9" * 40
|
|
elif mutation == "version":
|
|
artifact["controllerVersion"] = "2.1.11"
|
|
elif mutation == "artifact-sha":
|
|
artifact["artifactSha256"] = "9" * 64
|
|
elif mutation == "delivery-health":
|
|
initial["delivery"]["health"]["artifact"] = "degraded"
|
|
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
git = FakeProtectedGit("")
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(docker, git, initial_health=initial)
|
|
assert docker.created == []
|
|
|
|
|
|
def test_running_modern_child_returns_unknown_without_duplicate_or_stop():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
helper_spec = bootstrap.render_modern_helper_spec(
|
|
helper_source,
|
|
revision=REVISION,
|
|
image_id=CONTROLLER_IMAGE_ID,
|
|
current_controller_id=CONTROLLER_CONTAINER_ID,
|
|
current_controller_revision=CONTROLLER_REVISION,
|
|
current_controller_tree=CONTROLLER_TREE,
|
|
current_controller_version="2.1.12",
|
|
)
|
|
existing = _inspect_from_spec(
|
|
"itworx-controller-artifact-promoter-modern",
|
|
MODERN_HELPER_ID,
|
|
CONTROLLER_IMAGE_ID,
|
|
helper_spec,
|
|
state={"Running": True, "Status": "running"},
|
|
)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=existing)
|
|
git = FakeProtectedGit(helper_source)
|
|
|
|
result = _run_bootstrap(docker, git, health=_health())
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["reason"] == "controller_promoter_modern_helper_running"
|
|
assert docker.created == []
|
|
assert docker.started == []
|
|
|
|
|
|
def test_exact_failed_historical_child_is_removed_before_current_helper_launch():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
legacy_health = _health(
|
|
revision=CONTROLLER_REVISION,
|
|
tree=CONTROLLER_TREE,
|
|
version="2.1.12",
|
|
artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
failed_child = _retired_failed_child(helper_source)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=failed_child)
|
|
git = FakeProtectedGit(helper_source)
|
|
|
|
result = _run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[
|
|
legacy_health,
|
|
legacy_health,
|
|
legacy_health,
|
|
legacy_health,
|
|
_health(),
|
|
],
|
|
)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert docker.removed == [RETIRED_HELPER_ID]
|
|
assert docker.created and docker.started == [MODERN_HELPER_ID]
|
|
assert RETIRED_HELPER_ID in docker.inspected
|
|
assert docker.appops_inspect_count == 2
|
|
assert git.blobs_read.count(
|
|
(bootstrap.RETIRED_FAILED_MODERN_REVISION, bootstrap.MODERN_HELPER_SOURCE_PATH)
|
|
) == 1
|
|
|
|
|
|
def test_current_214_baseline_is_verified_and_bound_to_new_helper():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.containers["gitea-deploy-control"] = _modern_controller_source_inspect(
|
|
MODERN_CONTROLLER_REVISION
|
|
)
|
|
git = FakeProtectedGit(helper_source)
|
|
|
|
result = _run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[_modern_baseline_health(), _health(), _health()],
|
|
)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert git.controller_release_checks == [bootstrap.MODERN_CONTROLLER_BASELINE]
|
|
assert docker.created[0]["Labels"] == {
|
|
"itworx.controller.promoter": "true",
|
|
"itworx.source.repository": "jens/unraid-appops-gateway",
|
|
"itworx.source.revision": REVISION,
|
|
"itworx.controller.version": "2.1.14",
|
|
"itworx.controller.bootstrap.target.id": MODERN_CONTROLLER_CONTAINER_ID,
|
|
"itworx.controller.bootstrap.target.image": CONTROLLER_IMAGE_ID,
|
|
"itworx.controller.bootstrap.target.revision": MODERN_CONTROLLER_REVISION,
|
|
"itworx.controller.bootstrap.target.tree": MODERN_CONTROLLER_TREE,
|
|
"itworx.controller.bootstrap.target.version": "2.1.14",
|
|
}
|
|
|
|
|
|
def test_retired_failed_child_is_removed_only_after_current_214_profile_rechecks():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
failed_child = _retired_failed_child(helper_source)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=failed_child)
|
|
docker.containers["gitea-deploy-control"] = _modern_controller_source_inspect(
|
|
MODERN_CONTROLLER_REVISION
|
|
)
|
|
git = FakeProtectedGit(helper_source)
|
|
|
|
result = _run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[
|
|
_modern_baseline_health(),
|
|
_modern_baseline_health(),
|
|
_modern_baseline_health(),
|
|
_modern_baseline_health(),
|
|
_health(),
|
|
_health(),
|
|
],
|
|
)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert docker.removed == [RETIRED_HELPER_ID]
|
|
assert docker.created[0]["Labels"]["itworx.controller.bootstrap.target.id"] == (
|
|
MODERN_CONTROLLER_CONTAINER_ID
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"container_id",
|
|
"image_id",
|
|
"source_revision",
|
|
"source_tree",
|
|
"artifact_digest",
|
|
"version",
|
|
"compose_label",
|
|
"extra_label",
|
|
"missing_label",
|
|
"extra_mount",
|
|
"missing_mount",
|
|
"writable_mount",
|
|
"integer_rw_mount",
|
|
"missing_rw_mount",
|
|
"malformed_mount_source",
|
|
"malformed_mount_name",
|
|
"mount_alias",
|
|
"dead_controller",
|
|
"paused_controller",
|
|
"restarting_controller",
|
|
"missing_state_flag",
|
|
"health_revision",
|
|
"health_tree",
|
|
"health_artifact",
|
|
"health_version",
|
|
],
|
|
)
|
|
def test_retired_cleanup_refuses_any_current_214_identity_or_topology_drift(mutation: str):
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
failed_child = _retired_failed_child(helper_source)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=failed_child)
|
|
controller = _modern_controller_source_inspect(MODERN_CONTROLLER_REVISION)
|
|
health = _modern_baseline_health()
|
|
if mutation == "container_id":
|
|
controller["Id"] = "9" * 64
|
|
elif mutation == "image_id":
|
|
controller["Image"] = "sha256:" + "9" * 64
|
|
elif mutation == "source_revision":
|
|
controller["Mounts"][3]["Source"] = (
|
|
"/var/lib/docker/volumes/itworx-controller-hub-releases/_data/"
|
|
+ "9" * 40
|
|
)
|
|
elif mutation == "source_tree":
|
|
health["artifact"]["sourceTree"] = "9" * 40
|
|
elif mutation == "artifact_digest":
|
|
health["artifact"]["artifactSha256"] = "9" * 64
|
|
elif mutation == "version":
|
|
controller["Config"]["Labels"]["org.opencontainers.image.version"] = "forged"
|
|
elif mutation == "compose_label":
|
|
controller["Config"]["Labels"]["com.docker.compose.service"] = "other"
|
|
elif mutation == "extra_label":
|
|
controller["Config"]["Labels"]["unexpected"] = "value"
|
|
elif mutation == "missing_label":
|
|
del controller["Config"]["Labels"]["tech.itworx.role"]
|
|
elif mutation == "extra_mount":
|
|
controller["Mounts"].append(dict(controller["Mounts"][0]))
|
|
controller["Mounts"][-1]["Destination"] = "/unexpected"
|
|
elif mutation == "missing_mount":
|
|
controller["Mounts"].pop()
|
|
elif mutation == "writable_mount":
|
|
controller["Mounts"][1]["RW"] = True
|
|
elif mutation == "integer_rw_mount":
|
|
controller["Mounts"][1]["RW"] = 0
|
|
elif mutation == "missing_rw_mount":
|
|
del controller["Mounts"][1]["RW"]
|
|
elif mutation == "malformed_mount_source":
|
|
controller["Mounts"][1]["Source"] = 123
|
|
elif mutation == "malformed_mount_name":
|
|
controller["Mounts"][8]["Name"] = None
|
|
elif mutation == "mount_alias":
|
|
controller["Mounts"][9]["Source"] = "/tmp/controller-config"
|
|
elif mutation == "dead_controller":
|
|
controller["State"]["Dead"] = True
|
|
elif mutation == "paused_controller":
|
|
controller["State"]["Paused"] = True
|
|
elif mutation == "restarting_controller":
|
|
controller["State"]["Restarting"] = True
|
|
elif mutation == "missing_state_flag":
|
|
del controller["State"]["Dead"]
|
|
elif mutation == "health_revision":
|
|
health["artifact"]["sourceRevision"] = "9" * 40
|
|
elif mutation == "health_tree":
|
|
health["artifact"]["sourceTree"] = "9" * 40
|
|
elif mutation == "health_artifact":
|
|
health["artifact"]["artifactSha256"] = "9" * 64
|
|
elif mutation == "health_version":
|
|
health["artifact"]["controllerVersion"] = "2.1.13"
|
|
docker.containers["gitea-deploy-control"] = controller
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(
|
|
docker,
|
|
FakeProtectedGit(helper_source),
|
|
health_sequence=[health, health, health],
|
|
)
|
|
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
assert docker.started == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"running",
|
|
"successful-exit",
|
|
"boolean-exit",
|
|
"other-exit",
|
|
"wrong-id",
|
|
"config-volumes",
|
|
"extra-host-mount",
|
|
"cap-add",
|
|
"device-request",
|
|
"docker-root-writable",
|
|
"socket-bind-changed",
|
|
"legacy-controller-stopped",
|
|
"current-main-changed",
|
|
"second-name-inspect-changed",
|
|
"second-id-inspect-changed",
|
|
"appops-stopped-before-delete",
|
|
"historical-tree-proof-failed",
|
|
],
|
|
)
|
|
def test_retired_helper_cleanup_fails_closed_on_any_identity_or_state_drift(mutation: str):
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
failed_child = _retired_failed_child(helper_source)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=failed_child)
|
|
git = FakeProtectedGit(helper_source)
|
|
if mutation == "running":
|
|
failed_child["State"].update({"Running": True, "Status": "running"})
|
|
elif mutation == "successful-exit":
|
|
failed_child["State"]["ExitCode"] = 0
|
|
elif mutation == "boolean-exit":
|
|
failed_child["State"]["ExitCode"] = True
|
|
elif mutation == "other-exit":
|
|
failed_child["State"]["ExitCode"] = 2
|
|
elif mutation == "wrong-id":
|
|
failed_child["Id"] = "9" * 64
|
|
elif mutation == "config-volumes":
|
|
failed_child["Config"]["Volumes"] = {"/extra": {}}
|
|
elif mutation == "extra-host-mount":
|
|
failed_child["HostConfig"]["Mounts"] = [{"Type": "volume"}]
|
|
elif mutation == "cap-add":
|
|
failed_child["HostConfig"]["CapAdd"] = ["SYS_ADMIN"]
|
|
elif mutation == "device-request":
|
|
failed_child["HostConfig"]["DeviceRequests"] = [{"Capabilities": [["gpu"]]}]
|
|
elif mutation == "docker-root-writable":
|
|
failed_child["HostConfig"]["Binds"][2] = "/var/lib/docker:/var/lib/docker:rw"
|
|
failed_child["Mounts"][2]["RW"] = True
|
|
elif mutation == "socket-bind-changed":
|
|
failed_child["HostConfig"]["Binds"][4] = "/tmp/docker.sock:/var/run/docker.sock"
|
|
failed_child["Mounts"][4]["Source"] = "/tmp/docker.sock"
|
|
elif mutation == "legacy-controller-stopped":
|
|
docker.containers["gitea-deploy-control"]["State"]["Running"] = False
|
|
elif mutation == "current-main-changed":
|
|
git.fail_current_main_at = 2
|
|
elif mutation == "second-name-inspect-changed":
|
|
docker.named_inspect_override = json.loads(json.dumps(failed_child))
|
|
docker.named_inspect_override["Config"]["Cmd"].append("unexpected")
|
|
docker.named_inspect_override_at = 2
|
|
elif mutation == "second-id-inspect-changed":
|
|
docker.retired_id_inspect_override = json.loads(json.dumps(failed_child))
|
|
docker.retired_id_inspect_override["State"]["ExitCode"] = 0
|
|
elif mutation == "appops-stopped-before-delete":
|
|
docker.appops_inspect_override = json.loads(
|
|
json.dumps(docker.containers[APPOPS_CONTAINER_ID])
|
|
)
|
|
docker.appops_inspect_override["State"]["Running"] = False
|
|
docker.appops_inspect_override_at = 2
|
|
elif mutation == "historical-tree-proof-failed":
|
|
git.retired_tree_valid = False
|
|
|
|
legacy_health = _health(
|
|
revision=CONTROLLER_REVISION,
|
|
tree=CONTROLLER_TREE,
|
|
version="2.1.12",
|
|
artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
_run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[legacy_health, legacy_health, legacy_health],
|
|
)
|
|
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
assert docker.started == []
|
|
|
|
|
|
def test_retired_helper_remove_uses_non_force_non_volume_docker_delete():
|
|
client = bootstrap.DockerSocketClient()
|
|
calls: list[tuple[str, str]] = []
|
|
|
|
def request(method: str, path: str, **_kwargs: Any) -> tuple[int, bytes]:
|
|
calls.append((method, path))
|
|
return 204, b""
|
|
|
|
client._request = request # type: ignore[method-assign]
|
|
client.remove_retired_failed_container(RETIRED_HELPER_ID)
|
|
|
|
assert calls == [
|
|
("DELETE", f"/containers/{RETIRED_HELPER_ID}?force=false&v=false")
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("status", [404, 409])
|
|
def test_retired_helper_delete_refuses_unexpected_docker_status(status: int):
|
|
client = bootstrap.DockerSocketClient()
|
|
|
|
def request(_method: str, _path: str, **_kwargs: Any) -> tuple[int, bytes]:
|
|
return status, b""
|
|
|
|
client._request = request # type: ignore[method-assign]
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
client.remove_retired_failed_container(RETIRED_HELPER_ID)
|
|
|
|
|
|
def test_same_failed_source_revision_is_never_retried_or_removed():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
docker = FakeDocker(
|
|
_legacy_helper_spec(bootstrap.RETIRED_FAILED_MODERN_REVISION),
|
|
existing_child=_retired_failed_child(helper_source),
|
|
)
|
|
git = FakeProtectedGit(helper_source)
|
|
git.revision = bootstrap.RETIRED_FAILED_MODERN_REVISION
|
|
git.tree = bootstrap.RETIRED_FAILED_MODERN_TREE
|
|
legacy_health = _health(
|
|
revision=CONTROLLER_REVISION,
|
|
tree=CONTROLLER_TREE,
|
|
version="2.1.12",
|
|
artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256,
|
|
)
|
|
|
|
result = _run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[legacy_health],
|
|
source_revision=git.revision,
|
|
source_tree=git.tree,
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["exitCode"] == 1
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
|
|
|
|
def test_completed_modern_child_with_wrong_artifact_is_not_relaunched():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
helper_spec = bootstrap.render_modern_helper_spec(
|
|
helper_source,
|
|
revision=REVISION,
|
|
image_id=CONTROLLER_IMAGE_ID,
|
|
current_controller_id=CONTROLLER_CONTAINER_ID,
|
|
current_controller_revision=CONTROLLER_REVISION,
|
|
current_controller_tree=CONTROLLER_TREE,
|
|
current_controller_version="2.1.12",
|
|
)
|
|
existing = _inspect_from_spec(
|
|
"itworx-controller-artifact-promoter-modern",
|
|
MODERN_HELPER_ID,
|
|
CONTROLLER_IMAGE_ID,
|
|
helper_spec,
|
|
state={"Running": False, "ExitCode": 0, "Status": "exited"},
|
|
)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=existing)
|
|
git = FakeProtectedGit(helper_source)
|
|
|
|
result = _run_bootstrap(
|
|
docker,
|
|
git,
|
|
initial_health=_health(tree="9" * 40),
|
|
)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["reason"] == "controller_promoter_postcheck_failed"
|
|
assert docker.created == []
|
|
|
|
|
|
def test_completed_child_uses_new_health_after_controller_recreation():
|
|
helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
helper_spec = bootstrap.render_modern_helper_spec(
|
|
helper_source,
|
|
revision=REVISION,
|
|
image_id=CONTROLLER_IMAGE_ID,
|
|
current_controller_id=CONTROLLER_CONTAINER_ID,
|
|
current_controller_revision=CONTROLLER_REVISION,
|
|
current_controller_tree=CONTROLLER_TREE,
|
|
current_controller_version="2.1.12",
|
|
)
|
|
existing = _inspect_from_spec(
|
|
"itworx-controller-artifact-promoter-modern",
|
|
MODERN_HELPER_ID,
|
|
CONTROLLER_IMAGE_ID,
|
|
helper_spec,
|
|
state={"Running": False, "ExitCode": 0, "Status": "exited"},
|
|
)
|
|
docker = FakeDocker(_legacy_helper_spec(), existing_child=existing)
|
|
docker.containers["gitea-deploy-control"] = _modern_controller_inspect(REVISION)
|
|
git = FakeProtectedGit(helper_source)
|
|
|
|
result = _run_bootstrap(
|
|
docker,
|
|
git,
|
|
initial_health=_health(),
|
|
)
|
|
|
|
assert result["status"] == "already_applied"
|
|
assert result["revision"] == REVISION
|
|
assert docker.created == []
|
|
assert docker.started == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"post-image-config",
|
|
"post-compose-hash",
|
|
"post-compose-image",
|
|
"post-compose-extra",
|
|
"post-compose-environment-file",
|
|
"post-tech-label",
|
|
"post-tech-extra",
|
|
"post-artifact-revision",
|
|
"post-volume-name",
|
|
"post-volume-source",
|
|
"post-volume-mode",
|
|
"post-config-bind-source",
|
|
"post-config-bind-mode",
|
|
"post-code-bind-propagation",
|
|
"post-config-bind-propagation",
|
|
"post-missing-backup-volume",
|
|
"post-backup-volume-type",
|
|
"post-backup-volume-name",
|
|
"post-backup-volume-source",
|
|
"post-backup-volume-target",
|
|
"post-backup-volume-mode",
|
|
"post-backup-volume-propagation",
|
|
"post-duplicate-backup-volume",
|
|
"post-extra-mount",
|
|
],
|
|
)
|
|
def test_postactivation_controller_requires_exact_compose_runtime(mutation: str):
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
controller = _modern_controller_inspect(REVISION)
|
|
docker.containers["gitea-deploy-control"] = controller
|
|
|
|
if mutation == "post-image-config":
|
|
controller["Config"]["Image"] = CONTROLLER_IMAGE_REF
|
|
elif mutation == "post-compose-hash":
|
|
controller["Config"]["Labels"]["com.docker.compose.config-hash"] = "bad"
|
|
elif mutation == "post-compose-image":
|
|
controller["Config"]["Labels"]["com.docker.compose.image"] = "sha256:" + "d" * 64
|
|
elif mutation == "post-compose-extra":
|
|
controller["Config"]["Labels"]["com.docker.compose.extra"] = "unexpected"
|
|
elif mutation == "post-compose-environment-file":
|
|
controller["Config"]["Labels"]["com.docker.compose.project.environment_file"] = "/tmp/.env"
|
|
elif mutation == "post-tech-label":
|
|
controller["Config"]["Labels"]["tech.itworx.gitea.scope"] = "Other"
|
|
elif mutation == "post-tech-extra":
|
|
controller["Config"]["Labels"]["tech.itworx.extra"] = "unexpected"
|
|
elif mutation == "post-artifact-revision":
|
|
controller["Mounts"][3]["Source"] = (
|
|
"/var/lib/docker/volumes/itworx-controller-hub-releases/_data/"
|
|
f"controller-artifacts/{'d' * 40}"
|
|
)
|
|
elif mutation == "post-volume-name":
|
|
controller["Mounts"][8]["Name"] = "foreign-volume"
|
|
elif mutation == "post-volume-source":
|
|
controller["Mounts"][8]["Source"] = "/tmp/foreign-volume"
|
|
elif mutation == "post-volume-mode":
|
|
controller["Mounts"][8]["RW"] = False
|
|
elif mutation == "post-config-bind-source":
|
|
controller["Mounts"][9]["Source"] = "/tmp/config"
|
|
elif mutation == "post-config-bind-mode":
|
|
controller["Mounts"][9]["RW"] = True
|
|
elif mutation == "post-code-bind-propagation":
|
|
controller["Mounts"][3]["Propagation"] = "rslave"
|
|
elif mutation == "post-config-bind-propagation":
|
|
controller["Mounts"][9]["Propagation"] = "shared"
|
|
elif mutation == "post-missing-backup-volume":
|
|
controller["Mounts"].pop(10)
|
|
elif mutation == "post-backup-volume-type":
|
|
controller["Mounts"][10]["Type"] = "bind"
|
|
elif mutation == "post-backup-volume-name":
|
|
controller["Mounts"][10]["Name"] = "foreign-volume"
|
|
elif mutation == "post-backup-volume-source":
|
|
controller["Mounts"][10]["Source"] = "/tmp/foreign-volume"
|
|
elif mutation == "post-backup-volume-target":
|
|
controller["Mounts"][10]["Destination"] = "/tmp/backup"
|
|
elif mutation == "post-backup-volume-mode":
|
|
controller["Mounts"][10]["RW"] = False
|
|
elif mutation == "post-backup-volume-propagation":
|
|
controller["Mounts"][10]["Propagation"] = "rprivate"
|
|
elif mutation == "post-duplicate-backup-volume":
|
|
controller["Mounts"].append(dict(controller["Mounts"][10]))
|
|
elif mutation == "post-extra-mount":
|
|
controller["Mounts"].append(dict(controller["Mounts"][0]))
|
|
|
|
assert not bootstrap._verified_live_controller(docker, source_revision=REVISION)
|
|
|
|
|
|
def test_postactivation_accepts_well_formed_compose_hash():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
controller = _modern_controller_inspect(REVISION)
|
|
controller["Config"]["Labels"]["com.docker.compose.config-hash"] = "b" * 64
|
|
docker.containers["gitea-deploy-control"] = controller
|
|
|
|
assert bootstrap._verified_live_controller(docker, source_revision=REVISION)
|
|
|
|
|
|
def test_postactivation_accepts_exact_forge_backup_mount_in_eleven_mount_inventory():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
controller = _modern_controller_inspect(REVISION)
|
|
docker.containers["gitea-deploy-control"] = controller
|
|
|
|
assert bootstrap._verified_live_controller(docker, source_revision=REVISION)
|
|
|
|
|
|
def test_successful_child_requires_exact_source_tree_version_and_health():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
git = FakeProtectedGit("")
|
|
|
|
result = _run_bootstrap(docker, git)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert result["revision"] == REVISION
|
|
assert result["controllerVersion"] == "2.1.14"
|
|
assert result["artifactSha256"] == "1" * 64
|
|
assert docker.started == [MODERN_HELPER_ID]
|
|
assert docker.created[0]["Image"] == CONTROLLER_IMAGE_ID
|
|
assert docker.created[0]["HostConfig"]["ReadonlyRootfs"] is True
|
|
|
|
|
|
def test_child_logs_are_bounded_and_redacted_before_result_output():
|
|
docker = FakeDocker(_legacy_helper_spec())
|
|
docker.log_bytes = (
|
|
b"controller_v2_ready revision=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
|
|
b"GITEA_ADMIN_TOKEN=very-secret-value\n"
|
|
)
|
|
git = FakeProtectedGit("")
|
|
|
|
result = _run_bootstrap(docker, git)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert "very-secret-value" not in result["logTail"]
|
|
assert "[redacted]" in result["logTail"]
|
|
assert len(result["logTail"].encode("utf-8")) <= 4096
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"temporary_root_info,accepted",
|
|
[
|
|
("1777:0:0", True),
|
|
("700:0:0", True),
|
|
("777:0:0", False),
|
|
("755:0:0", False),
|
|
("770:0:0", False),
|
|
("1700:0:0", False),
|
|
("1777:1000:0", False),
|
|
("700:0:1000", False),
|
|
],
|
|
)
|
|
def test_legacy_upgrade_signature_fetches_bootstrap_only_from_protected_git(
|
|
tmp_path: Path, temporary_root_info: str, accepted: bool
|
|
):
|
|
bash = shutil.which("bash")
|
|
if bash is None:
|
|
pytest.skip("bash is required to exercise the upgrade script")
|
|
bin_dir = tmp_path / "bin"
|
|
bin_dir.mkdir()
|
|
git_trace = tmp_path / "git-args.jsonl"
|
|
python_trace = tmp_path / "python-args.json"
|
|
protected_source = "# trusted protected Git bootstrap source\n"
|
|
git_shim = bin_dir / "git"
|
|
git_shim.write_text(
|
|
f"#!{sys.executable}\n"
|
|
"import json, os, sys\n"
|
|
f"trace = {str(git_trace)!r}\n"
|
|
f"with open(trace, 'a', encoding='utf-8') as stream: stream.write(json.dumps(sys.argv[1:]) + '\\n')\n"
|
|
"args = [arg for arg in sys.argv[1:] if not arg.startswith('--git-dir=')]\n"
|
|
f"revision = {REVISION!r}\n"
|
|
f"tree = {TREE!r}\n"
|
|
"if args[-3:] == ['rev-parse', 'refs/heads/main', '--'] or args[-2:] == ['rev-parse', 'refs/heads/main']: print(revision)\n"
|
|
"elif any(arg.endswith('^{tree}') for arg in args): print(tree)\n"
|
|
f"elif len(args) >= 3 and args[:2] == ['cat-file', '-s']: print({len(protected_source)})\n"
|
|
"elif len(args) >= 3 and args[0] == 'cat-file' and args[1] == 'blob' and args[-1].endswith(':scripts/bootstrap_controller_artifact_promoter.py'):\n"
|
|
f" sys.stdout.write({protected_source!r})\n"
|
|
"else: raise SystemExit(91)\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat_shim = bin_dir / "stat"
|
|
stat_shim.write_text(
|
|
f"#!{sys.executable}\n"
|
|
"import pathlib, sys\n"
|
|
"args = sys.argv[1:]\n"
|
|
"target = args[-1]\n"
|
|
f"if target == '/tmp': print({temporary_root_info!r})\n"
|
|
"elif args[1] == '%u:%a': print('0:700')\n"
|
|
"elif args[1] == '%u:%a:%s': print('0:600:' + str(pathlib.Path(target).stat().st_size))\n"
|
|
"else: raise SystemExit(1)\n",
|
|
encoding="utf-8",
|
|
)
|
|
python_shim = bin_dir / "python3"
|
|
python_shim.write_text(
|
|
f"#!{sys.executable}\n"
|
|
"import json, pathlib, sys\n"
|
|
f"path = pathlib.Path({str(python_trace)!r})\n"
|
|
"path.write_text(json.dumps({'script': pathlib.Path(sys.argv[1]).read_text(), 'args': sys.argv[2:]}), encoding='utf-8')\n",
|
|
encoding="utf-8",
|
|
)
|
|
git_shim.chmod(0o755)
|
|
stat_shim.chmod(0o755)
|
|
python_shim.chmod(0o755)
|
|
archive = f"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{REVISION[:12]}/controller.tar"
|
|
result = subprocess.run(
|
|
[
|
|
bash,
|
|
str(ROOT / "scripts/upgrade_deployment_controller.sh"),
|
|
"--archive",
|
|
archive,
|
|
"--archive-sha256",
|
|
"a" * 64,
|
|
"--source-repository",
|
|
"jens/unraid-appops-gateway",
|
|
"--source-revision",
|
|
REVISION,
|
|
"--source-tree",
|
|
TREE,
|
|
"--controller-version",
|
|
"2.1.5",
|
|
"--confirm",
|
|
"UPGRADE_DEPLOYMENT_CONTROLLER_V2",
|
|
],
|
|
cwd=ROOT,
|
|
env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"},
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
if not accepted:
|
|
assert result.returncode == 66, result.stderr
|
|
assert "temporary root is unsafe" in result.stderr
|
|
assert not python_trace.exists()
|
|
calls = [json.loads(line) for line in git_trace.read_text(encoding="utf-8").splitlines()]
|
|
assert not any("cat-file" in args for args in calls)
|
|
return
|
|
assert result.returncode == 0, result.stderr
|
|
bootstrap_call = json.loads(python_trace.read_text(encoding="utf-8"))
|
|
assert bootstrap_call["script"] == protected_source
|
|
assert bootstrap_call["args"] == [
|
|
"--source-repository",
|
|
"jens/unraid-appops-gateway",
|
|
"--source-revision",
|
|
REVISION,
|
|
"--source-tree",
|
|
TREE,
|
|
]
|
|
calls = [json.loads(line) for line in git_trace.read_text(encoding="utf-8").splitlines()]
|
|
assert any(args[-1:] == [f"{REVISION}:scripts/bootstrap_controller_artifact_promoter.py"] for args in calls)
|
|
assert all(archive not in args for args in calls)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
["source", "revision", "tree", "version", "extra", "image-id"],
|
|
)
|
|
def test_upgrade_script_rejects_mutated_legacy_call_without_bootstrap(
|
|
tmp_path: Path, mutation: str
|
|
):
|
|
bash = shutil.which("bash")
|
|
if bash is None:
|
|
pytest.skip("bash is required to exercise the upgrade script")
|
|
bin_dir = tmp_path / "bin"
|
|
bin_dir.mkdir()
|
|
marker = tmp_path / "bootstrap-ran"
|
|
python_shim = bin_dir / "python3"
|
|
python_shim.write_text(
|
|
f"#!/usr/bin/env python3\nfrom pathlib import Path\nPath({str(marker)!r}).write_text('ran')\n",
|
|
encoding="utf-8",
|
|
)
|
|
python_shim.chmod(0o755)
|
|
args = [
|
|
"--archive",
|
|
f"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{REVISION[:12]}/controller.tar",
|
|
"--archive-sha256",
|
|
"a" * 64,
|
|
"--source-repository",
|
|
"jens/unraid-appops-gateway",
|
|
"--source-revision",
|
|
REVISION,
|
|
"--source-tree",
|
|
TREE,
|
|
"--controller-version",
|
|
"2.1.5",
|
|
"--confirm",
|
|
"UPGRADE_DEPLOYMENT_CONTROLLER_V2",
|
|
]
|
|
if mutation == "source":
|
|
args[args.index("jens/unraid-appops-gateway")] = "attacker/repo"
|
|
elif mutation == "revision":
|
|
args[args.index(REVISION)] = "A" * 40
|
|
elif mutation == "tree":
|
|
args[args.index(TREE)] = "f" * 40
|
|
elif mutation == "version":
|
|
args[args.index("2.1.5")] = "2.1.4"
|
|
elif mutation == "extra":
|
|
args.extend(["--unexpected", "value"])
|
|
elif mutation == "image-id":
|
|
args.extend(["--image-id", CONTROLLER_IMAGE_ID])
|
|
result = subprocess.run(
|
|
[bash, str(ROOT / "scripts/upgrade_deployment_controller.sh"), *args],
|
|
cwd=ROOT,
|
|
env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"},
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert not marker.exists()
|
|
|
|
|
|
@pytest.mark.parametrize("modern_call", ["missing-image", "shared-archive"])
|
|
@pytest.mark.parametrize("temporary_root_info", ["1777:0:0", "700:0:0", "777:0:0"])
|
|
def test_modern_upgrade_path_still_requires_private_archive_and_image_id(
|
|
tmp_path: Path, modern_call: str, temporary_root_info: str
|
|
):
|
|
bash = shutil.which("bash")
|
|
if bash is None:
|
|
pytest.skip("bash is required to exercise the upgrade script")
|
|
bin_dir = tmp_path / "bin"
|
|
bin_dir.mkdir()
|
|
stat_shim = bin_dir / "stat"
|
|
stat_shim.write_text(
|
|
f"#!{sys.executable}\nimport sys\n"
|
|
f"if sys.argv[-1] == '/tmp': print({temporary_root_info!r})\n"
|
|
"else: raise SystemExit(91)\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat_shim.chmod(0o755)
|
|
archive = (
|
|
f"/tmp/itworx-controller-artifact-promoter-{REVISION[:12]}.AAAAAA/controller.tar"
|
|
if modern_call == "missing-image"
|
|
else f"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{REVISION[:12]}/controller.tar"
|
|
)
|
|
args = [
|
|
"--archive",
|
|
archive,
|
|
"--archive-sha256",
|
|
"a" * 64,
|
|
"--source-repository",
|
|
"jens/unraid-appops-gateway",
|
|
"--source-revision",
|
|
REVISION,
|
|
"--source-tree",
|
|
TREE,
|
|
"--controller-version",
|
|
"2.1.14",
|
|
"--confirm",
|
|
"UPGRADE_DEPLOYMENT_CONTROLLER_V2",
|
|
]
|
|
if modern_call == "shared-archive":
|
|
args.extend(["--image-id", CONTROLLER_IMAGE_ID])
|
|
result = subprocess.run(
|
|
[bash, str(ROOT / "scripts/upgrade_deployment_controller.sh"), *args],
|
|
cwd=ROOT,
|
|
env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"},
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
if modern_call == "shared-archive":
|
|
if temporary_root_info == "777:0:0":
|
|
assert "helper temporary root is unsafe" in result.stderr
|
|
else:
|
|
assert "archive is outside the helper private temporary directory" in result.stderr
|
|
if modern_call == "missing-image":
|
|
assert "image ID" in result.stderr
|