628 lines
21 KiB
Python
628 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from backend.tests import test_controller_artifact_bootstrap as original
|
|
from scripts import bootstrap_controller_artifact_promoter as bootstrap
|
|
|
|
FAILED_REVISION = "1" * 40
|
|
FAILED_TREE = "2" * 40
|
|
FAILED_HELPER_ID = "3" * 64
|
|
CURRENT_HELPER_ID = "4" * 64
|
|
DRIFTED_HELPER_ID = "9" * 64
|
|
ZERO_DOCKER_TIME = "0001-01-01T00:00:00Z"
|
|
|
|
|
|
class Git(original.FakeProtectedGit):
|
|
def __init__(self, helper_source: str):
|
|
super().__init__(helper_source)
|
|
|
|
def verify_failed_modern_ancestor(
|
|
self, revision: str, current_revision: str
|
|
) -> str:
|
|
if (
|
|
revision != FAILED_REVISION
|
|
or current_revision != self.revision
|
|
or not self.retired_tree_valid
|
|
):
|
|
raise bootstrap.BootstrapError(
|
|
"controller_promoter_failed_helper_not_ancestor"
|
|
)
|
|
return FAILED_TREE
|
|
|
|
def read_blob(self, revision: str, path: str) -> bytes:
|
|
if revision == FAILED_REVISION:
|
|
self.blobs_read.append((revision, path))
|
|
if path != bootstrap.MODERN_HELPER_SOURCE_PATH:
|
|
raise bootstrap.BootstrapError(
|
|
"controller_promoter_source_blob_invalid"
|
|
)
|
|
return self.helper_source.encode("utf-8")
|
|
return super().read_blob(revision, path)
|
|
|
|
|
|
class Docker(original.FakeDocker):
|
|
def __init__(self, existing_child: dict[str, Any]):
|
|
super().__init__(original._legacy_helper_spec(), existing_child=existing_child)
|
|
self.name_reads = 0
|
|
self.name_drift_at: int | None = None
|
|
self.name_drift: str | None = None
|
|
self.id_reads = 0
|
|
self.id_drift_at: int | None = None
|
|
self.id_drift: str | None = None
|
|
self.create_attempts: list[tuple[str, dict[str, Any]]] = []
|
|
self.log_reads: list[str] = []
|
|
self.created_log_bytes = b""
|
|
self.created_log_failure = False
|
|
self.complete_existing_on_start = False
|
|
self.existing_exit_code = 0
|
|
|
|
def inspect_container(self, reference: str) -> dict[str, Any] | None:
|
|
if (
|
|
reference == bootstrap.MODERN_HELPER_NAME
|
|
and self.existing_child is not None
|
|
):
|
|
self.name_reads += 1
|
|
snapshot = copy.deepcopy(self.existing_child)
|
|
if self.name_reads == self.name_drift_at:
|
|
if self.name_drift == "id":
|
|
snapshot["Id"] = DRIFTED_HELPER_ID
|
|
elif self.name_drift == "name":
|
|
snapshot["Name"] = "/itworx-controller-artifact-promoter-foreign"
|
|
return snapshot
|
|
if self.existing_child is not None and reference == self.existing_child.get(
|
|
"Id"
|
|
):
|
|
self.id_reads += 1
|
|
snapshot = copy.deepcopy(self.existing_child)
|
|
if self.id_reads == self.id_drift_at:
|
|
if self.id_drift == "id":
|
|
snapshot["Id"] = DRIFTED_HELPER_ID
|
|
elif self.id_drift == "name":
|
|
snapshot["Name"] = "/itworx-controller-artifact-promoter-foreign"
|
|
return snapshot
|
|
return super().inspect_container(reference)
|
|
|
|
def create_container(self, name: str, spec: dict[str, Any]) -> str:
|
|
self.create_attempts.append((name, copy.deepcopy(spec)))
|
|
return super().create_container(name, spec)
|
|
|
|
def remove_retired_failed_container(self, identifier: str) -> None:
|
|
assert self.existing_child is not None
|
|
assert identifier == self.existing_child["Id"]
|
|
self.removed.append(identifier)
|
|
self.existing_child = None
|
|
|
|
def start_container(self, identifier: str) -> None:
|
|
self.started.append(identifier)
|
|
if self.existing_child is not None and identifier == self.existing_child["Id"]:
|
|
if self.complete_existing_on_start:
|
|
self.existing_child["State"].update(
|
|
Running=False,
|
|
Status="exited",
|
|
ExitCode=self.existing_exit_code,
|
|
)
|
|
source_revision = self.existing_child["Config"]["Labels"][
|
|
"itworx.source.revision"
|
|
]
|
|
self.containers[bootstrap.CONTROLLER_NAME] = (
|
|
original._modern_controller_inspect(
|
|
source_revision, identifier="e" * 64
|
|
)
|
|
)
|
|
else:
|
|
self.existing_child["State"].update(Running=True, Status="running")
|
|
return
|
|
child = self.containers.get(identifier)
|
|
if child is None:
|
|
return
|
|
child["State"].update(Running=True, Status="running")
|
|
source_revision = self.created[-1]["Labels"]["itworx.source.revision"]
|
|
self.containers[bootstrap.CONTROLLER_NAME] = (
|
|
original._modern_controller_inspect(source_revision, identifier="e" * 64)
|
|
)
|
|
|
|
def container_logs(self, identifier: str, *, tail: int, max_bytes: int) -> bytes:
|
|
self.log_reads.append(identifier)
|
|
assert tail <= 40
|
|
assert max_bytes <= bootstrap.MAX_DOCKER_LOG_BYTES
|
|
return self.log_bytes[:max_bytes]
|
|
|
|
def container_logs_for_created_container(
|
|
self, identifier: str, *, tail: int, max_bytes: int
|
|
) -> bytes:
|
|
if self.created_log_failure:
|
|
raise bootstrap.BootstrapError("controller_promoter_docker_logs_failed")
|
|
if self.existing_child is not None and identifier == self.existing_child["Id"]:
|
|
return self.created_log_bytes
|
|
return b""
|
|
|
|
|
|
def _created_state() -> dict[str, Any]:
|
|
return {
|
|
"Status": "created",
|
|
"Running": False,
|
|
"Paused": False,
|
|
"Restarting": False,
|
|
"Dead": False,
|
|
"Pid": 0,
|
|
"OOMKilled": False,
|
|
"ExitCode": 0,
|
|
"StartedAt": ZERO_DOCKER_TIME,
|
|
"FinishedAt": ZERO_DOCKER_TIME,
|
|
"Error": "",
|
|
}
|
|
|
|
|
|
def _inputs(*, current_child: bool) -> tuple[Docker, Git, dict[str, Any], str]:
|
|
helper_source = (original.ROOT / bootstrap.MODERN_HELPER_SOURCE_PATH).read_text(
|
|
encoding="utf-8"
|
|
)
|
|
child_revision = original.REVISION if current_child else FAILED_REVISION
|
|
child_id = CURRENT_HELPER_ID if current_child else FAILED_HELPER_ID
|
|
baseline = bootstrap.MODERN_CONTROLLER_BASELINE
|
|
spec = bootstrap.render_modern_helper_spec(
|
|
helper_source,
|
|
revision=child_revision,
|
|
image_id=baseline.image_id,
|
|
current_controller_id=baseline.container_id,
|
|
current_controller_revision=baseline.revision,
|
|
current_controller_tree=baseline.tree,
|
|
current_controller_version=baseline.version,
|
|
)
|
|
child = original._retired_failed_child()
|
|
child["Id"] = child_id
|
|
child["Image"] = baseline.image_id
|
|
child["State"] = _created_state()
|
|
child["RestartCount"] = 0
|
|
|
|
docker = Docker(child)
|
|
image_config = docker.images[baseline.image_id]["Config"]
|
|
image_config.update(
|
|
{
|
|
"Env": ["PATH=/usr/bin"],
|
|
"WorkingDir": "/tmp",
|
|
"Entrypoint": None,
|
|
"Shell": ["/bin/bash", "-c"],
|
|
}
|
|
)
|
|
base_image_labels = bootstrap._inspect_controller_base_image(docker)
|
|
child["Config"] = {
|
|
"Hostname": docker.host_name,
|
|
"Domainname": "",
|
|
"User": spec["User"],
|
|
"AttachStdin": False,
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"OpenStdin": False,
|
|
"StdinOnce": False,
|
|
"Cmd": copy.deepcopy(spec["Cmd"]),
|
|
"Image": spec["Image"],
|
|
"Labels": {**base_image_labels, **spec["Labels"]},
|
|
"Env": list(image_config["Env"]),
|
|
"Volumes": None,
|
|
"WorkingDir": image_config["WorkingDir"],
|
|
"Entrypoint": image_config["Entrypoint"],
|
|
"OnBuild": None,
|
|
}
|
|
child["HostConfig"] = {
|
|
**copy.deepcopy(bootstrap.FAILED_HELPER_HOST_DEFAULTS),
|
|
**copy.deepcopy(spec["HostConfig"]),
|
|
"RestartPolicy": {"Name": "no", "MaximumRetryCount": 0},
|
|
}
|
|
docker.containers[bootstrap.CONTROLLER_NAME] = (
|
|
original._modern_controller_source_inspect(baseline.revision)
|
|
)
|
|
return docker, Git(helper_source), child, child_revision
|
|
|
|
|
|
def _run(
|
|
docker: Docker,
|
|
git: Git,
|
|
*,
|
|
health: dict[str, Any] | None = None,
|
|
health_reader=None,
|
|
) -> dict[str, Any]:
|
|
payload = health or original._modern_baseline_health()
|
|
return bootstrap.run_bootstrap(
|
|
bootstrap.SOURCE_REPOSITORY,
|
|
git.revision,
|
|
git.tree,
|
|
docker=docker,
|
|
protected_git=git,
|
|
hostname=original.HOST_HOSTNAME,
|
|
health_reader=health_reader or (lambda: copy.deepcopy(payload)),
|
|
monotonic=lambda: 0.0,
|
|
sleeper=lambda _seconds: None,
|
|
child_timeout_seconds=0.0,
|
|
)
|
|
|
|
|
|
def test_current_created_child_resumes_after_exact_baseline_and_identity_rechecks():
|
|
docker, git, child, _revision = _inputs(current_child=True)
|
|
|
|
result = _run(docker, git)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert docker.started == [child["Id"]]
|
|
assert docker.created == []
|
|
assert docker.create_attempts == []
|
|
assert docker.removed == []
|
|
|
|
|
|
def test_current_created_child_success_is_reported_as_promoted():
|
|
docker, git, child, _revision = _inputs(current_child=True)
|
|
docker.complete_existing_on_start = True
|
|
health_reads = 0
|
|
|
|
def health_reader() -> dict[str, Any]:
|
|
nonlocal health_reads
|
|
health_reads += 1
|
|
if health_reads <= 3:
|
|
return original._modern_baseline_health()
|
|
return original._health(revision=git.revision, tree=git.tree)
|
|
|
|
result = _run(docker, git, health_reader=health_reader)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert result["revision"] == git.revision
|
|
assert docker.started == [child["Id"]]
|
|
assert docker.log_reads == [child["Id"]]
|
|
assert docker.create_attempts == []
|
|
assert docker.created == []
|
|
assert docker.removed == []
|
|
|
|
|
|
def test_resumed_child_failure_wins_even_if_health_has_reached_target():
|
|
docker, git, child, _revision = _inputs(current_child=True)
|
|
docker.complete_existing_on_start = True
|
|
docker.existing_exit_code = 17
|
|
health_reads = 0
|
|
|
|
def health_reader() -> dict[str, Any]:
|
|
nonlocal health_reads
|
|
health_reads += 1
|
|
if health_reads <= 3:
|
|
return original._modern_baseline_health()
|
|
return original._health(revision=git.revision, tree=git.tree)
|
|
|
|
result = _run(docker, git, health_reader=health_reader)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["exitCode"] == 17
|
|
assert docker.started == [child["Id"]]
|
|
assert docker.log_reads == [child["Id"]]
|
|
assert docker.create_attempts == []
|
|
assert docker.created == []
|
|
|
|
|
|
def test_noncurrent_created_child_is_retired_without_starting_its_old_revision():
|
|
docker, git, child, _revision = _inputs(current_child=False)
|
|
|
|
result = _run(docker, git)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert docker.removed == [child["Id"]]
|
|
assert docker.started == [original.MODERN_HELPER_ID]
|
|
assert docker.created[0]["Labels"]["itworx.source.revision"] == git.revision
|
|
assert len(docker.create_attempts) == 1
|
|
|
|
|
|
def test_current_created_name_collision_uses_the_same_recovery_proof():
|
|
docker, git, child, _revision = _inputs(current_child=True)
|
|
docker.existing_child = None
|
|
collision = copy.deepcopy(child)
|
|
|
|
def collide(_name: str, _spec: dict[str, Any]) -> str:
|
|
docker.create_attempts.append((_name, copy.deepcopy(_spec)))
|
|
docker.existing_child = collision
|
|
raise bootstrap.BootstrapError(
|
|
"controller_promoter_modern_helper_create_failed"
|
|
)
|
|
|
|
docker.create_container = collide
|
|
|
|
result = _run(docker, git)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert docker.started == [child["Id"]]
|
|
assert docker.created == []
|
|
assert len(docker.create_attempts) == 1
|
|
assert docker.removed == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"state",
|
|
[
|
|
{"Status": "created", "Running": False, "ExitCode": 0},
|
|
{"Status": "exited", "Running": True, "ExitCode": 0},
|
|
{"Status": "exited", "Running": False, "ExitCode": True},
|
|
{"Status": "exited", "Running": False, "ExitCode": "0"},
|
|
],
|
|
)
|
|
def test_completed_child_result_checks_terminal_integer_state_before_logs(
|
|
state: dict[str, Any],
|
|
):
|
|
docker, _git, child, _revision = _inputs(current_child=True)
|
|
child["State"] = state
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
bootstrap._completed_child_result(
|
|
child,
|
|
docker=docker,
|
|
health_reader=original._health,
|
|
source_revision=original.REVISION,
|
|
source_tree=original.TREE,
|
|
already_applied=True,
|
|
)
|
|
|
|
assert docker.log_reads == []
|
|
|
|
|
|
@pytest.mark.parametrize("current_child", [True, False])
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"status",
|
|
"exit_code_bool",
|
|
"exit_code_nonzero",
|
|
"exit_code_string",
|
|
"paused",
|
|
"restarting",
|
|
"dead",
|
|
"started_at",
|
|
"finished_at",
|
|
"started_at_type",
|
|
"finished_at_type",
|
|
"error",
|
|
"error_type",
|
|
"pid",
|
|
"pid_type",
|
|
"oom_killed",
|
|
"restart_count",
|
|
"restart_count_type",
|
|
],
|
|
)
|
|
def test_created_child_requires_exact_never_started_state(
|
|
current_child: bool, mutation: str
|
|
):
|
|
docker, git, _child, _revision = _inputs(current_child=current_child)
|
|
state = docker.existing_child["State"]
|
|
if mutation == "status":
|
|
state["Status"] = "created "
|
|
elif mutation == "exit_code_bool":
|
|
state["ExitCode"] = False
|
|
elif mutation == "exit_code_nonzero":
|
|
state["ExitCode"] = 1
|
|
elif mutation == "exit_code_string":
|
|
state["ExitCode"] = "0"
|
|
elif mutation in {"paused", "restarting", "dead"}:
|
|
state[mutation.capitalize()] = True
|
|
elif mutation == "started_at":
|
|
state["StartedAt"] = "2026-10-07T09:00:00Z"
|
|
elif mutation == "finished_at":
|
|
state["FinishedAt"] = "2026-10-07T09:00:00Z"
|
|
elif mutation == "started_at_type":
|
|
state["StartedAt"] = 0
|
|
elif mutation == "finished_at_type":
|
|
state["FinishedAt"] = 0
|
|
elif mutation == "pid":
|
|
state["Pid"] = 32
|
|
elif mutation == "pid_type":
|
|
state["Pid"] = False
|
|
elif mutation == "oom_killed":
|
|
state["OOMKilled"] = True
|
|
elif mutation == "restart_count":
|
|
docker.existing_child["RestartCount"] = 1
|
|
elif mutation == "restart_count_type":
|
|
docker.existing_child["RestartCount"] = False
|
|
elif mutation == "error_type":
|
|
state["Error"] = None
|
|
else:
|
|
state["Error"] = "start failed"
|
|
|
|
try:
|
|
result = _run(docker, git)
|
|
except bootstrap.BootstrapError:
|
|
pass
|
|
else:
|
|
assert result["status"] == "outcome_unknown"
|
|
|
|
assert docker.started == []
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
|
|
|
|
@pytest.mark.parametrize("current_child", [True, False])
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"command",
|
|
"environment",
|
|
"label",
|
|
"mount",
|
|
"logs",
|
|
"log_error",
|
|
"target_baseline",
|
|
"health_artifact",
|
|
"main",
|
|
"controller",
|
|
"appops",
|
|
],
|
|
)
|
|
def test_created_child_profile_or_frozen_baseline_drift_prevents_action(
|
|
current_child: bool, mutation: str
|
|
):
|
|
docker, git, _child, _revision = _inputs(current_child=current_child)
|
|
health = original._modern_baseline_health()
|
|
if mutation == "command":
|
|
docker.existing_child["Config"]["Cmd"][2] += "\necho changed"
|
|
elif mutation == "environment":
|
|
docker.existing_child["Config"]["Env"].append("CUSTOM_VARIABLE=extra")
|
|
elif mutation == "label":
|
|
docker.existing_child["Config"]["Labels"]["itworx.unexpected"] = "true"
|
|
elif mutation == "mount":
|
|
docker.existing_child["Mounts"][0]["RW"] = not docker.existing_child["Mounts"][
|
|
0
|
|
]["RW"]
|
|
elif mutation == "logs":
|
|
docker.created_log_bytes = b"unexpected pre-start output\n"
|
|
elif mutation == "log_error":
|
|
docker.created_log_failure = True
|
|
elif mutation in {"target_baseline", "health_artifact"}:
|
|
health = original._health()
|
|
if mutation == "health_artifact":
|
|
health = original._modern_baseline_health()
|
|
health["artifact"]["artifactSha256"] = "f" * 64
|
|
elif mutation == "main":
|
|
git.fail_current_main_at = 2
|
|
elif mutation == "controller":
|
|
docker.containers[bootstrap.CONTROLLER_NAME]["Id"] = DRIFTED_HELPER_ID
|
|
else:
|
|
docker.containers[original.APPOPS_CONTAINER_ID]["Image"] = "sha256:" + "8" * 64
|
|
|
|
try:
|
|
result = _run(docker, git, health=health)
|
|
except bootstrap.BootstrapError:
|
|
pass
|
|
else:
|
|
assert result["status"] == "outcome_unknown"
|
|
|
|
assert docker.started == []
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
|
|
|
|
@pytest.mark.parametrize("current_child", [True, False])
|
|
@pytest.mark.parametrize(
|
|
("reference", "read_number", "drift"),
|
|
[
|
|
("name", 2, "id"),
|
|
("name", 2, "name"),
|
|
("name", 3, "id"),
|
|
("name", 3, "name"),
|
|
("id", 1, "id"),
|
|
("id", 1, "name"),
|
|
("id", 2, "id"),
|
|
("id", 2, "name"),
|
|
],
|
|
)
|
|
def test_created_child_name_or_id_drift_on_reinspection_prevents_action(
|
|
current_child: bool, reference: str, read_number: int, drift: str
|
|
):
|
|
docker, git, _child, _revision = _inputs(current_child=current_child)
|
|
if reference == "name":
|
|
# The first fixed-name lookup precedes the two proof snapshot pairs.
|
|
docker.name_drift_at = read_number
|
|
docker.name_drift = drift
|
|
else:
|
|
docker.id_drift_at = read_number
|
|
docker.id_drift = drift
|
|
|
|
try:
|
|
result = _run(docker, git)
|
|
except bootstrap.BootstrapError:
|
|
pass
|
|
else:
|
|
assert result["status"] == "outcome_unknown"
|
|
|
|
assert docker.started == []
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
assert docker.create_attempts == []
|
|
|
|
|
|
def test_running_current_helper_remains_unknown_and_is_never_started_again():
|
|
docker, git, _child, _revision = _inputs(current_child=True)
|
|
docker.existing_child["State"].update(Running=True, Status="running")
|
|
|
|
result = _run(docker, git)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert docker.started == []
|
|
assert docker.removed == []
|
|
assert docker.created == []
|
|
|
|
|
|
def test_created_container_log_request_is_escaped_and_bounded():
|
|
client = object.__new__(bootstrap.DockerSocketClient)
|
|
requests: list[tuple[str, str, int]] = []
|
|
|
|
def request(method: str, path: str, *, max_bytes: int) -> tuple[int, bytes]:
|
|
requests.append((method, path, max_bytes))
|
|
return 200, b""
|
|
|
|
client._request = request # type: ignore[method-assign]
|
|
|
|
assert (
|
|
client.container_logs_for_created_container(
|
|
"abc/def", tail=500, max_bytes=bootstrap.MAX_DOCKER_LOG_BYTES * 2
|
|
)
|
|
== b""
|
|
)
|
|
assert requests == [
|
|
(
|
|
"GET",
|
|
"/containers/abc%2Fdef/logs?stdout=1&stderr=1&tail=40",
|
|
bootstrap.MAX_DOCKER_LOG_BYTES,
|
|
)
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("status", [204, 404, 500])
|
|
def test_created_container_log_request_rejects_non_200_status(status: int):
|
|
client = object.__new__(bootstrap.DockerSocketClient)
|
|
client._request = lambda *_args, **_kwargs: (status, b"") # type: ignore[method-assign]
|
|
|
|
with pytest.raises(bootstrap.BootstrapError, match="docker_logs_failed"):
|
|
client.container_logs_for_created_container(
|
|
"a" * 64, tail=40, max_bytes=bootstrap.MAX_DOCKER_LOG_BYTES
|
|
)
|
|
|
|
|
|
def test_created_container_log_request_keeps_response_size_guard(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
):
|
|
read_sizes: list[int] = []
|
|
|
|
class Response:
|
|
status = 200
|
|
|
|
def read(self, size: int) -> bytes:
|
|
read_sizes.append(size)
|
|
return b"x" * size
|
|
|
|
class Connection:
|
|
def __init__(self, _socket_path: str):
|
|
pass
|
|
|
|
def request(self, *_args: Any, **_kwargs: Any) -> None:
|
|
pass
|
|
|
|
def getresponse(self) -> Response:
|
|
return Response()
|
|
|
|
def close(self) -> None:
|
|
pass
|
|
|
|
monkeypatch.setattr(bootstrap, "_UnixHTTPConnection", Connection)
|
|
client = bootstrap.DockerSocketClient()
|
|
|
|
with pytest.raises(bootstrap.BootstrapError, match="response_too_large"):
|
|
client.container_logs_for_created_container(
|
|
"a" * 64, tail=40, max_bytes=bootstrap.MAX_DOCKER_LOG_BYTES * 2
|
|
)
|
|
|
|
assert read_sizes == [bootstrap.MAX_DOCKER_LOG_BYTES + 1]
|
|
|
|
|
|
def test_finished_child_logs_keep_existing_non_200_empty_behavior():
|
|
client = object.__new__(bootstrap.DockerSocketClient)
|
|
client._request = lambda *_args, **_kwargs: (404, b"") # type: ignore[method-assign]
|
|
|
|
assert client.container_logs("a" * 64, tail=40, max_bytes=8192) == b""
|