from __future__ import annotations import copy from typing import Any import pytest from backend.tests import test_controller_artifact_bootstrap as original from scripts import bootstrap_controller_artifact_promoter as bootstrap FAILED_REVISION = "1" * 40 FAILED_TREE = "2" * 40 FAILED_HELPER_ID = "3" * 64 CURRENT_HELPER_ID = "4" * 64 DRIFTED_HELPER_ID = "9" * 64 ZERO_DOCKER_TIME = "0001-01-01T00:00:00Z" class Git(original.FakeProtectedGit): def __init__(self, helper_source: str): super().__init__(helper_source) def verify_failed_modern_ancestor( self, revision: str, current_revision: str ) -> str: if ( revision != FAILED_REVISION or current_revision != self.revision or not self.retired_tree_valid ): raise bootstrap.BootstrapError( "controller_promoter_failed_helper_not_ancestor" ) return FAILED_TREE def read_blob(self, revision: str, path: str) -> bytes: if revision == FAILED_REVISION: self.blobs_read.append((revision, path)) if path != bootstrap.MODERN_HELPER_SOURCE_PATH: raise bootstrap.BootstrapError( "controller_promoter_source_blob_invalid" ) return self.helper_source.encode("utf-8") return super().read_blob(revision, path) class Docker(original.FakeDocker): def __init__(self, existing_child: dict[str, Any]): super().__init__(original._legacy_helper_spec(), existing_child=existing_child) self.name_reads = 0 self.name_drift_at: int | None = None self.name_drift: str | None = None self.id_reads = 0 self.id_drift_at: int | None = None self.id_drift: str | None = None self.create_attempts: list[tuple[str, dict[str, Any]]] = [] self.log_reads: list[str] = [] self.created_log_bytes = b"" self.created_log_failure = False self.complete_existing_on_start = False self.existing_exit_code = 0 def inspect_container(self, reference: str) -> dict[str, Any] | None: if ( reference == bootstrap.MODERN_HELPER_NAME and self.existing_child is not None ): self.name_reads += 1 snapshot = copy.deepcopy(self.existing_child) if self.name_reads == self.name_drift_at: if self.name_drift == "id": snapshot["Id"] = DRIFTED_HELPER_ID elif self.name_drift == "name": snapshot["Name"] = "/itworx-controller-artifact-promoter-foreign" return snapshot if self.existing_child is not None and reference == self.existing_child.get( "Id" ): self.id_reads += 1 snapshot = copy.deepcopy(self.existing_child) if self.id_reads == self.id_drift_at: if self.id_drift == "id": snapshot["Id"] = DRIFTED_HELPER_ID elif self.id_drift == "name": snapshot["Name"] = "/itworx-controller-artifact-promoter-foreign" return snapshot return super().inspect_container(reference) def create_container(self, name: str, spec: dict[str, Any]) -> str: self.create_attempts.append((name, copy.deepcopy(spec))) return super().create_container(name, spec) def remove_retired_failed_container(self, identifier: str) -> None: assert self.existing_child is not None assert identifier == self.existing_child["Id"] self.removed.append(identifier) self.existing_child = None def start_container(self, identifier: str) -> None: self.started.append(identifier) if self.existing_child is not None and identifier == self.existing_child["Id"]: if self.complete_existing_on_start: self.existing_child["State"].update( Running=False, Status="exited", ExitCode=self.existing_exit_code, ) source_revision = self.existing_child["Config"]["Labels"][ "itworx.source.revision" ] self.containers[bootstrap.CONTROLLER_NAME] = ( original._modern_controller_inspect( source_revision, identifier="e" * 64 ) ) else: self.existing_child["State"].update(Running=True, Status="running") return child = self.containers.get(identifier) if child is None: return child["State"].update(Running=True, Status="running") source_revision = self.created[-1]["Labels"]["itworx.source.revision"] self.containers[bootstrap.CONTROLLER_NAME] = ( original._modern_controller_inspect(source_revision, identifier="e" * 64) ) def container_logs(self, identifier: str, *, tail: int, max_bytes: int) -> bytes: self.log_reads.append(identifier) assert tail <= 40 assert max_bytes <= bootstrap.MAX_DOCKER_LOG_BYTES return self.log_bytes[:max_bytes] def container_logs_for_created_container( self, identifier: str, *, tail: int, max_bytes: int ) -> bytes: if self.created_log_failure: raise bootstrap.BootstrapError("controller_promoter_docker_logs_failed") if self.existing_child is not None and identifier == self.existing_child["Id"]: return self.created_log_bytes return b"" def _created_state() -> dict[str, Any]: return { "Status": "created", "Running": False, "Paused": False, "Restarting": False, "Dead": False, "Pid": 0, "OOMKilled": False, "ExitCode": 0, "StartedAt": ZERO_DOCKER_TIME, "FinishedAt": ZERO_DOCKER_TIME, "Error": "", } def _inputs(*, current_child: bool) -> tuple[Docker, Git, dict[str, Any], str]: helper_source = (original.ROOT / bootstrap.MODERN_HELPER_SOURCE_PATH).read_text( encoding="utf-8" ) child_revision = original.REVISION if current_child else FAILED_REVISION child_id = CURRENT_HELPER_ID if current_child else FAILED_HELPER_ID baseline = bootstrap.MODERN_CONTROLLER_BASELINE spec = bootstrap.render_modern_helper_spec( helper_source, revision=child_revision, image_id=baseline.image_id, current_controller_id=baseline.container_id, current_controller_revision=baseline.revision, current_controller_tree=baseline.tree, current_controller_version=baseline.version, ) child = original._retired_failed_child(helper_source) child["Id"] = child_id child["Image"] = baseline.image_id child["State"] = _created_state() child["RestartCount"] = 0 docker = Docker(child) image_config = docker.images[baseline.image_id]["Config"] image_config.update( { "Env": ["PATH=/usr/bin"], "WorkingDir": "/tmp", "Entrypoint": None, "Shell": ["/bin/bash", "-c"], } ) base_image_labels = bootstrap._inspect_controller_base_image(docker) child["Config"] = { "Hostname": docker.host_name, "Domainname": "", "User": spec["User"], "AttachStdin": False, "AttachStdout": False, "AttachStderr": False, "Tty": False, "OpenStdin": False, "StdinOnce": False, "Cmd": copy.deepcopy(spec["Cmd"]), "Image": spec["Image"], "Labels": {**base_image_labels, **spec["Labels"]}, "Env": list(image_config["Env"]), "Volumes": None, "WorkingDir": image_config["WorkingDir"], "Entrypoint": image_config["Entrypoint"], "OnBuild": None, } child["HostConfig"] = { **copy.deepcopy(bootstrap.FAILED_HELPER_HOST_DEFAULTS), **copy.deepcopy(spec["HostConfig"]), "RestartPolicy": {"Name": "no", "MaximumRetryCount": 0}, } docker.containers[bootstrap.CONTROLLER_NAME] = ( original._modern_controller_source_inspect(baseline.revision) ) return docker, Git(helper_source), child, child_revision def _run( docker: Docker, git: Git, *, health: dict[str, Any] | None = None, health_reader=None, ) -> dict[str, Any]: payload = health or original._modern_baseline_health() return bootstrap.run_bootstrap( bootstrap.SOURCE_REPOSITORY, git.revision, git.tree, docker=docker, protected_git=git, hostname=original.HOST_HOSTNAME, health_reader=health_reader or (lambda: copy.deepcopy(payload)), monotonic=lambda: 0.0, sleeper=lambda _seconds: None, child_timeout_seconds=0.0, ) def test_current_created_child_resumes_after_exact_baseline_and_identity_rechecks(): docker, git, child, _revision = _inputs(current_child=True) result = _run(docker, git) assert result["status"] == "outcome_unknown" assert docker.started == [child["Id"]] assert docker.created == [] assert docker.create_attempts == [] assert docker.removed == [] def test_current_created_child_success_is_reported_as_promoted(): docker, git, child, _revision = _inputs(current_child=True) docker.complete_existing_on_start = True health_reads = 0 def health_reader() -> dict[str, Any]: nonlocal health_reads health_reads += 1 if health_reads <= 3: return original._modern_baseline_health() return original._health(revision=git.revision, tree=git.tree) result = _run(docker, git, health_reader=health_reader) assert result["status"] == "promoted" assert result["revision"] == git.revision assert docker.started == [child["Id"]] assert docker.log_reads == [child["Id"]] assert docker.create_attempts == [] assert docker.created == [] assert docker.removed == [] def test_resumed_child_failure_wins_even_if_health_has_reached_target(): docker, git, child, _revision = _inputs(current_child=True) docker.complete_existing_on_start = True docker.existing_exit_code = 17 health_reads = 0 def health_reader() -> dict[str, Any]: nonlocal health_reads health_reads += 1 if health_reads <= 3: return original._modern_baseline_health() return original._health(revision=git.revision, tree=git.tree) result = _run(docker, git, health_reader=health_reader) assert result["status"] == "failed" assert result["exitCode"] == 17 assert docker.started == [child["Id"]] assert docker.log_reads == [child["Id"]] assert docker.create_attempts == [] assert docker.created == [] def test_noncurrent_created_child_is_retired_without_starting_its_old_revision(): docker, git, child, _revision = _inputs(current_child=False) result = _run(docker, git) assert result["status"] == "outcome_unknown" assert docker.removed == [child["Id"]] assert docker.started == [original.MODERN_HELPER_ID] assert docker.created[0]["Labels"]["itworx.source.revision"] == git.revision assert len(docker.create_attempts) == 1 def test_current_created_name_collision_uses_the_same_recovery_proof(): docker, git, child, _revision = _inputs(current_child=True) docker.existing_child = None collision = copy.deepcopy(child) def collide(_name: str, _spec: dict[str, Any]) -> str: docker.create_attempts.append((_name, copy.deepcopy(_spec))) docker.existing_child = collision raise bootstrap.BootstrapError( "controller_promoter_modern_helper_create_failed" ) docker.create_container = collide result = _run(docker, git) assert result["status"] == "outcome_unknown" assert docker.started == [child["Id"]] assert docker.created == [] assert len(docker.create_attempts) == 1 assert docker.removed == [] @pytest.mark.parametrize( "state", [ {"Status": "created", "Running": False, "ExitCode": 0}, {"Status": "exited", "Running": True, "ExitCode": 0}, {"Status": "exited", "Running": False, "ExitCode": True}, {"Status": "exited", "Running": False, "ExitCode": "0"}, ], ) def test_completed_child_result_checks_terminal_integer_state_before_logs( state: dict[str, Any], ): docker, _git, child, _revision = _inputs(current_child=True) child["State"] = state with pytest.raises(bootstrap.BootstrapError): bootstrap._completed_child_result( child, docker=docker, health_reader=original._health, source_revision=original.REVISION, source_tree=original.TREE, already_applied=True, ) assert docker.log_reads == [] @pytest.mark.parametrize("current_child", [True, False]) @pytest.mark.parametrize( "mutation", [ "status", "exit_code_bool", "exit_code_nonzero", "exit_code_string", "paused", "restarting", "dead", "started_at", "finished_at", "started_at_type", "finished_at_type", "error", "error_type", "pid", "pid_type", "oom_killed", "restart_count", "restart_count_type", ], ) def test_created_child_requires_exact_never_started_state( current_child: bool, mutation: str ): docker, git, _child, _revision = _inputs(current_child=current_child) state = docker.existing_child["State"] if mutation == "status": state["Status"] = "created " elif mutation == "exit_code_bool": state["ExitCode"] = False elif mutation == "exit_code_nonzero": state["ExitCode"] = 1 elif mutation == "exit_code_string": state["ExitCode"] = "0" elif mutation in {"paused", "restarting", "dead"}: state[mutation.capitalize()] = True elif mutation == "started_at": state["StartedAt"] = "2026-10-07T09:00:00Z" elif mutation == "finished_at": state["FinishedAt"] = "2026-10-07T09:00:00Z" elif mutation == "started_at_type": state["StartedAt"] = 0 elif mutation == "finished_at_type": state["FinishedAt"] = 0 elif mutation == "pid": state["Pid"] = 32 elif mutation == "pid_type": state["Pid"] = False elif mutation == "oom_killed": state["OOMKilled"] = True elif mutation == "restart_count": docker.existing_child["RestartCount"] = 1 elif mutation == "restart_count_type": docker.existing_child["RestartCount"] = False elif mutation == "error_type": state["Error"] = None else: state["Error"] = "start failed" try: result = _run(docker, git) except bootstrap.BootstrapError: pass else: assert result["status"] == "outcome_unknown" assert docker.started == [] assert docker.removed == [] assert docker.created == [] @pytest.mark.parametrize("current_child", [True, False]) @pytest.mark.parametrize( "mutation", [ "command", "environment", "label", "mount", "logs", "log_error", "target_baseline", "health_artifact", "main", "controller", "appops", ], ) def test_created_child_profile_or_frozen_baseline_drift_prevents_action( current_child: bool, mutation: str ): docker, git, _child, _revision = _inputs(current_child=current_child) health = original._modern_baseline_health() if mutation == "command": docker.existing_child["Config"]["Cmd"][2] += "\necho changed" elif mutation == "environment": docker.existing_child["Config"]["Env"].append("CUSTOM_VARIABLE=extra") elif mutation == "label": docker.existing_child["Config"]["Labels"]["itworx.unexpected"] = "true" elif mutation == "mount": docker.existing_child["Mounts"][0]["RW"] = not docker.existing_child["Mounts"][ 0 ]["RW"] elif mutation == "logs": docker.created_log_bytes = b"unexpected pre-start output\n" elif mutation == "log_error": docker.created_log_failure = True elif mutation in {"target_baseline", "health_artifact"}: health = original._health() if mutation == "health_artifact": health = original._modern_baseline_health() health["artifact"]["artifactSha256"] = "f" * 64 elif mutation == "main": git.fail_current_main_at = 2 elif mutation == "controller": docker.containers[bootstrap.CONTROLLER_NAME]["Id"] = DRIFTED_HELPER_ID else: docker.containers[original.APPOPS_CONTAINER_ID]["Image"] = "sha256:" + "8" * 64 try: result = _run(docker, git, health=health) except bootstrap.BootstrapError: pass else: assert result["status"] == "outcome_unknown" assert docker.started == [] assert docker.removed == [] assert docker.created == [] @pytest.mark.parametrize("current_child", [True, False]) @pytest.mark.parametrize( ("reference", "read_number", "drift"), [ ("name", 2, "id"), ("name", 2, "name"), ("name", 3, "id"), ("name", 3, "name"), ("id", 1, "id"), ("id", 1, "name"), ("id", 2, "id"), ("id", 2, "name"), ], ) def test_created_child_name_or_id_drift_on_reinspection_prevents_action( current_child: bool, reference: str, read_number: int, drift: str ): docker, git, _child, _revision = _inputs(current_child=current_child) if reference == "name": # The first fixed-name lookup precedes the two proof snapshot pairs. docker.name_drift_at = read_number docker.name_drift = drift else: docker.id_drift_at = read_number docker.id_drift = drift try: result = _run(docker, git) except bootstrap.BootstrapError: pass else: assert result["status"] == "outcome_unknown" assert docker.started == [] assert docker.removed == [] assert docker.created == [] assert docker.create_attempts == [] def test_running_current_helper_remains_unknown_and_is_never_started_again(): docker, git, _child, _revision = _inputs(current_child=True) docker.existing_child["State"].update(Running=True, Status="running") result = _run(docker, git) assert result["status"] == "outcome_unknown" assert docker.started == [] assert docker.removed == [] assert docker.created == [] def test_created_container_log_request_is_escaped_and_bounded(): client = object.__new__(bootstrap.DockerSocketClient) requests: list[tuple[str, str, int]] = [] def request(method: str, path: str, *, max_bytes: int) -> tuple[int, bytes]: requests.append((method, path, max_bytes)) return 200, b"" client._request = request # type: ignore[method-assign] assert ( client.container_logs_for_created_container( "abc/def", tail=500, max_bytes=bootstrap.MAX_DOCKER_LOG_BYTES * 2 ) == b"" ) assert requests == [ ( "GET", "/containers/abc%2Fdef/logs?stdout=1&stderr=1&tail=40", bootstrap.MAX_DOCKER_LOG_BYTES, ) ] @pytest.mark.parametrize("status", [204, 404, 500]) def test_created_container_log_request_rejects_non_200_status(status: int): client = object.__new__(bootstrap.DockerSocketClient) client._request = lambda *_args, **_kwargs: (status, b"") # type: ignore[method-assign] with pytest.raises(bootstrap.BootstrapError, match="docker_logs_failed"): client.container_logs_for_created_container( "a" * 64, tail=40, max_bytes=bootstrap.MAX_DOCKER_LOG_BYTES ) def test_created_container_log_request_keeps_response_size_guard( monkeypatch: pytest.MonkeyPatch, ): read_sizes: list[int] = [] class Response: status = 200 def read(self, size: int) -> bytes: read_sizes.append(size) return b"x" * size class Connection: def __init__(self, _socket_path: str): pass def request(self, *_args: Any, **_kwargs: Any) -> None: pass def getresponse(self) -> Response: return Response() def close(self) -> None: pass monkeypatch.setattr(bootstrap, "_UnixHTTPConnection", Connection) client = bootstrap.DockerSocketClient() with pytest.raises(bootstrap.BootstrapError, match="response_too_large"): client.container_logs_for_created_container( "a" * 64, tail=40, max_bytes=bootstrap.MAX_DOCKER_LOG_BYTES * 2 ) assert read_sizes == [bootstrap.MAX_DOCKER_LOG_BYTES + 1] def test_finished_child_logs_keep_existing_non_200_empty_behavior(): client = object.__new__(bootstrap.DockerSocketClient) client._request = lambda *_args, **_kwargs: (404, b"") # type: ignore[method-assign] assert client.container_logs("a" * 64, tail=40, max_bytes=8192) == b""