552 lines
18 KiB
Python
552 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import os
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
from scripts import bootstrap_controller_artifact_promoter as bootstrap
|
|
from backend.tests import test_controller_artifact_bootstrap as original
|
|
|
|
FAILED_REVISION = "1" * 40
|
|
FAILED_TREE = "2" * 40
|
|
FAILED_ID = "3" * 64
|
|
|
|
|
|
class Git(original.FakeProtectedGit):
|
|
def verify_failed_modern_ancestor(self, revision, current_revision):
|
|
if (
|
|
revision != FAILED_REVISION
|
|
or current_revision != self.revision
|
|
or not self.retired_tree_valid
|
|
):
|
|
raise bootstrap.BootstrapError(
|
|
"controller_promoter_failed_helper_not_ancestor"
|
|
)
|
|
return FAILED_TREE
|
|
|
|
def read_blob(self, revision, path):
|
|
if revision == FAILED_REVISION:
|
|
self.blobs_read.append((revision, path))
|
|
assert path == bootstrap.MODERN_HELPER_SOURCE_PATH
|
|
return self.helper_source.encode()
|
|
return super().read_blob(revision, path)
|
|
|
|
|
|
class Docker(original.FakeDocker):
|
|
def inspect_container(self, reference):
|
|
if reference == FAILED_ID:
|
|
return self.existing_child
|
|
return super().inspect_container(reference)
|
|
|
|
def remove_retired_failed_container(self, identifier):
|
|
assert identifier == FAILED_ID
|
|
self.removed.append(identifier)
|
|
self.existing_child = None
|
|
|
|
|
|
def inputs():
|
|
source = (original.ROOT / bootstrap.MODERN_HELPER_SOURCE_PATH).read_text()
|
|
child = original._retired_failed_child(source)
|
|
baseline = bootstrap.MODERN_CONTROLLER_BASELINE
|
|
spec = bootstrap.render_modern_helper_spec(
|
|
source,
|
|
revision=FAILED_REVISION,
|
|
image_id=baseline.image_id,
|
|
current_controller_id=baseline.container_id,
|
|
current_controller_revision=baseline.revision,
|
|
current_controller_tree=baseline.tree,
|
|
current_controller_version=baseline.version,
|
|
)
|
|
child["Id"] = FAILED_ID
|
|
child["Config"].update(
|
|
{
|
|
"Hostname": original.HOST_HOSTNAME,
|
|
"Cmd": spec["Cmd"],
|
|
"Labels": {**original._controller_labels(), **spec["Labels"]},
|
|
"Env": ["PATH=/usr/bin"],
|
|
"WorkingDir": "/tmp",
|
|
"Entrypoint": None,
|
|
"Domainname": "",
|
|
"AttachStdin": False,
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"OpenStdin": False,
|
|
"StdinOnce": False,
|
|
"OnBuild": None,
|
|
}
|
|
)
|
|
child["State"].update({"Paused": False, "Restarting": False, "Dead": False})
|
|
child["HostConfig"].update(copy.deepcopy(bootstrap.FAILED_HELPER_HOST_DEFAULTS))
|
|
child["HostConfig"]["RestartPolicy"] = {"Name": "no", "MaximumRetryCount": 0}
|
|
docker = Docker(original._legacy_helper_spec(), existing_child=child)
|
|
docker.containers[bootstrap.CONTROLLER_NAME] = (
|
|
original._modern_controller_source_inspect(baseline.revision)
|
|
)
|
|
docker.images[baseline.image_id]["Config"].update(
|
|
{
|
|
"Env": ["PATH=/usr/bin"],
|
|
"WorkingDir": "/tmp",
|
|
"Entrypoint": None,
|
|
"Shell": ["/bin/bash", "-c"],
|
|
}
|
|
)
|
|
return docker, Git(source), child
|
|
|
|
|
|
def run(docker, git):
|
|
return original._run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[
|
|
original._modern_baseline_health(),
|
|
original._modern_baseline_health(),
|
|
original._modern_baseline_health(),
|
|
original._modern_baseline_health(),
|
|
original._health(),
|
|
original._health(),
|
|
],
|
|
)
|
|
|
|
|
|
_MOUNT_ORDERS = {
|
|
("name", 1): (3, 0, 4, 1, 2),
|
|
("name", 2): (4, 2, 0, 3, 1),
|
|
("name", 3): (1, 3, 0, 4, 2),
|
|
("id", 1): (2, 4, 1, 0, 3),
|
|
("id", 2): (4, 1, 3, 2, 0),
|
|
}
|
|
|
|
|
|
def configure_reinspection(
|
|
docker,
|
|
*,
|
|
drift=None,
|
|
drift_reference="id",
|
|
drift_pair=1,
|
|
identifier=FAILED_ID,
|
|
):
|
|
"""Model Docker returning the same inspect with independent mount ordering."""
|
|
original_inspect = docker.inspect_container
|
|
counts = {"name": 0, "id": 0}
|
|
reads = []
|
|
|
|
def inspect(reference):
|
|
if reference == bootstrap.MODERN_HELPER_NAME:
|
|
scope = "name"
|
|
elif reference == identifier:
|
|
scope = "id"
|
|
else:
|
|
return original_inspect(reference)
|
|
|
|
counts[scope] += 1
|
|
reads.append((scope, counts[scope]))
|
|
if docker.existing_child is None:
|
|
return None
|
|
|
|
snapshot = copy.deepcopy(docker.existing_child)
|
|
order = _MOUNT_ORDERS.get((scope, counts[scope]))
|
|
if order is not None:
|
|
snapshot["Mounts"] = [snapshot["Mounts"][index] for index in order]
|
|
|
|
pair_number = (
|
|
1
|
|
if (scope == "name" and counts[scope] == 2)
|
|
or (scope == "id" and counts[scope] == 1)
|
|
else 2
|
|
if (scope == "name" and counts[scope] == 3)
|
|
or (scope == "id" and counts[scope] == 2)
|
|
else None
|
|
)
|
|
is_drift_read = pair_number == drift_pair and (
|
|
drift_reference == "both" or scope == drift_reference
|
|
)
|
|
if drift is not None and is_drift_read:
|
|
mount = snapshot["Mounts"][0]
|
|
if drift == "mount_type":
|
|
mount["Type"] = "volume"
|
|
elif drift == "mount_source":
|
|
mount["Source"] = "/tmp/changed-source"
|
|
elif drift == "mount_destination":
|
|
mount["Destination"] = "/tmp/changed-destination"
|
|
elif drift == "mount_rw":
|
|
mount["RW"] = not mount["RW"]
|
|
elif drift == "mount_propagation":
|
|
mount["Propagation"] = "shared"
|
|
elif drift == "mount_name":
|
|
mount["Name"] = "unexpected-volume"
|
|
elif drift == "missing_mount":
|
|
snapshot["Mounts"].pop()
|
|
elif drift == "extra_mount":
|
|
extra = copy.deepcopy(mount)
|
|
extra["Destination"] = "/unexpected"
|
|
snapshot["Mounts"].append(extra)
|
|
elif drift == "duplicate_destination":
|
|
snapshot["Mounts"].append(copy.deepcopy(mount))
|
|
elif drift == "metadata_field":
|
|
cache_mount = next(
|
|
item
|
|
for item in snapshot["Mounts"]
|
|
if item["Destination"] == "/mnt/cache"
|
|
)
|
|
cache_mount["Mode"] = "ro"
|
|
elif drift == "environment_order":
|
|
snapshot["Config"]["Env"] = list(reversed(snapshot["Config"]["Env"]))
|
|
elif drift == "binds_order":
|
|
snapshot["HostConfig"]["Binds"] = list(
|
|
reversed(snapshot["HostConfig"]["Binds"])
|
|
)
|
|
elif drift == "state_drift":
|
|
snapshot["State"]["ExitCode"] = 2
|
|
else:
|
|
raise AssertionError(f"unhandled test drift: {drift}")
|
|
return snapshot
|
|
|
|
docker.inspect_container = inspect
|
|
return counts, reads
|
|
|
|
|
|
def historical_inputs():
|
|
helper_source = (
|
|
original.ROOT / "backend/app/services/controller_artifact_helper_spec.py"
|
|
).read_text(encoding="utf-8")
|
|
child = original._retired_failed_child(helper_source)
|
|
docker = original.FakeDocker(
|
|
original._legacy_helper_spec(), existing_child=child
|
|
)
|
|
docker.containers["gitea-deploy-control"] = original._modern_controller_source_inspect(
|
|
bootstrap.MODERN_CONTROLLER_REVISION
|
|
)
|
|
git = original.FakeProtectedGit(helper_source)
|
|
return docker, git
|
|
|
|
|
|
def run_historical(docker, git):
|
|
return original._run_bootstrap(
|
|
docker,
|
|
git,
|
|
health_sequence=[
|
|
original._modern_baseline_health(),
|
|
original._modern_baseline_health(),
|
|
original._modern_baseline_health(),
|
|
original._modern_baseline_health(),
|
|
original._health(),
|
|
original._health(),
|
|
],
|
|
)
|
|
|
|
|
|
def test_failed_child_inspect_ignores_mount_order_across_name_and_id_reads():
|
|
docker, git, _child = inputs()
|
|
counts, reads = configure_reinspection(docker)
|
|
|
|
result = run(docker, git)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert counts == {"name": 4, "id": 2}
|
|
assert reads[:5] == [
|
|
("name", 1),
|
|
("name", 2),
|
|
("id", 1),
|
|
("name", 3),
|
|
("id", 2),
|
|
]
|
|
assert docker.removed == [FAILED_ID]
|
|
assert len(docker.created) == len(docker.started) == 1
|
|
|
|
|
|
def test_historical_failed_child_mount_order_is_ignored_across_name_and_id_reads():
|
|
docker, git = historical_inputs()
|
|
counts, reads = configure_reinspection(
|
|
docker, identifier=original.RETIRED_HELPER_ID
|
|
)
|
|
|
|
result = run_historical(docker, git)
|
|
|
|
assert result["status"] == "promoted"
|
|
assert counts == {"name": 4, "id": 2}
|
|
assert reads[:5] == [
|
|
("name", 1),
|
|
("name", 2),
|
|
("id", 1),
|
|
("name", 3),
|
|
("id", 2),
|
|
]
|
|
assert docker.removed == [original.RETIRED_HELPER_ID]
|
|
assert len(docker.created) == len(docker.started) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("drift_pair", [1, 2])
|
|
def test_historical_failed_child_rejects_mount_metadata_change_from_initial_snapshot(
|
|
drift_pair,
|
|
):
|
|
docker, git = historical_inputs()
|
|
configure_reinspection(
|
|
docker,
|
|
drift="metadata_field",
|
|
drift_reference="both",
|
|
drift_pair=drift_pair,
|
|
identifier=original.RETIRED_HELPER_ID,
|
|
)
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run_historical(docker, git)
|
|
|
|
assert docker.removed == docker.created == docker.started == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"drift",
|
|
[
|
|
"mount_type",
|
|
"mount_source",
|
|
"mount_destination",
|
|
"mount_rw",
|
|
"mount_propagation",
|
|
"mount_name",
|
|
"missing_mount",
|
|
"extra_mount",
|
|
"duplicate_destination",
|
|
"metadata_field",
|
|
"environment_order",
|
|
"binds_order",
|
|
"state_drift",
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("drift_reference", ["name", "id"])
|
|
def test_failed_child_reinspection_rejects_any_non_mount_order_drift(
|
|
drift, drift_reference
|
|
):
|
|
docker, git, _child = inputs()
|
|
# Make the environment order check meaningful while keeping the initial
|
|
# image and child Config.Env identical.
|
|
docker.existing_child["Config"]["Env"] = ["PATH=/usr/bin", "LANG=C"]
|
|
docker.images[bootstrap.LEGACY_CONTROLLER_IMAGE_ID]["Config"]["Env"] = [
|
|
"PATH=/usr/bin",
|
|
"LANG=C",
|
|
]
|
|
_counts, _reads = configure_reinspection(
|
|
docker, drift=drift, drift_reference=drift_reference
|
|
)
|
|
|
|
if drift == "binds_order":
|
|
assert len(docker.existing_child["HostConfig"]["Binds"]) > 1
|
|
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run(docker, git)
|
|
|
|
assert docker.removed == docker.created == docker.started == []
|
|
|
|
|
|
def test_noncurrent_protected_ancestor_failure_releases_only_its_verified_name():
|
|
docker, git, child = inputs()
|
|
result = run(docker, git)
|
|
assert result["status"] == "promoted"
|
|
assert docker.removed == [FAILED_ID]
|
|
assert (
|
|
git.blobs_read.count((FAILED_REVISION, bootstrap.MODERN_HELPER_SOURCE_PATH))
|
|
== 1
|
|
)
|
|
assert (
|
|
docker.created[0]["Labels"]["itworx.controller.bootstrap.target.id"]
|
|
== bootstrap.MODERN_CONTROLLER_ID
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
[
|
|
"running",
|
|
"succeeded",
|
|
"bool_exit",
|
|
"paused",
|
|
"dead",
|
|
"restarting",
|
|
"source",
|
|
"target",
|
|
"command",
|
|
"entrypoint",
|
|
"environment",
|
|
"mount",
|
|
"host_capability",
|
|
"controller",
|
|
"appops",
|
|
"ancestor",
|
|
],
|
|
)
|
|
def test_noncurrent_failed_child_recovery_refuses_unattested_identity_or_baseline(
|
|
change,
|
|
):
|
|
docker, git, child = inputs()
|
|
if change == "running":
|
|
child["State"].update(Running=True, Status="running")
|
|
elif change == "succeeded":
|
|
child["State"]["ExitCode"] = 0
|
|
elif change == "bool_exit":
|
|
child["State"]["ExitCode"] = True
|
|
elif change in {"paused", "dead", "restarting"}:
|
|
child["State"][change.capitalize()] = True
|
|
elif change == "source":
|
|
child["Config"]["Labels"]["itworx.source.repository"] = "other/repository"
|
|
elif change == "target":
|
|
child["Config"]["Labels"]["itworx.controller.bootstrap.target.id"] = "9" * 64
|
|
elif change == "command":
|
|
child["Config"]["Cmd"][2] += "\necho changed"
|
|
elif change == "entrypoint":
|
|
child["Config"]["Entrypoint"] = ["/bin/sh"]
|
|
elif change == "environment":
|
|
child["Config"]["Env"].append("CUSTOM_VARIABLE=extra")
|
|
elif change == "mount":
|
|
child["Mounts"][2]["RW"] = True
|
|
elif change == "host_capability":
|
|
child["HostConfig"]["CapAdd"] = ["SYS_ADMIN"]
|
|
elif change == "controller":
|
|
docker.containers[bootstrap.CONTROLLER_NAME]["Id"] = "9" * 64
|
|
elif change == "appops":
|
|
docker.containers[original.APPOPS_CONTAINER_ID]["Image"] = "sha256:" + "9" * 64
|
|
else:
|
|
git.retired_tree_valid = False
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run(docker, git)
|
|
assert docker.removed == docker.created == docker.started == []
|
|
|
|
|
|
def test_noncurrent_child_identity_drift_on_final_named_reread_refuses_delete():
|
|
docker, git, child = inputs()
|
|
docker.named_inspect_override = copy.deepcopy(child)
|
|
docker.named_inspect_override["Id"] = "9" * 64
|
|
docker.named_inspect_override_at = 3
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run(docker, git)
|
|
assert not docker.removed
|
|
|
|
|
|
def test_dynamic_failed_child_delete_retains_nonforce_and_no_volume_policy(monkeypatch):
|
|
client = bootstrap.DockerSocketClient()
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
client,
|
|
"_request",
|
|
lambda method, path, **kw: (calls.append((method, path)) or (204, b"")),
|
|
)
|
|
client.remove_retired_failed_container(FAILED_ID)
|
|
assert calls == [("DELETE", f"/containers/{FAILED_ID}?force=false&v=false")]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
["cpu", "dns", "masked_paths", "logging", "boolean_default", "unknown_option"],
|
|
)
|
|
def test_failed_child_requires_complete_known_host_defaults(change):
|
|
docker, git, child = inputs()
|
|
if change == "cpu":
|
|
child["HostConfig"]["CpuShares"] = 100
|
|
elif change == "dns":
|
|
child["HostConfig"]["Dns"] = ["192.0.2.1"]
|
|
elif change == "masked_paths":
|
|
child["HostConfig"]["MaskedPaths"] = []
|
|
elif change == "logging":
|
|
child["HostConfig"]["LogConfig"]["Type"] = "none"
|
|
elif change == "boolean_default":
|
|
child["HostConfig"]["CpuShares"] = False
|
|
else:
|
|
child["HostConfig"]["UnreviewedOption"] = None
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run(docker, git)
|
|
assert not docker.removed
|
|
|
|
|
|
def test_raw_docker_api_nil_mount_omission_matches_cli_null():
|
|
docker, git, child = inputs()
|
|
child["HostConfig"].pop("Mounts")
|
|
assert run(docker, git)["status"] == "promoted"
|
|
|
|
|
|
def test_appops_drift_on_final_baseline_reread_refuses_cleanup():
|
|
docker, git, child = inputs()
|
|
docker.appops_inspect_override = copy.deepcopy(
|
|
docker.containers[original.APPOPS_CONTAINER_ID]
|
|
)
|
|
docker.appops_inspect_override["State"]["Running"] = False
|
|
docker.appops_inspect_override_at = 3
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run(docker, git)
|
|
assert not docker.removed
|
|
|
|
|
|
def test_protected_main_drift_immediately_before_delete_refuses_cleanup():
|
|
docker, git, child = inputs()
|
|
git.fail_current_main_at = 5
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
run(docker, git)
|
|
assert not docker.removed
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"identifier", ["", "short", "A" * 64, "3" * 63, "../containers/other"]
|
|
)
|
|
def test_delete_adapter_rejects_noncanonical_full_container_identity(
|
|
monkeypatch, identifier
|
|
):
|
|
client = bootstrap.DockerSocketClient()
|
|
monkeypatch.setattr(
|
|
client,
|
|
"_request",
|
|
lambda *_args, **_kw: pytest.fail("unexpected Docker mutation"),
|
|
)
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
client.remove_retired_failed_container(identifier)
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", ["ancestor", "unrelated", "replace_ref"])
|
|
def test_git_ancestor_proof_reads_original_protected_objects(
|
|
tmp_path, monkeypatch, scenario
|
|
):
|
|
def git(*args, allow_replace_objects=False):
|
|
environment = None
|
|
if allow_replace_objects:
|
|
# Only the attack demonstration opts out of the inherited guard.
|
|
# The production proof still reads original protected Git objects.
|
|
environment = dict(os.environ)
|
|
environment.pop("GIT_NO_REPLACE_OBJECTS", None)
|
|
return (
|
|
subprocess.run(
|
|
["git", "-C", str(tmp_path), *args],
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=environment,
|
|
)
|
|
.stdout.decode()
|
|
.strip()
|
|
)
|
|
|
|
git("init", "-q", "-b", "main")
|
|
git("config", "user.name", "Test")
|
|
git("config", "user.email", "test@example.invalid")
|
|
path = tmp_path / "tracked"
|
|
path.write_text("protected ancestor")
|
|
git("add", ".")
|
|
git("commit", "-qm", "ancestor")
|
|
old = git("rev-parse", "HEAD")
|
|
tree = git("rev-parse", "HEAD^{tree}")
|
|
path.write_text("new protected main")
|
|
git("commit", "-qam", "main")
|
|
current = git("rev-parse", "HEAD")
|
|
monkeypatch.setattr(bootstrap, "PROTECTED_REPO", str(tmp_path / ".git"))
|
|
proof = bootstrap.ProtectedGitRepository()
|
|
if scenario == "replace_ref":
|
|
git("replace", old, current)
|
|
assert git("rev-parse", f"{old}^{{tree}}", allow_replace_objects=True) != tree
|
|
if scenario == "unrelated":
|
|
git("checkout", "-qb", "side", old)
|
|
path.write_text("unrelated child")
|
|
git("commit", "-qam", "side")
|
|
old = git("rev-parse", "HEAD")
|
|
with pytest.raises(bootstrap.BootstrapError):
|
|
proof.verify_failed_modern_ancestor(old, current)
|
|
else:
|
|
assert proof.verify_failed_modern_ancestor(old, current) == tree
|