355 lines
12 KiB
Python
355 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
from types import SimpleNamespace
|
|
|
|
import httpx
|
|
import pytest
|
|
from sqlmodel import Session, SQLModel, create_engine
|
|
|
|
DIAGNOSTIC = (
|
|
"APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC "
|
|
"code=helper_failed phase=source_archive exitCode=19"
|
|
)
|
|
SAFE_DIAGNOSTICS = {"code": "helper_failed", "phase": "source_archive", "exitCode": 19}
|
|
SECRET = "must-never-escape-helper-log"
|
|
|
|
|
|
def _docker_frame(stream_type, payload):
|
|
return bytes([stream_type, 0, 0, 0]) + len(payload).to_bytes(4, "big") + payload
|
|
|
|
|
|
def _default_docker_logs():
|
|
marker = DIAGNOSTIC.encode()
|
|
split_at = len(marker) // 2
|
|
return b"".join(
|
|
(
|
|
_docker_frame(1, b"helper started\n"),
|
|
_docker_frame(2, f"token={SECRET}\n".encode() + marker[:split_at]),
|
|
_docker_frame(2, marker[split_at:] + b"\n"),
|
|
)
|
|
)
|
|
|
|
|
|
class _Response:
|
|
def __init__(self, payload=None):
|
|
self.payload = payload or {}
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
|
|
class _Portainer:
|
|
def __init__(self, *, exit_code=19, logs=None, running=False):
|
|
self.exit_code = exit_code
|
|
self.logs = logs if logs is not None else _default_docker_logs()
|
|
self.running = running
|
|
self.requests = []
|
|
self.request_kwargs = []
|
|
|
|
def _config(self):
|
|
return SimpleNamespace(endpoint_id="1")
|
|
|
|
def list_containers(self):
|
|
return {"containers": []}
|
|
|
|
def _request(self, method, path, **kwargs):
|
|
self.requests.append((method, path))
|
|
self.request_kwargs.append(kwargs)
|
|
if method == "POST" and path.endswith("/containers/create"):
|
|
return _Response({"Id": "helper-123"})
|
|
if method == "GET" and path.endswith("/helper-123/json"):
|
|
return _Response(
|
|
{"State": {"Running": self.running, "ExitCode": self.exit_code}}
|
|
)
|
|
if method == "GET" and path.endswith("/helper-123/logs"):
|
|
return httpx.Response(status_code=200, content=self.logs)
|
|
return _Response()
|
|
|
|
def get_container_logs(self, container_name, tail=100):
|
|
return {"status": "ok", "container": container_name, "logs": self.logs}
|
|
|
|
|
|
def _session(tmp_path):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'controller-diagnostics.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
return Session(engine)
|
|
|
|
|
|
def test_helper_spec_emits_only_fixed_failure_markers():
|
|
from backend.app.services.controller_artifact_helper_spec import (
|
|
helper_container_spec,
|
|
)
|
|
|
|
command = helper_container_spec(revision="a" * 40, image_id="sha256:" + "b" * 64)["Cmd"][2]
|
|
assert "APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC" in command
|
|
assert "phase=source_commit" in command
|
|
assert "phase=source_archive" in command
|
|
assert "phase=upgrade" in command
|
|
assert "phase=cleanup" in command
|
|
assert "exitCode=" in command
|
|
|
|
|
|
def test_generated_helper_command_parses_without_evaluating_command_substitutions():
|
|
from backend.app.services.controller_artifact_helper_spec import (
|
|
helper_container_spec,
|
|
)
|
|
|
|
bash = shutil.which("bash")
|
|
if bash is None:
|
|
pytest.skip("bash is required to parse the generated helper command")
|
|
command = helper_container_spec(
|
|
revision="a" * 40,
|
|
image_id="sha256:" + "b" * 64,
|
|
)["Cmd"][2]
|
|
result = subprocess.run(
|
|
[bash, "-n"],
|
|
input=command,
|
|
capture_output=True,
|
|
text=True,
|
|
shell=False,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def test_failed_workspace_validation_does_not_run_cleanup_or_remove_unvalidated_path():
|
|
from backend.app.services.controller_artifact_helper_spec import (
|
|
helper_container_spec,
|
|
)
|
|
|
|
bash = shutil.which("bash")
|
|
if bash is None:
|
|
pytest.skip("bash is required to exercise the generated helper command")
|
|
command = helper_container_spec(
|
|
revision="a" * 40,
|
|
image_id="sha256:" + "b" * 64,
|
|
)["Cmd"][2]
|
|
shell_stubs = """\
|
|
stat() {
|
|
case "$*" in
|
|
*"%a:%u:%g"*) printf '1777:0:0\\n' ;;
|
|
*) printf '999:700\\n' ;;
|
|
esac
|
|
}
|
|
mktemp() { printf '/tmp\\n'; }
|
|
rm() { printf 'RM_WOULD_RUN\\n'; }
|
|
"""
|
|
result = subprocess.run(
|
|
[bash, "-s"],
|
|
input=shell_stubs + command,
|
|
capture_output=True,
|
|
text=True,
|
|
shell=False,
|
|
)
|
|
assert result.returncode != 0
|
|
assert "phase=workspace_validate" in result.stderr
|
|
assert "RM_WOULD_RUN" not in result.stdout + result.stderr
|
|
|
|
|
|
def test_diagnostic_parser_accepts_fixed_marker_and_returns_no_log_text():
|
|
from backend.app.services import controller_artifact_promoter as promoter
|
|
|
|
parser = getattr(promoter, "parse_helper_diagnostics", None)
|
|
assert callable(parser), "the helper diagnostic parser is not implemented"
|
|
assert parser(f"safe status line\n{DIAGNOSTIC}", 19) == SAFE_DIAGNOSTICS
|
|
|
|
|
|
def test_docker_log_demux_reads_stdout_stderr_and_markers_across_frames():
|
|
from backend.app.services import controller_artifact_promoter as promoter
|
|
|
|
demux = getattr(promoter, "_decode_docker_log_stream", None)
|
|
assert callable(demux), "the bounded Docker stdcopy log demux is not implemented"
|
|
marker = DIAGNOSTIC.encode()
|
|
split_at = len(marker) // 2
|
|
framed = b"".join(
|
|
(
|
|
_docker_frame(1, b"stdout line\n"),
|
|
_docker_frame(2, b"stderr prefix\n" + marker[:split_at]),
|
|
_docker_frame(2, marker[split_at:] + b"\n"),
|
|
)
|
|
)
|
|
assert demux(framed) == f"stdout line\nstderr prefix\n{DIAGNOSTIC}\n"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"framed",
|
|
[
|
|
b"",
|
|
_docker_frame(3, b"unknown stream"),
|
|
bytes([2, 1, 0, 0]) + (1).to_bytes(4, "big") + b"x",
|
|
bytes([2, 0, 0, 0]),
|
|
_docker_frame(2, b"truncated")[:-1],
|
|
_docker_frame(2, b"\xff"),
|
|
bytes([2, 0, 0, 0]) + (4001).to_bytes(4, "big"),
|
|
b"".join(_docker_frame(1, b"x") for _ in range(201)),
|
|
],
|
|
)
|
|
def test_docker_log_demux_rejects_invalid_truncated_or_oversized_streams(framed):
|
|
from backend.app.services import controller_artifact_promoter as promoter
|
|
|
|
demux = getattr(promoter, "_decode_docker_log_stream", None)
|
|
assert callable(demux), "the bounded Docker stdcopy log demux is not implemented"
|
|
assert demux(framed) is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("log_text", "exit_code"),
|
|
[
|
|
("APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC code=helper_failed phase=unlisted exitCode=19", 19),
|
|
("APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC code=helper_failed phase=source_archive exitCode=19 extra=secret", 19),
|
|
("APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC code=helper_failed phase=source_archive exitCode=256", 19),
|
|
(DIAGNOSTIC, 256),
|
|
(DIAGNOSTIC + "\n" + DIAGNOSTIC, 19),
|
|
("x" * 4001 + "\n" + DIAGNOSTIC, 19),
|
|
(DIAGNOSTIC, True),
|
|
],
|
|
)
|
|
def test_diagnostic_parser_rejects_malformed_oversized_or_adversarial_input(log_text, exit_code):
|
|
from backend.app.services import controller_artifact_promoter as promoter
|
|
|
|
parser = getattr(promoter, "parse_helper_diagnostics", None)
|
|
assert callable(parser), "the helper diagnostic parser is not implemented"
|
|
expected_exit = exit_code if type(exit_code) is int and 0 <= exit_code <= 255 else None
|
|
assert parser(log_text, exit_code) == {
|
|
"code": "helper_failed",
|
|
"phase": "unknown",
|
|
"exitCode": expected_exit,
|
|
}
|
|
|
|
|
|
def test_helper_failure_diagnostics_reach_job_audit_and_operation_result_without_logs(tmp_path, monkeypatch):
|
|
from backend.app.db.models import AuditEvent, JobRun
|
|
from backend.app.services import action_execution
|
|
from backend.app.services.controller_artifact_promoter import (
|
|
ControllerArtifactPromoter,
|
|
)
|
|
|
|
portainer = _Portainer()
|
|
promoted = ControllerArtifactPromoter(portainer=portainer)._run_helper(
|
|
revision="a" * 40,
|
|
image_id="sha256:" + "b" * 64,
|
|
)
|
|
assert promoted["status"] == "failed"
|
|
assert promoted["diagnostics"] == SAFE_DIAGNOSTICS
|
|
assert "logTail" not in promoted
|
|
assert any(
|
|
method == "GET" and path.endswith("/helper-123/logs")
|
|
for method, path in portainer.requests
|
|
)
|
|
log_request_index = next(
|
|
index
|
|
for index, (method, path) in enumerate(portainer.requests)
|
|
if method == "GET" and path.endswith("/helper-123/logs")
|
|
)
|
|
assert portainer.request_kwargs[log_request_index]["params"] == {
|
|
"stdout": 1,
|
|
"stderr": 1,
|
|
"tail": 200,
|
|
}
|
|
assert SECRET not in json.dumps(promoted)
|
|
|
|
monkeypatch.setattr(ControllerArtifactPromoter, "run", lambda self: promoted)
|
|
with _session(tmp_path) as session:
|
|
result = action_execution._execute_controller_artifact_promote(
|
|
session,
|
|
source="operator",
|
|
risk="high",
|
|
reason="diagnose the failed promotion",
|
|
original_policy_decision="approval_required",
|
|
)
|
|
job = session.get(JobRun, result["jobId"])
|
|
audit = session.get(AuditEvent, result["auditId"])
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["diagnostics"] == SAFE_DIAGNOSTICS
|
|
assert job is not None and "phase=source_archive" in job.result_summary
|
|
assert audit is not None
|
|
assert audit.after_state["diagnostics"] == SAFE_DIAGNOSTICS
|
|
assert audit.redacted_metadata["diagnostics"] == SAFE_DIAGNOSTICS
|
|
durable = json.dumps(
|
|
{
|
|
"operation": result,
|
|
"jobSummary": job.result_summary,
|
|
"auditAfter": audit.after_state,
|
|
"auditMetadata": audit.redacted_metadata,
|
|
}
|
|
)
|
|
assert SECRET not in durable
|
|
assert "logTail" not in durable
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"logs",
|
|
[
|
|
b"",
|
|
_docker_frame(
|
|
2,
|
|
b"APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC code=helper_failed phase=unknown exitCode=19",
|
|
),
|
|
_docker_frame(
|
|
2,
|
|
b"APPOPS_CONTROLLER_PROMOTION_DIAGNOSTIC code=helper_failed phase=source_archive exitCode=19 extra=value",
|
|
),
|
|
_docker_frame(2, b"x" * 4001),
|
|
_docker_frame(2, b"truncated")[:-1],
|
|
],
|
|
)
|
|
def test_missing_or_unparseable_helper_phase_is_explicitly_unknown(logs):
|
|
from backend.app.services.controller_artifact_promoter import (
|
|
ControllerArtifactPromoter,
|
|
)
|
|
|
|
result = ControllerArtifactPromoter(portainer=_Portainer(exit_code=19, logs=logs))._run_helper(
|
|
revision="a" * 40,
|
|
image_id="sha256:" + "b" * 64,
|
|
)
|
|
assert result["status"] == "failed"
|
|
assert result["diagnostics"] == {
|
|
"code": "helper_failed",
|
|
"phase": "unknown",
|
|
"exitCode": 19,
|
|
}
|
|
assert "logTail" not in result
|
|
assert SECRET not in json.dumps(result)
|
|
|
|
|
|
def test_successful_helper_remains_promoted_and_never_returns_logs():
|
|
from backend.app.services.controller_artifact_promoter import (
|
|
ControllerArtifactPromoter,
|
|
)
|
|
|
|
portainer = _Portainer(exit_code=0)
|
|
portainer.get_container_logs = lambda *args, **kwargs: pytest.fail("success must not fetch helper logs")
|
|
result = ControllerArtifactPromoter(portainer=portainer)._run_helper(
|
|
revision="a" * 40,
|
|
image_id="sha256:" + "b" * 64,
|
|
)
|
|
assert result == {"status": "promoted", "revision": "a" * 40}
|
|
assert not any(path.endswith("/logs") for _, path in portainer.requests)
|
|
|
|
|
|
def test_helper_timeout_remains_outcome_unknown_without_cleanup_or_log_fetch(monkeypatch):
|
|
from backend.app.services import controller_artifact_promoter as promoter
|
|
|
|
portainer = _Portainer(running=True)
|
|
portainer.get_container_logs = lambda *args, **kwargs: pytest.fail(
|
|
"a running helper has no terminal diagnostic to read"
|
|
)
|
|
clock_values = iter((100.0, 341.0))
|
|
monkeypatch.setattr(promoter.time, "monotonic", lambda: next(clock_values))
|
|
|
|
result = promoter.ControllerArtifactPromoter(portainer=portainer)._run_helper(
|
|
revision="a" * 40,
|
|
image_id="sha256:" + "b" * 64,
|
|
)
|
|
assert result == {
|
|
"status": "outcome_unknown",
|
|
"revision": "a" * 40,
|
|
"reason": "controller_promoter_timeout",
|
|
"helperId": "helper-123",
|
|
}
|
|
assert not any(method == "DELETE" for method, _ in portainer.requests)
|
|
assert not any(path.endswith("/logs") for _, path in portainer.requests)
|