491 lines
17 KiB
Python
491 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
BROKER = ROOT / "ops" / "gitea-deploy-broker"
|
|
if str(BROKER) not in sys.path:
|
|
sys.path.insert(0, str(BROKER))
|
|
|
|
|
|
@pytest.fixture
|
|
def credentials_module():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"forge_database_credentials_test",
|
|
BROKER / "forge_database_credentials.py",
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _secrets(module):
|
|
return {
|
|
module.RUNTIME_ROLE: "runtime-secret-SENTINEL-0123456789abcdef",
|
|
module.MIGRATOR_ROLE: "migrator-secret-SENTINEL-fedcba9876543210",
|
|
}
|
|
|
|
|
|
def _uri(role: str, password: str) -> bytes:
|
|
return f"postgresql://{role}:{password}@postgres:5432/forge".encode("ascii")
|
|
|
|
|
|
def _install_fake_docker(
|
|
module,
|
|
tmp_path: Path,
|
|
secrets: dict[str, str],
|
|
*,
|
|
behavior: str = "success",
|
|
):
|
|
marker = tmp_path / "fake-docker-state.txt"
|
|
script = tmp_path / "fake-docker"
|
|
secret_literals = json.dumps(secrets)
|
|
marker_literal = json.dumps(str(marker))
|
|
script.write_text(
|
|
f"#!{sys.executable}\n"
|
|
"import json, os, sys, termios, time\n"
|
|
f"expected = {secret_literals}\n"
|
|
f"marker = {marker_literal}\n"
|
|
f"behavior = {behavior!r}\n"
|
|
"args = sys.argv[1:]\n"
|
|
"try:\n"
|
|
" role_arg = args[args.index('-c') + 1]\n"
|
|
" role = role_arg.split(' ', 1)[1]\n"
|
|
"except Exception:\n"
|
|
" open(marker, 'a').write('bad-command\\n')\n"
|
|
" raise SystemExit(90)\n"
|
|
f"expected_args = ['exec', '-i', '-t', '--user', '70:70', '--env', 'LANG=C', '--env', 'LC_ALL=C', '--env', 'PGPASSFILE=/dev/null', '--env', 'PGSERVICEFILE=/dev/null', '--env', 'PGPASSWORD=', '--env', 'PGOPTIONS=', 'itworx-forge-postgres-1', '/bin/busybox', 'timeout', '-s', 'TERM', '-k', '{module.REMOTE_KILL_AFTER_SECONDS}', '{module.REMOTE_TIMEOUT_SECONDS}', 'psql', '-X', '-w', '-v', 'ON_ERROR_STOP=1', '-h', '/var/run/postgresql', '-p', '5432', '-U', 'forge_bootstrap', '-d', 'forge', '-c', role_arg]\n"
|
|
"if args != expected_args or role not in expected:\n"
|
|
" open(marker, 'a').write('bad-command\\n')\n"
|
|
" raise SystemExit(91)\n"
|
|
"secret = expected[role]\n"
|
|
"if any(secret in value for value in sys.argv + list(os.environ.values())):\n"
|
|
" open(marker, 'a').write('secret-in-command-or-env\\n')\n"
|
|
" raise SystemExit(92)\n"
|
|
"if termios.tcgetattr(0)[3] & (termios.ECHO | termios.ECHONL):\n"
|
|
" open(marker, 'a').write('echo-enabled\\n')\n"
|
|
" raise SystemExit(93)\n"
|
|
"if behavior == 'exit-before-prompt':\n"
|
|
" raise SystemExit(7)\n"
|
|
"if behavior == 'timeout-before-prompt':\n"
|
|
" open(marker + '.pid', 'w').write(str(os.getpid()))\n"
|
|
" time.sleep(30)\n"
|
|
" raise SystemExit(0)\n"
|
|
"if behavior == 'output-flood':\n"
|
|
" os.write(2, b'x' * 65536)\n"
|
|
" time.sleep(30)\n"
|
|
" raise SystemExit(0)\n"
|
|
"if behavior == 'prompts-then-exit':\n"
|
|
" os.write(2, f'Enter new password for user \\\"{role}\\\": Enter it again: '.encode())\n"
|
|
" raise SystemExit(0)\n"
|
|
"os.write(2, f'Enter new password for user \\\"{role}\\\": '.encode())\n"
|
|
"first = sys.stdin.readline().rstrip('\\r\\n')\n"
|
|
"if first != secret:\n"
|
|
" open(marker, 'a').write('first-mismatch\\n')\n"
|
|
" raise SystemExit(94)\n"
|
|
"if behavior == 'echo-received':\n"
|
|
" os.write(2, first.encode() + b'\\n')\n"
|
|
"if behavior == 'exit-after-first':\n"
|
|
" open(marker, 'a').write('first-input-validated\\n')\n"
|
|
" raise SystemExit(8)\n"
|
|
"os.write(2, b'Enter it again: ')\n"
|
|
"second = sys.stdin.readline().rstrip('\\r\\n')\n"
|
|
"if second != secret:\n"
|
|
" open(marker, 'a').write('second-mismatch\\n')\n"
|
|
" raise SystemExit(95)\n"
|
|
"open(marker, 'a').write(role + '=matched-noecho\\n')\n"
|
|
"raise SystemExit(0)\n",
|
|
encoding="utf-8",
|
|
)
|
|
script.chmod(0o700)
|
|
module._DOCKER_CLI = str(script)
|
|
module._CHILD_PATH = str(tmp_path)
|
|
return marker
|
|
|
|
|
|
def _install_uri_reader(monkeypatch, module, secrets):
|
|
uri_by_name = {
|
|
"runtime_database_url": _uri(module.RUNTIME_ROLE, secrets[module.RUNTIME_ROLE]),
|
|
"migrator_database_url": _uri(module.MIGRATOR_ROLE, secrets[module.MIGRATOR_ROLE]),
|
|
}
|
|
reads = []
|
|
|
|
def read(_root, name, role):
|
|
reads.append((name, role))
|
|
return uri_by_name.get(name)
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"read_database_uri",
|
|
read,
|
|
)
|
|
return reads
|
|
|
|
|
|
def _journal(initial):
|
|
states = dict(initial)
|
|
calls: list[tuple[str, str]] = []
|
|
|
|
def role_state(role):
|
|
return states[role]
|
|
|
|
def transition(name, allowed, destination):
|
|
def apply(role):
|
|
if states[role] not in allowed:
|
|
raise RuntimeError("compare-and-swap rejected")
|
|
states[role] = destination
|
|
calls.append((name, role))
|
|
return True
|
|
|
|
return apply
|
|
|
|
return states, calls, {
|
|
"role_state": role_state,
|
|
"persist_role_intent": transition("intent", {"pending", "retryable"}, "intent"),
|
|
"persist_role_retryable": transition("retryable", {"intent"}, "retryable"),
|
|
"persist_role_unknown": transition("unknown", {"intent"}, "unknown"),
|
|
"persist_role_complete": transition("complete", {"intent"}, "completed"),
|
|
}
|
|
|
|
|
|
def _run(module, secret_root, journal):
|
|
return module.set_forge_database_role_passwords(secret_root, **journal)
|
|
|
|
|
|
def test_fixed_roles_are_set_through_hidden_prompts_without_secret_output(
|
|
credentials_module, monkeypatch, tmp_path
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
reads = _install_uri_reader(monkeypatch, module, secrets)
|
|
marker = _install_fake_docker(module, tmp_path, secrets, behavior="echo-received")
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert result == {
|
|
"status": "completed",
|
|
"completedRoles": [module.RUNTIME_ROLE, module.MIGRATOR_ROLE],
|
|
"activeRole": None,
|
|
}
|
|
assert states == {
|
|
module.RUNTIME_ROLE: "completed",
|
|
module.MIGRATOR_ROLE: "completed",
|
|
}
|
|
assert calls == [
|
|
("intent", module.RUNTIME_ROLE),
|
|
("complete", module.RUNTIME_ROLE),
|
|
("intent", module.MIGRATOR_ROLE),
|
|
("complete", module.MIGRATOR_ROLE),
|
|
]
|
|
assert reads == [
|
|
("runtime_database_url", module.RUNTIME_ROLE),
|
|
("migrator_database_url", module.MIGRATOR_ROLE),
|
|
]
|
|
assert marker.read_text(encoding="utf-8").splitlines() == [
|
|
f"{module.RUNTIME_ROLE}=matched-noecho",
|
|
f"{module.MIGRATOR_ROLE}=matched-noecho",
|
|
]
|
|
serialized = json.dumps(result)
|
|
assert all(secret not in serialized for secret in secrets.values())
|
|
|
|
|
|
def test_completed_role_is_skipped_and_uncertain_role_is_never_replayed(
|
|
credentials_module, monkeypatch, tmp_path
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
_install_uri_reader(monkeypatch, module, secrets)
|
|
marker = _install_fake_docker(module, tmp_path, secrets)
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "completed", module.MIGRATOR_ROLE: "intent"}
|
|
)
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["completedRoles"] == [module.RUNTIME_ROLE]
|
|
assert result["activeRole"] == module.MIGRATOR_ROLE
|
|
assert calls == []
|
|
assert not marker.exists()
|
|
|
|
|
|
def test_migrator_cannot_be_journaled_complete_before_runtime_role(
|
|
credentials_module, tmp_path
|
|
):
|
|
module = credentials_module
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "completed"}
|
|
)
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert result["status"] == "blocked"
|
|
assert result["reason"] == "role_journal_order_invalid"
|
|
assert result["completedRoles"] == []
|
|
assert calls == []
|
|
|
|
|
|
@pytest.mark.parametrize("behavior", ["exit-before-prompt", "timeout-before-prompt", "output-flood"])
|
|
def test_no_input_failures_are_bounded_reaped_and_retryable(
|
|
credentials_module, monkeypatch, tmp_path, behavior
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
_install_uri_reader(monkeypatch, module, secrets)
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
monkeypatch.setattr(module, "PROMPT_TIMEOUT_SECONDS", 0.15)
|
|
monkeypatch.setattr(module, "ROLE_TIMEOUT_SECONDS", 0.6)
|
|
monkeypatch.setattr(module, "REMOTE_TIMEOUT_SECONDS", 1)
|
|
monkeypatch.setattr(module, "REMOTE_KILL_AFTER_SECONDS", 1)
|
|
monkeypatch.setattr(module, "REMOTE_TERMINATION_MARGIN_SECONDS", 0.1)
|
|
monkeypatch.setattr(module, "MAX_TRANSCRIPT_BYTES", 128)
|
|
marker = _install_fake_docker(module, tmp_path, secrets, behavior=behavior)
|
|
started = __import__("time").monotonic()
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
elapsed = __import__("time").monotonic() - started
|
|
assert result["status"] == "failed"
|
|
assert result["reason"] == "psql_failed_before_password_input"
|
|
assert result["activeRole"] == module.RUNTIME_ROLE
|
|
assert states[module.RUNTIME_ROLE] == "retryable"
|
|
assert states[module.MIGRATOR_ROLE] == "pending"
|
|
assert calls == [
|
|
("intent", module.RUNTIME_ROLE),
|
|
("retryable", module.RUNTIME_ROLE),
|
|
]
|
|
assert elapsed < 3
|
|
if behavior == "timeout-before-prompt":
|
|
assert elapsed >= 1.8
|
|
child_pid = int(Path(str(marker) + ".pid").read_text(encoding="utf-8"))
|
|
with pytest.raises(ProcessLookupError):
|
|
os.kill(child_pid, 0)
|
|
assert not marker.exists() or not any(
|
|
value in marker.read_text(encoding="utf-8")
|
|
for value in secrets.values()
|
|
)
|
|
|
|
|
|
def test_any_password_input_without_success_becomes_unknown(
|
|
credentials_module, monkeypatch, tmp_path
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
_install_uri_reader(monkeypatch, module, secrets)
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
# Prove the post-input branch, rather than timing out Python startup under load.
|
|
monkeypatch.setattr(module, "PROMPT_TIMEOUT_SECONDS", 2.0)
|
|
monkeypatch.setattr(module, "REMOTE_TIMEOUT_SECONDS", 1)
|
|
monkeypatch.setattr(module, "REMOTE_KILL_AFTER_SECONDS", 1)
|
|
monkeypatch.setattr(module, "REMOTE_TERMINATION_MARGIN_SECONDS", 0.1)
|
|
marker = _install_fake_docker(module, tmp_path, secrets, behavior="exit-after-first")
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert marker.read_text(encoding="utf-8") == "first-input-validated\n"
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["reason"] == "psql_result_uncertain"
|
|
assert states[module.RUNTIME_ROLE] == "unknown"
|
|
assert states[module.MIGRATOR_ROLE] == "pending"
|
|
assert calls == [
|
|
("intent", module.RUNTIME_ROLE),
|
|
("unknown", module.RUNTIME_ROLE),
|
|
]
|
|
assert not marker.exists() or not any(
|
|
value in marker.read_text(encoding="utf-8")
|
|
for value in secrets.values()
|
|
)
|
|
|
|
|
|
def test_invalid_fixed_uri_blocks_before_any_intent_or_process(
|
|
credentials_module, monkeypatch, tmp_path
|
|
):
|
|
module = credentials_module
|
|
monkeypatch.setattr(module, "read_database_uri", lambda *_args: None)
|
|
secrets = _secrets(module)
|
|
marker = _install_fake_docker(module, tmp_path, secrets)
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert result["status"] == "blocked"
|
|
assert result["reason"] == "database_uri_unavailable"
|
|
assert states == {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
assert calls == []
|
|
assert not marker.exists()
|
|
|
|
|
|
def test_concurrent_call_stops_on_durable_intent_without_starting_a_second_child(
|
|
credentials_module, monkeypatch, tmp_path
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
_install_uri_reader(monkeypatch, module, secrets)
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
child_calls = []
|
|
|
|
def hold_child(role, _password):
|
|
child_calls.append(role)
|
|
entered.set()
|
|
assert release.wait(timeout=2)
|
|
return "failed", False
|
|
|
|
monkeypatch.setattr(module, "_run_password_command", hold_child)
|
|
first_result = []
|
|
|
|
thread = threading.Thread(
|
|
target=lambda: first_result.append(_run(module, tmp_path, journal)),
|
|
daemon=True,
|
|
)
|
|
thread.start()
|
|
assert entered.wait(timeout=1)
|
|
|
|
second_result = _run(module, tmp_path, journal)
|
|
release.set()
|
|
thread.join(timeout=2)
|
|
|
|
assert not thread.is_alive()
|
|
assert second_result["status"] == "outcome_unknown"
|
|
assert second_result["reason"] == "role_requires_login_reconciliation"
|
|
assert first_result[0]["status"] == "failed"
|
|
assert child_calls == [module.RUNTIME_ROLE]
|
|
assert states[module.RUNTIME_ROLE] == "retryable"
|
|
assert calls == [
|
|
("intent", module.RUNTIME_ROLE),
|
|
("retryable", module.RUNTIME_ROLE),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("claim_result", [False, None])
|
|
def test_failed_durable_intent_claim_never_starts_child(
|
|
credentials_module, monkeypatch, tmp_path, claim_result
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
_install_uri_reader(monkeypatch, module, secrets)
|
|
marker = _install_fake_docker(module, tmp_path, secrets)
|
|
states, calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
journal["persist_role_intent"] = lambda _role: claim_result
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["reason"] == "role_intent_persistence_failed"
|
|
assert states[module.RUNTIME_ROLE] == "pending"
|
|
assert calls == []
|
|
assert not marker.exists()
|
|
|
|
|
|
def test_exited_child_prompt_output_is_drained_without_sending_password(
|
|
credentials_module, monkeypatch, tmp_path
|
|
):
|
|
module = credentials_module
|
|
secrets = _secrets(module)
|
|
_install_uri_reader(monkeypatch, module, secrets)
|
|
_install_fake_docker(module, tmp_path, secrets, behavior="prompts-then-exit")
|
|
states, _calls, journal = _journal(
|
|
{module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"}
|
|
)
|
|
child = []
|
|
writes = []
|
|
original_popen = module.subprocess.Popen
|
|
original_select = module.select.select
|
|
original_write_password = module._write_password_line
|
|
|
|
def capture_popen(*args, **kwargs):
|
|
process = original_popen(*args, **kwargs)
|
|
child.append(process)
|
|
return process
|
|
|
|
def select_after_exit(*args, **kwargs):
|
|
child[0].wait(timeout=2)
|
|
return original_select(*args, **kwargs)
|
|
|
|
def record_write(*args, **kwargs):
|
|
writes.append(True)
|
|
return original_write_password(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(module.subprocess, "Popen", capture_popen)
|
|
monkeypatch.setattr(module.select, "select", select_after_exit)
|
|
monkeypatch.setattr(module, "_write_password_line", record_write)
|
|
|
|
result = _run(module, tmp_path, journal)
|
|
|
|
assert result["status"] != "completed"
|
|
assert writes == []
|
|
assert states[module.RUNTIME_ROLE] != "completed"
|
|
assert states[module.RUNTIME_ROLE] == "retryable"
|
|
|
|
|
|
def test_zero_length_pty_write_fails_closed(credentials_module, monkeypatch):
|
|
module = credentials_module
|
|
monkeypatch.setattr(module.os, "write", lambda *_args: 0)
|
|
monkeypatch.setattr(
|
|
module.select,
|
|
"select",
|
|
lambda _readers, writers, _errors, _timeout: ([], writers, []),
|
|
)
|
|
|
|
with pytest.raises(OSError):
|
|
module._write_password_line(123, bytearray(b"bounded-secret"))
|
|
|
|
|
|
def test_pty_write_backpressure_expires_within_fixed_budget(
|
|
credentials_module, monkeypatch
|
|
):
|
|
module = credentials_module
|
|
ticks = iter(range(0, 10))
|
|
monkeypatch.setattr(module.time, "monotonic", lambda: float(next(ticks)))
|
|
monkeypatch.setattr(
|
|
module.select,
|
|
"select",
|
|
lambda _readers, _writers, _errors, _timeout: ([], [], []),
|
|
)
|
|
monkeypatch.setattr(module, "PTY_WRITE_TIMEOUT_SECONDS", 2.0)
|
|
|
|
with pytest.raises(TimeoutError):
|
|
module._write_password_line(123, bytearray(b"bounded-secret"))
|
|
|
|
|
|
def test_uri_password_decoder_rejects_malformed_or_unbounded_values(credentials_module):
|
|
module = credentials_module
|
|
role = module.RUNTIME_ROLE
|
|
|
|
assert module._password_from_uri(_uri(role, "x" * 31), role) is None
|
|
assert module._password_from_uri(_uri(role, "x" * 257), role) is None
|
|
assert module._password_from_uri(
|
|
f"postgresql://{role}:bad%Q0value-with-more-than-32-characters@postgres:5432/forge".encode(),
|
|
role,
|
|
) is None
|
|
|
|
decoded = module._password_from_uri(
|
|
f"postgresql://{role}:encoded%2Dpassword%2D0123456789abcdef@postgres:5432/forge".encode(),
|
|
role,
|
|
)
|
|
assert decoded == bytearray(b"encoded-password-0123456789abcdef")
|
|
decoded[:] = b"\0" * len(decoded)
|