from __future__ import annotations import importlib.util import json import os import sys import threading from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] BROKER = ROOT / "ops" / "gitea-deploy-broker" if str(BROKER) not in sys.path: sys.path.insert(0, str(BROKER)) @pytest.fixture def credentials_module(): spec = importlib.util.spec_from_file_location( "forge_database_credentials_test", BROKER / "forge_database_credentials.py", ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _secrets(module): return { module.RUNTIME_ROLE: "runtime-secret-SENTINEL-0123456789abcdef", module.MIGRATOR_ROLE: "migrator-secret-SENTINEL-fedcba9876543210", } def _uri(role: str, password: str) -> bytes: return f"postgresql://{role}:{password}@postgres:5432/forge".encode("ascii") def _install_fake_docker( module, tmp_path: Path, secrets: dict[str, str], *, behavior: str = "success", ): marker = tmp_path / "fake-docker-state.txt" script = tmp_path / "fake-docker" secret_literals = json.dumps(secrets) marker_literal = json.dumps(str(marker)) script.write_text( f"#!{sys.executable}\n" "import json, os, sys, termios, time\n" f"expected = {secret_literals}\n" f"marker = {marker_literal}\n" f"behavior = {behavior!r}\n" "args = sys.argv[1:]\n" "try:\n" " role_arg = args[args.index('-c') + 1]\n" " role = role_arg.split(' ', 1)[1]\n" "except Exception:\n" " open(marker, 'a').write('bad-command\\n')\n" " raise SystemExit(90)\n" f"expected_args = ['exec', '-i', '-t', '--user', '70:70', '--env', 'LANG=C', '--env', 'LC_ALL=C', '--env', 'PGPASSFILE=/dev/null', '--env', 'PGSERVICEFILE=/dev/null', '--env', 'PGPASSWORD=', '--env', 'PGOPTIONS=', 'itworx-forge-postgres-1', '/bin/busybox', 'timeout', '-s', 'TERM', '-k', '{module.REMOTE_KILL_AFTER_SECONDS}', '{module.REMOTE_TIMEOUT_SECONDS}', 'psql', '-X', '-w', '-v', 'ON_ERROR_STOP=1', '-h', '/var/run/postgresql', '-p', '5432', '-U', 'forge_bootstrap', '-d', 'forge', '-c', role_arg]\n" "if args != expected_args or role not in expected:\n" " open(marker, 'a').write('bad-command\\n')\n" " raise SystemExit(91)\n" "secret = expected[role]\n" "if any(secret in value for value in sys.argv + list(os.environ.values())):\n" " open(marker, 'a').write('secret-in-command-or-env\\n')\n" " raise SystemExit(92)\n" "if termios.tcgetattr(0)[3] & (termios.ECHO | termios.ECHONL):\n" " open(marker, 'a').write('echo-enabled\\n')\n" " raise SystemExit(93)\n" "if behavior == 'exit-before-prompt':\n" " raise SystemExit(7)\n" "if behavior == 'timeout-before-prompt':\n" " open(marker + '.pid', 'w').write(str(os.getpid()))\n" " time.sleep(30)\n" " raise SystemExit(0)\n" "if behavior == 'output-flood':\n" " os.write(2, b'x' * 65536)\n" " time.sleep(30)\n" " raise SystemExit(0)\n" "if behavior == 'prompts-then-exit':\n" " os.write(2, f'Enter new password for user \\\"{role}\\\": Enter it again: '.encode())\n" " raise SystemExit(0)\n" "os.write(2, f'Enter new password for user \\\"{role}\\\": '.encode())\n" "first = sys.stdin.readline().rstrip('\\r\\n')\n" "if first != secret:\n" " open(marker, 'a').write('first-mismatch\\n')\n" " raise SystemExit(94)\n" "if behavior == 'echo-received':\n" " os.write(2, first.encode() + b'\\n')\n" "if behavior == 'exit-after-first':\n" " raise SystemExit(8)\n" "os.write(2, b'Enter it again: ')\n" "second = sys.stdin.readline().rstrip('\\r\\n')\n" "if second != secret:\n" " open(marker, 'a').write('second-mismatch\\n')\n" " raise SystemExit(95)\n" "open(marker, 'a').write(role + '=matched-noecho\\n')\n" "raise SystemExit(0)\n", encoding="utf-8", ) script.chmod(0o700) module._DOCKER_CLI = str(script) module._CHILD_PATH = str(tmp_path) return marker def _install_uri_reader(monkeypatch, module, secrets): uri_by_name = { "runtime_database_url": _uri(module.RUNTIME_ROLE, secrets[module.RUNTIME_ROLE]), "migrator_database_url": _uri(module.MIGRATOR_ROLE, secrets[module.MIGRATOR_ROLE]), } reads = [] def read(_root, name, role): reads.append((name, role)) return uri_by_name.get(name) monkeypatch.setattr( module, "read_database_uri", read, ) return reads def _journal(initial): states = dict(initial) calls: list[tuple[str, str]] = [] def role_state(role): return states[role] def transition(name, allowed, destination): def apply(role): if states[role] not in allowed: raise RuntimeError("compare-and-swap rejected") states[role] = destination calls.append((name, role)) return True return apply return states, calls, { "role_state": role_state, "persist_role_intent": transition("intent", {"pending", "retryable"}, "intent"), "persist_role_retryable": transition("retryable", {"intent"}, "retryable"), "persist_role_unknown": transition("unknown", {"intent"}, "unknown"), "persist_role_complete": transition("complete", {"intent"}, "completed"), } def _run(module, secret_root, journal): return module.set_forge_database_role_passwords(secret_root, **journal) def test_fixed_roles_are_set_through_hidden_prompts_without_secret_output( credentials_module, monkeypatch, tmp_path ): module = credentials_module secrets = _secrets(module) reads = _install_uri_reader(monkeypatch, module, secrets) marker = _install_fake_docker(module, tmp_path, secrets, behavior="echo-received") states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) result = _run(module, tmp_path, journal) assert result == { "status": "completed", "completedRoles": [module.RUNTIME_ROLE, module.MIGRATOR_ROLE], "activeRole": None, } assert states == { module.RUNTIME_ROLE: "completed", module.MIGRATOR_ROLE: "completed", } assert calls == [ ("intent", module.RUNTIME_ROLE), ("complete", module.RUNTIME_ROLE), ("intent", module.MIGRATOR_ROLE), ("complete", module.MIGRATOR_ROLE), ] assert reads == [ ("runtime_database_url", module.RUNTIME_ROLE), ("migrator_database_url", module.MIGRATOR_ROLE), ] assert marker.read_text(encoding="utf-8").splitlines() == [ f"{module.RUNTIME_ROLE}=matched-noecho", f"{module.MIGRATOR_ROLE}=matched-noecho", ] serialized = json.dumps(result) assert all(secret not in serialized for secret in secrets.values()) def test_completed_role_is_skipped_and_uncertain_role_is_never_replayed( credentials_module, monkeypatch, tmp_path ): module = credentials_module secrets = _secrets(module) _install_uri_reader(monkeypatch, module, secrets) marker = _install_fake_docker(module, tmp_path, secrets) states, calls, journal = _journal( {module.RUNTIME_ROLE: "completed", module.MIGRATOR_ROLE: "intent"} ) result = _run(module, tmp_path, journal) assert result["status"] == "outcome_unknown" assert result["completedRoles"] == [module.RUNTIME_ROLE] assert result["activeRole"] == module.MIGRATOR_ROLE assert calls == [] assert not marker.exists() def test_migrator_cannot_be_journaled_complete_before_runtime_role( credentials_module, tmp_path ): module = credentials_module states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "completed"} ) result = _run(module, tmp_path, journal) assert result["status"] == "blocked" assert result["reason"] == "role_journal_order_invalid" assert result["completedRoles"] == [] assert calls == [] @pytest.mark.parametrize("behavior", ["exit-before-prompt", "timeout-before-prompt", "output-flood"]) def test_no_input_failures_are_bounded_reaped_and_retryable( credentials_module, monkeypatch, tmp_path, behavior ): module = credentials_module secrets = _secrets(module) _install_uri_reader(monkeypatch, module, secrets) states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) monkeypatch.setattr(module, "PROMPT_TIMEOUT_SECONDS", 0.15) monkeypatch.setattr(module, "ROLE_TIMEOUT_SECONDS", 0.6) monkeypatch.setattr(module, "REMOTE_TIMEOUT_SECONDS", 1) monkeypatch.setattr(module, "REMOTE_KILL_AFTER_SECONDS", 1) monkeypatch.setattr(module, "REMOTE_TERMINATION_MARGIN_SECONDS", 0.1) monkeypatch.setattr(module, "MAX_TRANSCRIPT_BYTES", 128) marker = _install_fake_docker(module, tmp_path, secrets, behavior=behavior) started = __import__("time").monotonic() result = _run(module, tmp_path, journal) elapsed = __import__("time").monotonic() - started assert result["status"] == "failed" assert result["reason"] == "psql_failed_before_password_input" assert result["activeRole"] == module.RUNTIME_ROLE assert states[module.RUNTIME_ROLE] == "retryable" assert states[module.MIGRATOR_ROLE] == "pending" assert calls == [ ("intent", module.RUNTIME_ROLE), ("retryable", module.RUNTIME_ROLE), ] assert elapsed < 3 if behavior == "timeout-before-prompt": assert elapsed >= 1.8 child_pid = int(Path(str(marker) + ".pid").read_text(encoding="utf-8")) with pytest.raises(ProcessLookupError): os.kill(child_pid, 0) assert not marker.exists() or not any( value in marker.read_text(encoding="utf-8") for value in secrets.values() ) def test_any_password_input_without_success_becomes_unknown( credentials_module, monkeypatch, tmp_path ): module = credentials_module secrets = _secrets(module) _install_uri_reader(monkeypatch, module, secrets) states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) monkeypatch.setattr(module, "PROMPT_TIMEOUT_SECONDS", 0.15) monkeypatch.setattr(module, "REMOTE_TIMEOUT_SECONDS", 1) monkeypatch.setattr(module, "REMOTE_KILL_AFTER_SECONDS", 1) monkeypatch.setattr(module, "REMOTE_TERMINATION_MARGIN_SECONDS", 0.1) marker = _install_fake_docker(module, tmp_path, secrets, behavior="exit-after-first") result = _run(module, tmp_path, journal) assert result["status"] == "outcome_unknown" assert result["reason"] == "psql_result_uncertain" assert states[module.RUNTIME_ROLE] == "unknown" assert states[module.MIGRATOR_ROLE] == "pending" assert calls == [ ("intent", module.RUNTIME_ROLE), ("unknown", module.RUNTIME_ROLE), ] assert not marker.exists() or not any( value in marker.read_text(encoding="utf-8") for value in secrets.values() ) def test_invalid_fixed_uri_blocks_before_any_intent_or_process( credentials_module, monkeypatch, tmp_path ): module = credentials_module monkeypatch.setattr(module, "read_database_uri", lambda *_args: None) secrets = _secrets(module) marker = _install_fake_docker(module, tmp_path, secrets) states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) result = _run(module, tmp_path, journal) assert result["status"] == "blocked" assert result["reason"] == "database_uri_unavailable" assert states == {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} assert calls == [] assert not marker.exists() def test_concurrent_call_stops_on_durable_intent_without_starting_a_second_child( credentials_module, monkeypatch, tmp_path ): module = credentials_module secrets = _secrets(module) _install_uri_reader(monkeypatch, module, secrets) states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) entered = threading.Event() release = threading.Event() child_calls = [] def hold_child(role, _password): child_calls.append(role) entered.set() assert release.wait(timeout=2) return "failed", False monkeypatch.setattr(module, "_run_password_command", hold_child) first_result = [] thread = threading.Thread( target=lambda: first_result.append(_run(module, tmp_path, journal)), daemon=True, ) thread.start() assert entered.wait(timeout=1) second_result = _run(module, tmp_path, journal) release.set() thread.join(timeout=2) assert not thread.is_alive() assert second_result["status"] == "outcome_unknown" assert second_result["reason"] == "role_requires_login_reconciliation" assert first_result[0]["status"] == "failed" assert child_calls == [module.RUNTIME_ROLE] assert states[module.RUNTIME_ROLE] == "retryable" assert calls == [ ("intent", module.RUNTIME_ROLE), ("retryable", module.RUNTIME_ROLE), ] @pytest.mark.parametrize("claim_result", [False, None]) def test_failed_durable_intent_claim_never_starts_child( credentials_module, monkeypatch, tmp_path, claim_result ): module = credentials_module secrets = _secrets(module) _install_uri_reader(monkeypatch, module, secrets) marker = _install_fake_docker(module, tmp_path, secrets) states, calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) journal["persist_role_intent"] = lambda _role: claim_result result = _run(module, tmp_path, journal) assert result["status"] == "outcome_unknown" assert result["reason"] == "role_intent_persistence_failed" assert states[module.RUNTIME_ROLE] == "pending" assert calls == [] assert not marker.exists() def test_exited_child_prompt_output_is_drained_without_sending_password( credentials_module, monkeypatch, tmp_path ): module = credentials_module secrets = _secrets(module) _install_uri_reader(monkeypatch, module, secrets) _install_fake_docker(module, tmp_path, secrets, behavior="prompts-then-exit") states, _calls, journal = _journal( {module.RUNTIME_ROLE: "pending", module.MIGRATOR_ROLE: "pending"} ) child = [] writes = [] original_popen = module.subprocess.Popen original_select = module.select.select original_write_password = module._write_password_line def capture_popen(*args, **kwargs): process = original_popen(*args, **kwargs) child.append(process) return process def select_after_exit(*args, **kwargs): child[0].wait(timeout=2) return original_select(*args, **kwargs) def record_write(*args, **kwargs): writes.append(True) return original_write_password(*args, **kwargs) monkeypatch.setattr(module.subprocess, "Popen", capture_popen) monkeypatch.setattr(module.select, "select", select_after_exit) monkeypatch.setattr(module, "_write_password_line", record_write) result = _run(module, tmp_path, journal) assert result["status"] != "completed" assert writes == [] assert states[module.RUNTIME_ROLE] != "completed" assert states[module.RUNTIME_ROLE] == "retryable" def test_zero_length_pty_write_fails_closed(credentials_module, monkeypatch): module = credentials_module monkeypatch.setattr(module.os, "write", lambda *_args: 0) monkeypatch.setattr( module.select, "select", lambda _readers, writers, _errors, _timeout: ([], writers, []), ) with pytest.raises(OSError): module._write_password_line(123, bytearray(b"bounded-secret")) def test_pty_write_backpressure_expires_within_fixed_budget( credentials_module, monkeypatch ): module = credentials_module ticks = iter(range(0, 10)) monkeypatch.setattr(module.time, "monotonic", lambda: float(next(ticks))) monkeypatch.setattr( module.select, "select", lambda _readers, _writers, _errors, _timeout: ([], [], []), ) monkeypatch.setattr(module, "PTY_WRITE_TIMEOUT_SECONDS", 2.0) with pytest.raises(TimeoutError): module._write_password_line(123, bytearray(b"bounded-secret")) def test_uri_password_decoder_rejects_malformed_or_unbounded_values(credentials_module): module = credentials_module role = module.RUNTIME_ROLE assert module._password_from_uri(_uri(role, "x" * 31), role) is None assert module._password_from_uri(_uri(role, "x" * 257), role) is None assert module._password_from_uri( f"postgresql://{role}:bad%Q0value-with-more-than-32-characters@postgres:5432/forge".encode(), role, ) is None decoded = module._password_from_uri( f"postgresql://{role}:encoded%2Dpassword%2D0123456789abcdef@postgres:5432/forge".encode(), role, ) assert decoded == bytearray(b"encoded-password-0123456789abcdef") decoded[:] = b"\0" * len(decoded)