2062 lines
83 KiB
Python
2062 lines
83 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import types
|
|
from email.message import Message
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import urlparse
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
BROKER = ROOT / "ops" / "gitea-deploy-broker"
|
|
_FORGE_PRIVATE_OWNER = "itworx-forge-private"
|
|
_FORGE_SOURCE_URL = "https://gitea.example.com/Jens/itworx-forge"
|
|
|
|
|
|
def _load(path: Path, name: str):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def controller_module():
|
|
return _load(BROKER / "forge_first_install.py", "forge_first_install_controller_test")
|
|
|
|
|
|
@pytest.fixture
|
|
def status_module():
|
|
if "fcntl" not in sys.modules:
|
|
fcntl = types.ModuleType("fcntl")
|
|
fcntl.LOCK_EX = 2
|
|
fcntl.LOCK_UN = 8
|
|
fcntl.flock = lambda *_args: None
|
|
sys.modules["fcntl"] = fcntl
|
|
return _load(BROKER / "status_server.py", "forge_first_install_status_test")
|
|
|
|
|
|
def _profile(module):
|
|
return {
|
|
"api_version": module.API_VERSION,
|
|
"revision": "a" * 40,
|
|
"workspace_image_digest": "sha256:" + "b" * 64,
|
|
"frontdoor_image_digest": "sha256:" + "c" * 64,
|
|
"preflight_job_id": "forge-preflight-" + "d" * 16,
|
|
"preflight_sha256": "e" * 64,
|
|
}
|
|
|
|
|
|
def _passed_preflight(profile):
|
|
checks = {
|
|
name: {"status": "passed"}
|
|
for name in (
|
|
"exact_main", "source_manifest", "managed_appdata", "proxy_network",
|
|
"resource_collisions", "push_validation", "image_registry", "oidc",
|
|
"secret_delivery",
|
|
)
|
|
}
|
|
return {
|
|
"apiVersion": "itworx.forge-bootstrap-preflight/v1",
|
|
"status": "preflight_ok",
|
|
"revision": profile["revision"],
|
|
"workspaceImageDigest": profile["workspace_image_digest"],
|
|
"frontdoorImageDigest": profile["frontdoor_image_digest"],
|
|
"checks": checks,
|
|
"evidence": {},
|
|
}
|
|
|
|
|
|
def _forge_oidc_test_certificate(*, elliptic_curve: bool = False, rsa_key_size: int = 2048):
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from cryptography import x509
|
|
from cryptography.hazmat.primitives import hashes, serialization
|
|
from cryptography.hazmat.primitives.asymmetric import ec, rsa
|
|
from cryptography.x509.oid import NameOID
|
|
|
|
if elliptic_curve:
|
|
private_key = ec.generate_private_key(ec.SECP256R1())
|
|
else:
|
|
private_key = rsa.generate_private_key(public_exponent=65537, key_size=rsa_key_size)
|
|
subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "forge-oidc-test")])
|
|
now = datetime.now(timezone.utc)
|
|
certificate = (
|
|
x509.CertificateBuilder()
|
|
.subject_name(subject)
|
|
.issuer_name(subject)
|
|
.public_key(private_key.public_key())
|
|
.serial_number(x509.random_serial_number())
|
|
.not_valid_before((now - timedelta(minutes=1)).replace(tzinfo=None))
|
|
.not_valid_after((now + timedelta(days=1)).replace(tzinfo=None))
|
|
.sign(private_key, hashes.SHA256())
|
|
)
|
|
certificate_der = certificate.public_bytes(serialization.Encoding.DER)
|
|
certificate_pem = certificate.public_bytes(serialization.Encoding.PEM).decode("ascii")
|
|
public_numbers = private_key.public_key().public_numbers()
|
|
|
|
def encode(value: int) -> str:
|
|
raw = value.to_bytes((value.bit_length() + 7) // 8, "big")
|
|
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
|
|
|
jwk = None
|
|
if not elliptic_curve:
|
|
jwk = {
|
|
"kty": "RSA",
|
|
"alg": "RS256",
|
|
"use": "sig",
|
|
"kid": "8cf5f32a6c6d9a876fd0284980a25ce0",
|
|
"n": encode(public_numbers.n),
|
|
"e": encode(public_numbers.e),
|
|
}
|
|
return {
|
|
"data": certificate_pem,
|
|
"fingerprint": hashlib.sha256(certificate_der).hexdigest(),
|
|
"jwk": jwk,
|
|
}
|
|
|
|
|
|
def _controller(module, tmp_path, profile, *, command_runner=None):
|
|
controller = module.ForgeFirstInstallController(
|
|
state_root=tmp_path / "state",
|
|
secret_root=tmp_path / "secrets",
|
|
config_path=tmp_path / "forge-first-install.json",
|
|
repository_path=tmp_path / "itworx-forge.git",
|
|
compose_sha256="f" * 64,
|
|
preflight_check=lambda _profile: _passed_preflight(profile),
|
|
resume_check=lambda _profile: _passed_preflight(profile),
|
|
command_runner=command_runner,
|
|
)
|
|
controller.remember_preflight(profile, _passed_preflight(profile))
|
|
return controller
|
|
|
|
|
|
def _forge_config(**overrides):
|
|
config = {
|
|
"schema": "itworx.forge-first-install-config/v4",
|
|
"issuer": "https://auth.example-host.com/application/o/itworx-forge/",
|
|
"audience": "itworx-forge",
|
|
"client_id": "itworx-forge",
|
|
"authentik_api_url": "https://auth.example-host.com",
|
|
"launch_url": "https://forge.example.com",
|
|
"authorization_flow": "forge-authz",
|
|
"invalidation_flow": "forge-logout",
|
|
"authentik_openid_mapping_id": "01234567-89ab-4cde-8fab-0123456789ab",
|
|
"authentik_email_mapping_id": "21234567-89ab-4cde-8fab-0123456789ab",
|
|
"authentik_profile_mapping_id": "31234567-89ab-4cde-8fab-0123456789ab",
|
|
"authentik_provider_id": "7",
|
|
"authentik_signing_key_id": "11234567-89ab-4cde-8fab-0123456789ab",
|
|
"private_package_owner": _FORGE_PRIVATE_OWNER,
|
|
}
|
|
config.update(overrides)
|
|
return config
|
|
|
|
|
|
def _write_protected_config(path, value):
|
|
path.write_text(json.dumps(value), encoding="utf-8")
|
|
if os.name != "nt":
|
|
path.chmod(0o600)
|
|
|
|
|
|
_FORGE_ROLE_NAMES = ("forge_runtime", "forge_migrator")
|
|
|
|
|
|
def _credential_resume_record(
|
|
module, controller, profile, key, *, status="running",
|
|
phase="resume_intent_persisted", roles=None,
|
|
):
|
|
request = module.request_id(profile, key)
|
|
record = {
|
|
"schema": module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": module.REPOSITORY,
|
|
"operation": module.OPERATION,
|
|
"profile": profile,
|
|
"status": status,
|
|
"phase": phase,
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"credentialRoles": roles or {role: "pending" for role in _FORGE_ROLE_NAMES},
|
|
"resumeGeneration": 1,
|
|
"resumeClaimed": status == "running",
|
|
"resumeAttempts": 1,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
return request, record
|
|
|
|
|
|
def _install_successful_role_helper(monkeypatch, before_send=None):
|
|
calls = []
|
|
helper = types.ModuleType("forge_database_credentials")
|
|
|
|
def set_passwords(secret_root, *, role_state, persist_role_intent,
|
|
persist_role_retryable, persist_role_unknown,
|
|
persist_role_complete):
|
|
calls.append((secret_root, "called"))
|
|
completed = []
|
|
for role in _FORGE_ROLE_NAMES:
|
|
state = role_state(role)
|
|
if state == "completed":
|
|
completed.append(role)
|
|
continue
|
|
if state not in {"pending", "retryable"} or persist_role_intent(role) is not True:
|
|
return {"status": "outcome_unknown", "completedRoles": completed,
|
|
"activeRole": role, "reason": "role_intent_persistence_failed"}
|
|
calls.append((secret_root, role))
|
|
if before_send is not None:
|
|
before_send(role)
|
|
if persist_role_complete(role) is not True:
|
|
persist_role_unknown(role)
|
|
return {"status": "outcome_unknown", "completedRoles": completed,
|
|
"activeRole": role, "reason": "role_completion_persistence_failed"}
|
|
completed.append(role)
|
|
return {"status": "completed", "completedRoles": completed, "activeRole": None}
|
|
|
|
helper.set_forge_database_role_passwords = set_passwords
|
|
monkeypatch.setitem(sys.modules, "forge_database_credentials", helper)
|
|
return calls
|
|
|
|
|
|
def _prepare_credential_resume(controller_module, controller, tmp_path, profile):
|
|
_write_protected_config(controller.config_path, _forge_config())
|
|
controller.resume_check = lambda _profile: _passed_preflight(profile)
|
|
controller._database_roles_are_safe = lambda: True
|
|
controller._wait_health = lambda *_args, **_kwargs: True
|
|
controller._existing_source = lambda _request: tmp_path
|
|
controller._compose_config_valid = lambda *_args, **_kwargs: True
|
|
return controller._compose_command(tmp_path)
|
|
|
|
|
|
def test_controller_config_v4_requires_safe_private_owner_and_distinct_authentik_ids(controller_module, tmp_path):
|
|
config_path = tmp_path / "forge-first-install.json"
|
|
config = _forge_config()
|
|
_write_protected_config(config_path, config)
|
|
|
|
loaded = controller_module.load_config(config_path)
|
|
assert loaded is not None
|
|
assert loaded["private_package_owner"] == _FORGE_PRIVATE_OWNER
|
|
|
|
invalid_configs = [
|
|
{**config, "schema": "itworx.forge-first-install-config/v3"},
|
|
{key: value for key, value in config.items() if key != "private_package_owner"},
|
|
{**config, "private_package_owner": "Jens"},
|
|
{**config, "private_package_owner": "owner.with.dot"},
|
|
{**config, "private_package_owner": "_owner"},
|
|
{**config, "private_package_owner": "a" * 40},
|
|
{**config, "authentik_provider_id": "0"},
|
|
{**config, "authentik_provider_id": "07"},
|
|
{**config, "authentik_provider_id": "provider"},
|
|
{**config, "authentik_provider_id": "1" * 11},
|
|
{**config, "authentik_provider_id": 7},
|
|
{**config, "authentik_email_mapping_id": config["authentik_openid_mapping_id"]},
|
|
{key: value for key, value in config.items() if key != "authentik_profile_mapping_id"},
|
|
]
|
|
for invalid in invalid_configs:
|
|
_write_protected_config(config_path, invalid)
|
|
assert controller_module.load_config(config_path) is None
|
|
assert controller_module.load_config_bytes(json.dumps(invalid).encode("utf-8")) is None
|
|
|
|
|
|
def test_held_config_matches_file_validation_without_filesystem_access(controller_module, tmp_path, monkeypatch):
|
|
config = _forge_config()
|
|
config_path = tmp_path / "forge-first-install.json"
|
|
_write_protected_config(config_path, config)
|
|
expected = controller_module.load_config(config_path)
|
|
assert expected is not None
|
|
|
|
def unexpected_read(*_args, **_kwargs):
|
|
raise AssertionError("held configuration validation must not read a file")
|
|
|
|
monkeypatch.setattr(controller_module, "_read_json", unexpected_read)
|
|
assert controller_module.load_config_bytes(json.dumps(config, indent=2).encode("utf-8")) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("raw", [
|
|
b"", b"\xff", b"[]", b"null", b"{\"schema\":\"a\",\"schema\":\"b\"}",
|
|
b" " * 8193, b"[" * 2000 + b"]" * 2000, "{}", bytearray(b"{}"),
|
|
])
|
|
def test_held_config_rejects_malformed_duplicate_oversized_or_nonbyte_input(controller_module, raw):
|
|
assert controller_module.load_config_bytes(raw) is None
|
|
|
|
|
|
def test_compose_and_pulls_use_only_the_protected_private_package_owner(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
config_path = tmp_path / "protected-config.json"
|
|
_write_protected_config(config_path, _forge_config())
|
|
config = controller_module.load_config(config_path)
|
|
assert config is not None
|
|
environment = controller._compose_environment(profile, config)
|
|
assert environment["FORGE_PRIVATE_PACKAGE_OWNER"] == _FORGE_PRIVATE_OWNER
|
|
assert controller._expected_image(profile, "api", _FORGE_PRIVATE_OWNER) == (
|
|
f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge@{profile['workspace_image_digest']}"
|
|
)
|
|
assert controller._expected_image(profile, "itworx-forge", _FORGE_PRIVATE_OWNER) == (
|
|
f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge-frontdoor@{profile['frontdoor_image_digest']}"
|
|
)
|
|
|
|
commands = []
|
|
controller._command_ok = lambda command, **_kwargs: commands.append(command) or True
|
|
assert controller._pull_images(profile, _FORGE_PRIVATE_OWNER)
|
|
assert commands == [
|
|
["docker", "pull", f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge@{profile['workspace_image_digest']}"],
|
|
["docker", "pull", f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge-frontdoor@{profile['frontdoor_image_digest']}"],
|
|
]
|
|
assert not controller._pull_images(profile, "Jens")
|
|
|
|
|
|
def test_request_id_uses_the_shared_canonical_identity(controller_module):
|
|
profile = _profile(controller_module)
|
|
request = controller_module.request_id(profile, " forge-bootstrap-2026 ")
|
|
payload = {
|
|
"repository": "jens/itworx-forge",
|
|
"operation": "forge_first_install",
|
|
"profile": profile,
|
|
"idempotencyKey": "forge-bootstrap-2026",
|
|
}
|
|
expected = hashlib.sha256(
|
|
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
|
).hexdigest()
|
|
assert request == expected
|
|
|
|
|
|
def test_apply_persists_intent_before_worker_and_replay_never_starts_another(
|
|
controller_module, tmp_path
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
worker_entered = threading.Event()
|
|
release_worker = threading.Event()
|
|
observed = []
|
|
|
|
def hold_worker(record):
|
|
saved = controller._read_record(record["request"])
|
|
observed.append((record["request"], saved["status"] if saved else None))
|
|
worker_entered.set()
|
|
release_worker.wait(timeout=3)
|
|
|
|
controller._execute = hold_worker
|
|
try:
|
|
accepted = controller.apply(profile, "forge-first-install-2026")
|
|
assert accepted["status"] == "accepted"
|
|
assert worker_entered.wait(timeout=2)
|
|
request = accepted["request"]
|
|
assert observed == [(request, "accepted")]
|
|
path = controller._record_path(request)
|
|
assert path.is_file()
|
|
replay = controller.apply(profile, "forge-first-install-2026")
|
|
assert replay["request"] == request
|
|
assert replay["status"] == "accepted"
|
|
assert len(observed) == 1
|
|
finally:
|
|
release_worker.set()
|
|
thread = controller._threads.get(accepted["request"])
|
|
if thread is not None:
|
|
thread.join(timeout=2)
|
|
|
|
|
|
@pytest.mark.parametrize("gate", ["image_registry", "oidc", "secret_delivery"])
|
|
def test_apply_rechecks_security_gates_before_any_service_mutation(
|
|
controller_module, tmp_path, gate
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
current = _passed_preflight(profile)
|
|
current["checks"][gate]["status"] = "blocked"
|
|
controller.preflight_check = lambda _profile: current
|
|
prepared = []
|
|
pulls = []
|
|
actions = []
|
|
controller._prepare_source = lambda *_args, **_kwargs: prepared.append(True)
|
|
controller._pull_images = lambda *_args, **_kwargs: pulls.append(True)
|
|
controller._compose_action = lambda *_args, **_kwargs: actions.append(True)
|
|
|
|
accepted = controller.apply(profile, f"forge-gate-block-{gate}-0001")
|
|
thread = controller._threads[accepted["request"]]
|
|
thread.join(timeout=2)
|
|
|
|
assert controller.status(accepted["request"])["status"] == "preflight_changed"
|
|
assert prepared == []
|
|
assert pulls == []
|
|
assert actions == []
|
|
|
|
|
|
def test_resume_is_durable_and_one_shot_only_from_database_credential_pause(
|
|
controller_module, tmp_path
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request = controller_module.request_id(profile, "forge-first-install-resume")
|
|
record = {
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "needs_database_credentials",
|
|
"phase": "awaiting_external_role_passwords",
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
worker_entered = threading.Event()
|
|
release_worker = threading.Event()
|
|
calls = []
|
|
|
|
def hold_resume(saved):
|
|
calls.append(saved["request"])
|
|
worker_entered.set()
|
|
release_worker.wait(timeout=3)
|
|
|
|
controller._execute_after_database_credentials = hold_resume
|
|
try:
|
|
response = controller.resume(request)
|
|
assert response["status"] == "running"
|
|
assert worker_entered.wait(timeout=2)
|
|
persisted = controller._read_record(request)
|
|
assert persisted["resumeClaimed"] is True
|
|
assert persisted["resumeAttempts"] == 1
|
|
assert controller.resume(request)["status"] == "running"
|
|
assert calls == [request]
|
|
finally:
|
|
release_worker.set()
|
|
thread = controller._threads.get(request)
|
|
if thread is not None:
|
|
thread.join(timeout=2)
|
|
|
|
|
|
def test_resume_refuses_migration_started_or_uncertain_records(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request = controller_module.request_id(profile, "forge-first-install-uncertain")
|
|
controller._write_record(
|
|
{
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "needs_database_credentials",
|
|
"phase": "awaiting_external_role_passwords",
|
|
"mutationStarted": True,
|
|
"migrationStarted": True,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
)
|
|
result = controller.resume(request)
|
|
assert result is not None
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["request"] == request
|
|
assert not controller._threads
|
|
|
|
|
|
def test_resume_rejects_negative_attempt_counter_without_starting_a_worker(
|
|
controller_module, tmp_path
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module,
|
|
controller,
|
|
profile,
|
|
"forge-resume-negative-attempt-counter",
|
|
status="needs_database_credentials",
|
|
phase="awaiting_external_role_passwords",
|
|
)
|
|
record["resumeGeneration"] = 0
|
|
record["resumeAttempts"] = -1
|
|
record["resumeClaimed"] = False
|
|
controller._write_record(record)
|
|
|
|
response = controller.resume(request)
|
|
|
|
saved = controller._read_record(request)
|
|
assert response is not None and response["status"] == "outcome_unknown"
|
|
assert not controller._threads
|
|
assert saved is not None
|
|
assert saved["resumeAttempts"] == -1
|
|
assert saved["phase"] == "credential_journal_invalid"
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_resume_prerequisites_run_before_any_role_login_or_password_helper(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-gate-before-roles"
|
|
)
|
|
events = []
|
|
blocked = _passed_preflight(profile)
|
|
blocked["checks"]["push_validation"]["status"] = "blocked"
|
|
controller.resume_check = lambda _profile: events.append("resume_gate") or blocked
|
|
controller._test_database_uri = lambda _name, role: events.append(("login", role)) or False
|
|
mutations = []
|
|
helper_calls = _install_successful_role_helper(
|
|
monkeypatch, before_send=lambda role: mutations.append(role)
|
|
)
|
|
migrations = []
|
|
controller._run_migration = lambda *_args: migrations.append(True)
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
assert events == ["resume_gate"]
|
|
assert helper_calls == []
|
|
assert mutations == []
|
|
assert migrations == []
|
|
saved = controller._read_record(request)
|
|
assert saved is not None and saved["migrationStarted"] is False
|
|
|
|
|
|
def test_resume_skips_password_changes_for_already_working_roles(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-known-role-logins"
|
|
)
|
|
login_events = []
|
|
controller._test_database_uri = lambda _name, role: login_events.append(role) or True
|
|
helper_calls = _install_successful_role_helper(monkeypatch)
|
|
migrations = []
|
|
controller._run_migration = lambda saved, *_args: migrations.append(
|
|
(saved["credentialRoles"].copy(), tuple(login_events))
|
|
)
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
assert helper_calls == []
|
|
assert login_events == [*_FORGE_ROLE_NAMES, *_FORGE_ROLE_NAMES]
|
|
assert migrations == [({role: "completed" for role in _FORGE_ROLE_NAMES}, tuple(login_events))]
|
|
saved = controller._read_record(request)
|
|
assert saved is not None and saved["migrationStarted"] is True
|
|
|
|
|
|
def test_resume_sets_only_pending_roles_then_proves_both_logins_before_migration(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-set-then-prove"
|
|
)
|
|
events = []
|
|
changed_roles = []
|
|
helper_calls = _install_successful_role_helper(
|
|
monkeypatch,
|
|
before_send=lambda role: (changed_roles.append(role), events.append(("set", role))),
|
|
)
|
|
|
|
def login(_name, role):
|
|
valid = all(item in changed_roles for item in _FORGE_ROLE_NAMES)
|
|
events.append(("login", role, valid))
|
|
return valid
|
|
|
|
controller._test_database_uri = login
|
|
|
|
def migrate(saved, *_args):
|
|
events.append(("migration", saved["credentialRoles"].copy()))
|
|
|
|
controller._run_migration = migrate
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
assert helper_calls == [
|
|
(controller.secret_root, "called"),
|
|
(controller.secret_root, "forge_runtime"),
|
|
(controller.secret_root, "forge_migrator"),
|
|
]
|
|
assert events == [
|
|
("login", "forge_runtime", False),
|
|
("login", "forge_migrator", False),
|
|
("set", "forge_runtime"),
|
|
("set", "forge_migrator"),
|
|
("login", "forge_runtime", True),
|
|
("login", "forge_migrator", True),
|
|
("migration", {role: "completed" for role in _FORGE_ROLE_NAMES}),
|
|
]
|
|
saved = controller._read_record(request)
|
|
assert saved is not None and saved["migrationStarted"] is True
|
|
|
|
|
|
def test_resume_never_replays_intent_role_when_independent_login_fails(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module,
|
|
controller,
|
|
profile,
|
|
"forge-resume-intent-requires-login",
|
|
roles={"forge_runtime": "completed", "forge_migrator": "intent"},
|
|
)
|
|
controller._test_database_uri = lambda _name, role: role == "forge_runtime"
|
|
helper_calls = _install_successful_role_helper(monkeypatch)
|
|
migrations = []
|
|
controller._run_migration = lambda *_args: migrations.append(True)
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
saved = controller._read_record(request)
|
|
assert helper_calls == []
|
|
assert migrations == []
|
|
assert saved is not None
|
|
assert saved["status"] == "outcome_unknown"
|
|
assert saved["phase"] in {"resume_intent_persisted", "role_credentials"}
|
|
assert saved["credentialRoles"] == {"forge_runtime": "completed", "forge_migrator": "unknown"}
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_retry_after_second_role_zero_input_failure_preserves_first_role_completion(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-partial-role-retry"
|
|
)
|
|
changed_roles = []
|
|
helper_calls = []
|
|
helper = types.ModuleType("forge_database_credentials")
|
|
|
|
def set_passwords(secret_root, *, role_state, persist_role_intent,
|
|
persist_role_retryable, persist_role_unknown,
|
|
persist_role_complete):
|
|
helper_calls.append((role_state("forge_runtime"), role_state("forge_migrator")))
|
|
completed = []
|
|
for role in _FORGE_ROLE_NAMES:
|
|
state = role_state(role)
|
|
if state == "completed":
|
|
completed.append(role)
|
|
continue
|
|
assert state in {"pending", "retryable"}
|
|
assert persist_role_intent(role) is True
|
|
if role == "forge_migrator" and len(helper_calls) == 1:
|
|
assert persist_role_retryable(role) is True
|
|
return {"status": "failed", "completedRoles": completed,
|
|
"activeRole": role, "reason": "psql_failed_before_password_input"}
|
|
changed_roles.append(role)
|
|
assert persist_role_complete(role) is True
|
|
completed.append(role)
|
|
return {"status": "completed", "completedRoles": completed, "activeRole": None}
|
|
|
|
helper.set_forge_database_role_passwords = set_passwords
|
|
monkeypatch.setitem(sys.modules, "forge_database_credentials", helper)
|
|
controller._test_database_uri = lambda _name, role: role in changed_roles
|
|
migrations = []
|
|
controller._run_migration = lambda *_args: migrations.append(True)
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
first = controller._read_record(request)
|
|
assert first is not None
|
|
assert first["status"] == "needs_database_credentials"
|
|
assert first["phase"] == "awaiting_external_role_passwords"
|
|
assert first["credentialRoles"] == {"forge_runtime": "completed", "forge_migrator": "retryable"}
|
|
assert first["resumeClaimed"] is False
|
|
assert migrations == []
|
|
|
|
response = controller.resume(request)
|
|
assert response is not None and response["status"] == "running"
|
|
thread = controller._threads[request]
|
|
thread.join(timeout=3)
|
|
assert not thread.is_alive()
|
|
|
|
final = controller._read_record(request)
|
|
assert final is not None
|
|
assert helper_calls == [("pending", "pending"), ("completed", "retryable")]
|
|
assert changed_roles == ["forge_runtime", "forge_migrator"]
|
|
assert final["credentialRoles"] == {"forge_runtime": "completed", "forge_migrator": "completed"}
|
|
assert migrations == [True]
|
|
|
|
|
|
def test_explicit_resume_reconciles_only_a_crashed_credentials_phase(
|
|
controller_module, tmp_path
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module,
|
|
controller,
|
|
profile,
|
|
"forge-resume-crashed-credentials-only",
|
|
status="outcome_unknown",
|
|
phase="role_credentials",
|
|
roles={"forge_runtime": "intent", "forge_migrator": "completed"},
|
|
)
|
|
record["resumeClaimed"] = True
|
|
controller._write_record(record)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
calls = []
|
|
|
|
def hold_resume(saved):
|
|
calls.append(saved["resumeGeneration"])
|
|
entered.set()
|
|
release.wait(timeout=3)
|
|
|
|
controller._execute_after_database_credentials = hold_resume
|
|
try:
|
|
response = controller.resume(request)
|
|
assert response is not None and response["status"] == "running"
|
|
assert entered.wait(timeout=2)
|
|
persisted = controller._read_record(request)
|
|
assert persisted is not None
|
|
assert persisted["resumeGeneration"] == 2
|
|
assert persisted["resumeAttempts"] == 2
|
|
assert persisted["phase"] == "resume_intent_persisted"
|
|
assert controller.resume(request)["status"] == "running"
|
|
assert calls == [2]
|
|
finally:
|
|
release.set()
|
|
thread = controller._threads.get(request)
|
|
if thread is not None:
|
|
thread.join(timeout=2)
|
|
|
|
|
|
def test_old_generation_callback_cannot_overwrite_new_role_journal(
|
|
controller_module, tmp_path
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, _record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-old-callback"
|
|
)
|
|
callbacks = controller._credential_role_callbacks(request, profile, 1)
|
|
assert callbacks["role_state"]("forge_runtime") == "pending"
|
|
newer = controller._read_record(request)
|
|
assert newer is not None
|
|
newer["resumeGeneration"] = 2
|
|
newer["credentialRoles"]["forge_runtime"] = "completed"
|
|
controller._write_record(newer)
|
|
|
|
assert callbacks["persist_role_intent"]("forge_runtime") is False
|
|
saved = controller._read_record(request)
|
|
assert saved is not None
|
|
assert saved["resumeGeneration"] == 2
|
|
assert saved["credentialRoles"]["forge_runtime"] == "completed"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("stored_generation", "stored_phase", "expected_generation"),
|
|
[
|
|
(2, "resume_intent_persisted", 1),
|
|
(2, "role_credentials", 2),
|
|
],
|
|
)
|
|
def test_resume_phase_advance_rejects_stale_generation_or_phase(
|
|
controller_module,
|
|
tmp_path,
|
|
stored_generation,
|
|
stored_phase,
|
|
expected_generation,
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module,
|
|
controller,
|
|
profile,
|
|
f"forge-resume-stale-phase-advance-{stored_generation}-{stored_phase}",
|
|
phase=stored_phase,
|
|
)
|
|
record["resumeGeneration"] = stored_generation
|
|
controller._write_record(record)
|
|
|
|
advanced = controller._advance_resume_to_credentials(
|
|
request, profile, expected_generation
|
|
)
|
|
|
|
saved = controller._read_record(request)
|
|
assert advanced is None
|
|
assert saved is not None
|
|
assert saved["resumeGeneration"] == stored_generation
|
|
assert saved["phase"] == stored_phase
|
|
|
|
|
|
def test_stale_generation_cannot_claim_migration(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module,
|
|
controller,
|
|
profile,
|
|
"forge-resume-stale-migration-claim",
|
|
phase="role_credentials",
|
|
roles={role: "completed" for role in _FORGE_ROLE_NAMES},
|
|
)
|
|
record["resumeGeneration"] = 2
|
|
controller._write_record(record)
|
|
|
|
claim = controller._claim_resume_migration(request, profile, 1)
|
|
|
|
saved = controller._read_record(request)
|
|
assert claim is None
|
|
assert saved is not None
|
|
assert saved["resumeGeneration"] == 2
|
|
assert saved["phase"] == "role_credentials"
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_migration_claim_requires_both_roles_complete(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module,
|
|
controller,
|
|
profile,
|
|
"forge-resume-incomplete-migration-claim",
|
|
phase="role_credentials",
|
|
roles={"forge_runtime": "completed", "forge_migrator": "retryable"},
|
|
)
|
|
controller._write_record(record)
|
|
|
|
claim = controller._claim_resume_migration(request, profile, 1)
|
|
|
|
saved = controller._read_record(request)
|
|
assert claim is None
|
|
assert saved is not None
|
|
assert saved["phase"] == "role_credentials"
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_status_reconciliation_preserves_a_completed_role_journal(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-status-read-preserves-completion"
|
|
)
|
|
record["credentialRoles"] = {role: "completed" for role in _FORGE_ROLE_NAMES}
|
|
controller._write_record(record)
|
|
|
|
response = controller.status(request)
|
|
|
|
saved = controller._read_record(request)
|
|
assert response is not None and response["status"] == "outcome_unknown"
|
|
assert saved is not None
|
|
assert saved["phase"] in {"resume_intent_persisted", "role_credentials"}
|
|
assert saved["credentialRoles"] == {role: "completed" for role in _FORGE_ROLE_NAMES}
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_failed_role_journal_completion_blocks_migration(controller_module, tmp_path, monkeypatch):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-journal-write-failure"
|
|
)
|
|
helper_calls = _install_successful_role_helper(monkeypatch)
|
|
controller._test_database_uri = lambda _name, _role: False
|
|
migrations = []
|
|
controller._run_migration = lambda *_args: migrations.append(True)
|
|
original_write = controller._write_record
|
|
|
|
def fail_completed_journal(saved):
|
|
roles = saved.get("credentialRoles", {})
|
|
if "completed" in roles.values():
|
|
raise OSError("private journal failure")
|
|
original_write(saved)
|
|
|
|
controller._write_record = fail_completed_journal
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
saved = controller._read_record(request)
|
|
assert helper_calls
|
|
assert migrations == []
|
|
assert saved is not None
|
|
assert saved["status"] == "outcome_unknown"
|
|
assert saved["migrationStarted"] is False
|
|
assert "private journal failure" not in json.dumps(controller._response(saved))
|
|
|
|
|
|
def test_final_login_failure_stops_before_migration(controller_module, tmp_path, monkeypatch):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-final-login-failure"
|
|
)
|
|
changed_roles = []
|
|
helper_calls = _install_successful_role_helper(
|
|
monkeypatch, before_send=lambda role: changed_roles.append(role)
|
|
)
|
|
login_calls = []
|
|
|
|
def login(_name, role):
|
|
login_calls.append(role)
|
|
return bool(changed_roles) and role != "forge_migrator"
|
|
|
|
controller._test_database_uri = login
|
|
migrations = []
|
|
controller._run_migration = lambda *_args: migrations.append(True)
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
saved = controller._read_record(request)
|
|
assert helper_calls
|
|
assert login_calls == [*_FORGE_ROLE_NAMES, *_FORGE_ROLE_NAMES]
|
|
assert migrations == []
|
|
assert saved is not None
|
|
assert saved["status"] == "outcome_unknown"
|
|
assert saved["credentialRoles"] == {"forge_runtime": "completed", "forge_migrator": "unknown"}
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_final_resume_gate_failure_preserves_completed_credentials_and_skips_migration(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
_prepare_credential_resume(controller_module, controller, tmp_path, profile)
|
|
request, record = _credential_resume_record(
|
|
controller_module, controller, profile, "forge-resume-final-gate-failure"
|
|
)
|
|
changed_roles = []
|
|
helper_calls = _install_successful_role_helper(
|
|
monkeypatch, before_send=lambda role: changed_roles.append(role)
|
|
)
|
|
controller._test_database_uri = lambda _name, role: role in changed_roles
|
|
gate_calls = []
|
|
|
|
def resume_gate(_profile):
|
|
gate_calls.append(True)
|
|
result = _passed_preflight(profile)
|
|
if len(gate_calls) == 2:
|
|
result["checks"]["exact_main"]["status"] = "blocked"
|
|
return result
|
|
|
|
controller.resume_check = resume_gate
|
|
migrations = []
|
|
controller._run_migration = lambda *_args: migrations.append(True)
|
|
|
|
controller._execute_after_database_credentials(record)
|
|
|
|
saved = controller._read_record(request)
|
|
assert len(gate_calls) == 2
|
|
assert helper_calls
|
|
assert migrations == []
|
|
assert saved is not None
|
|
assert saved["status"] == "outcome_unknown"
|
|
assert saved["credentialRoles"] == {role: "completed" for role in _FORGE_ROLE_NAMES}
|
|
assert saved["migrationStarted"] is False
|
|
|
|
|
|
def test_get_at_credential_pause_never_probes_database_or_loads_password_helper(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
request = controller_module.request_id(profile, "forge-resume-get-no-remote-writes")
|
|
record = {
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "needs_database_credentials",
|
|
"phase": "awaiting_external_role_passwords",
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"credentialRoles": {role: "pending" for role in _FORGE_ROLE_NAMES},
|
|
"resumeGeneration": 0,
|
|
"resumeAttempts": 0,
|
|
"resumeClaimed": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
controller._test_database_uri = lambda *_args: pytest.fail("GET must not run a login probe")
|
|
helper = types.ModuleType("forge_database_credentials")
|
|
helper.set_forge_database_role_passwords = lambda *_args, **_kwargs: pytest.fail(
|
|
"GET must not load or call the password helper"
|
|
)
|
|
monkeypatch.setitem(sys.modules, "forge_database_credentials", helper)
|
|
|
|
response = controller.status(request)
|
|
|
|
assert response is not None and response["status"] == "needs_database_credentials"
|
|
saved = controller._read_record(request)
|
|
assert saved is not None
|
|
assert saved["credentialRoles"] == {role: "pending" for role in _FORGE_ROLE_NAMES}
|
|
assert saved["resumeGeneration"] == 0
|
|
|
|
|
|
def test_runtime_verification_requires_worker_health_not_just_running(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
service_names = (
|
|
"itworx-forge", "postgres", "migrate", "api", "worker", "dashboard"
|
|
)
|
|
containers = dict(zip(service_names, controller_module.CONTAINER_NAMES))
|
|
inspected = {}
|
|
for service, name in containers.items():
|
|
if service == "migrate":
|
|
continue
|
|
inspected[name] = {
|
|
"Config": {
|
|
"Image": controller._expected_image(profile, service, _FORGE_PRIVATE_OWNER),
|
|
"Labels": {
|
|
"com.docker.compose.project": controller_module.PROJECT_NAME,
|
|
"com.docker.compose.service": service,
|
|
**(
|
|
{
|
|
"org.opencontainers.image.source": _FORGE_SOURCE_URL,
|
|
"org.opencontainers.image.revision": profile["revision"],
|
|
}
|
|
if service in {"itworx-forge", "api", "worker", "dashboard"}
|
|
else {}
|
|
),
|
|
},
|
|
},
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
}
|
|
inspected[containers["worker"]]["State"].pop("Health")
|
|
controller._inspect_container = lambda name: inspected.get(name)
|
|
assert not controller._all_services_verified(
|
|
profile, private_package_owner=_FORGE_PRIVATE_OWNER, migration_completed=True
|
|
)
|
|
inspected[containers["worker"]]["State"]["Health"] = {"Status": "healthy"}
|
|
assert controller._all_services_verified(
|
|
profile, private_package_owner=_FORGE_PRIVATE_OWNER, migration_completed=True
|
|
)
|
|
inspected[containers["api"]]["Config"]["Labels"]["org.opencontainers.image.source"] = (
|
|
"https://example.invalid/copied-image"
|
|
)
|
|
assert not controller._all_services_verified(
|
|
profile, private_package_owner=_FORGE_PRIVATE_OWNER, migration_completed=True
|
|
)
|
|
inspected[containers["api"]]["Config"]["Labels"]["org.opencontainers.image.source"] = _FORGE_SOURCE_URL
|
|
inspected[containers["api"]]["Config"]["Labels"]["org.opencontainers.image.revision"] = "d" * 40
|
|
assert not controller._all_services_verified(
|
|
profile, private_package_owner=_FORGE_PRIVATE_OWNER, migration_completed=True
|
|
)
|
|
|
|
|
|
def test_worker_start_waits_for_health_before_starting_later_services(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
services = ("itworx-forge", "postgres", "migrate", "api", "worker", "dashboard")
|
|
containers = dict(zip(services, controller_module.CONTAINER_NAMES))
|
|
worker_states = iter(("starting", "healthy"))
|
|
worker_health_seen = []
|
|
actions = []
|
|
|
|
def inspect(name):
|
|
service = next(service for service, container in containers.items() if container == name)
|
|
health_status = next(worker_states) if service == "worker" else "healthy"
|
|
if service == "worker":
|
|
worker_health_seen.append(health_status)
|
|
image = (
|
|
f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge-frontdoor@{profile['frontdoor_image_digest']}"
|
|
if service == "itworx-forge"
|
|
else f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge@{profile['workspace_image_digest']}"
|
|
)
|
|
return {
|
|
"Config": {
|
|
"Image": image,
|
|
"Labels": {
|
|
"com.docker.compose.project": controller_module.PROJECT_NAME,
|
|
"com.docker.compose.service": service,
|
|
**(
|
|
{
|
|
"org.opencontainers.image.source": _FORGE_SOURCE_URL,
|
|
"org.opencontainers.image.revision": profile["revision"],
|
|
}
|
|
if service in {"itworx-forge", "api", "worker", "dashboard"}
|
|
else {}
|
|
),
|
|
},
|
|
},
|
|
"State": {"Running": True, "Status": "running", "Health": {"Status": health_status}},
|
|
}
|
|
|
|
def compose_action(_compose, action, _source, _environment):
|
|
if action == ["up", "--detach", "--no-deps", "dashboard"]:
|
|
assert worker_health_seen == ["starting", "healthy"]
|
|
actions.append(action)
|
|
return True
|
|
|
|
controller._inspect_container = inspect
|
|
controller._compose_action = compose_action
|
|
controller._all_services_verified = lambda *_args, **_kwargs: True
|
|
monkeypatch.setattr(
|
|
controller_module,
|
|
"time",
|
|
types.SimpleNamespace(monotonic=lambda: 0.0, sleep=lambda _seconds: None, time=lambda: 1),
|
|
)
|
|
request = controller_module.request_id(profile, "forge-worker-health-starting-0001")
|
|
record = {
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "running",
|
|
"phase": "migration_intent",
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
|
|
controller._run_migration(
|
|
record, ["docker", "compose"], tmp_path,
|
|
{"FORGE_PRIVATE_PACKAGE_OWNER": _FORGE_PRIVATE_OWNER},
|
|
)
|
|
|
|
assert worker_health_seen == ["starting", "healthy"]
|
|
assert actions[-1] == ["up", "--detach", "--no-deps", "itworx-forge"]
|
|
assert controller.status(request)["status"] == "deployed"
|
|
|
|
|
|
def test_worker_without_healthcheck_stops_startup_before_later_services(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
services = ("itworx-forge", "postgres", "migrate", "api", "worker", "dashboard")
|
|
containers = dict(zip(services, controller_module.CONTAINER_NAMES))
|
|
actions = []
|
|
|
|
def inspect_worker(name):
|
|
assert name == containers["worker"]
|
|
return {
|
|
"Config": {
|
|
"Image": f"gitea.example.com/{_FORGE_PRIVATE_OWNER}/itworx-forge@{profile['workspace_image_digest']}",
|
|
"Labels": {
|
|
"com.docker.compose.project": controller_module.PROJECT_NAME,
|
|
"com.docker.compose.service": "worker",
|
|
"org.opencontainers.image.source": _FORGE_SOURCE_URL,
|
|
"org.opencontainers.image.revision": profile["revision"],
|
|
},
|
|
},
|
|
"State": {"Running": True, "Status": "running"},
|
|
}
|
|
|
|
actual_wait_health = controller._wait_health
|
|
|
|
def wait_health(container, *, expected_image, timeout_seconds, expected_revision=None):
|
|
if container == containers["worker"]:
|
|
return actual_wait_health(
|
|
container,
|
|
expected_image=expected_image,
|
|
expected_revision=expected_revision,
|
|
timeout_seconds=0.5,
|
|
)
|
|
return True
|
|
|
|
ticks = iter((0.0, 0.0, 1.0))
|
|
controller._inspect_container = inspect_worker
|
|
controller._wait_health = wait_health
|
|
controller._compose_action = lambda _compose, action, _source, _environment: actions.append(action) or True
|
|
controller._all_services_verified = lambda *_args, **_kwargs: True
|
|
monkeypatch.setattr(
|
|
controller_module,
|
|
"time",
|
|
types.SimpleNamespace(monotonic=lambda: next(ticks), sleep=lambda _seconds: None, time=lambda: 1),
|
|
)
|
|
request = controller_module.request_id(profile, "forge-worker-no-health-0001")
|
|
record = {
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "running",
|
|
"phase": "migration_intent",
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
|
|
controller._run_migration(
|
|
record, ["docker", "compose"], tmp_path,
|
|
{"FORGE_PRIVATE_PACKAGE_OWNER": _FORGE_PRIVATE_OWNER},
|
|
)
|
|
|
|
assert actions == [
|
|
["run", "--rm", "--no-deps", "--pull", "never", "migrate"],
|
|
["up", "--detach", "--no-deps", "api"],
|
|
["up", "--detach", "--no-deps", "worker"],
|
|
]
|
|
assert controller.status(request)["status"] == "outcome_unknown"
|
|
|
|
|
|
def test_migration_and_each_service_activation_use_no_deps(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
actions = []
|
|
controller._compose_action = lambda _compose, action, _source, _environment: actions.append(action) or True
|
|
controller._wait_health = lambda *_args, **_kwargs: True
|
|
controller._wait_running = lambda *_args, **_kwargs: True
|
|
controller._all_services_verified = lambda *_args, **_kwargs: True
|
|
request = controller_module.request_id(profile, "forge-first-install-compose")
|
|
record = {
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "running",
|
|
"phase": "migration_intent",
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
controller._run_migration(
|
|
record, ["docker", "compose"], tmp_path,
|
|
{"FORGE_PRIVATE_PACKAGE_OWNER": _FORGE_PRIVATE_OWNER},
|
|
)
|
|
assert actions[0] == ["run", "--rm", "--no-deps", "--pull", "never", "migrate"]
|
|
assert actions[1:] == [
|
|
["up", "--detach", "--no-deps", service]
|
|
for service in ("api", "worker", "dashboard", "itworx-forge")
|
|
]
|
|
assert controller.status(request)["receipt"]["migration"]["result"] == "succeeded"
|
|
|
|
|
|
def test_migration_does_not_start_without_a_safe_protected_owner(controller_module, tmp_path):
|
|
profile = _profile(controller_module)
|
|
controller = _controller(controller_module, tmp_path, profile)
|
|
actions = []
|
|
controller._compose_action = lambda _compose, action, _source, _environment: actions.append(action) or True
|
|
request = controller_module.request_id(profile, "forge-private-owner-required-0001")
|
|
record = {
|
|
"schema": controller_module.STATE_SCHEMA,
|
|
"kind": "request",
|
|
"request": request,
|
|
"repository": controller_module.REPOSITORY,
|
|
"operation": controller_module.OPERATION,
|
|
"profile": profile,
|
|
"status": "running",
|
|
"phase": "credentials_verified",
|
|
"mutationStarted": True,
|
|
"migrationStarted": False,
|
|
"migrationCompleted": False,
|
|
"failureCode": None,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
controller._write_record(record)
|
|
|
|
controller._run_migration(record, ["docker", "compose"], tmp_path, {})
|
|
|
|
current = controller.status(request)
|
|
assert actions == []
|
|
assert current["status"] == "failed"
|
|
assert current["failure"] == {"code": "service_start_failed", "preservedData": True}
|
|
saved = controller._read_record(request)
|
|
assert saved is not None and saved["migrationStarted"] is False
|
|
|
|
|
|
def test_database_uri_password_is_supplied_on_stdin_not_argv_or_environment(
|
|
controller_module, tmp_path, monkeypatch
|
|
):
|
|
profile = _profile(controller_module)
|
|
captured = {}
|
|
|
|
def runner(arguments, **kwargs):
|
|
captured["arguments"] = arguments
|
|
captured.update(kwargs)
|
|
return subprocess.CompletedProcess(
|
|
arguments, 0, stdout=b"forge_runtime|forge\n", stderr=b""
|
|
)
|
|
|
|
controller = _controller(controller_module, tmp_path, profile, command_runner=runner)
|
|
password = b"not-a-real-secret-value"
|
|
uri = b"postgresql://forge_runtime:not-a-real-secret-value@postgres:5432/forge"
|
|
monkeypatch.setattr(controller_module, "read_database_uri", lambda *_args: uri)
|
|
assert controller._test_database_uri("runtime_database_url", "forge_runtime")
|
|
assert password not in " ".join(captured["arguments"]).encode()
|
|
assert password in captured["input"]
|
|
assert "env" not in captured
|
|
assert captured["capture_output"] is True
|
|
|
|
|
|
class _Response:
|
|
def __init__(self, body=b"", status=200, headers=None):
|
|
self.body = body
|
|
self.status = status
|
|
self.headers = headers or {}
|
|
self.read_sizes = []
|
|
|
|
def read(self, size=-1):
|
|
self.read_sizes.append(size)
|
|
return self.body if size < 0 else self.body[:size]
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return None
|
|
|
|
|
|
class _StaticOpener:
|
|
def __init__(self, outcome):
|
|
self.outcome = outcome
|
|
self.requests = []
|
|
|
|
def open(self, request, timeout=None):
|
|
self.requests.append((request, timeout))
|
|
if isinstance(self.outcome, Exception):
|
|
raise self.outcome
|
|
return self.outcome
|
|
|
|
|
|
def _registry_artifact(labels):
|
|
config = json.dumps(
|
|
{"architecture": "amd64", "os": "linux", "config": {"Labels": labels}},
|
|
separators=(",", ":"),
|
|
).encode()
|
|
config_digest = "sha256:" + hashlib.sha256(config).hexdigest()
|
|
manifest = json.dumps(
|
|
{
|
|
"schemaVersion": 2,
|
|
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
|
"config": {
|
|
"mediaType": "application/vnd.oci.image.config.v1+json",
|
|
"digest": config_digest,
|
|
"size": len(config),
|
|
},
|
|
"layers": [],
|
|
},
|
|
separators=(",", ":"),
|
|
).encode()
|
|
manifest_digest = "sha256:" + hashlib.sha256(manifest).hexdigest()
|
|
return manifest_digest, manifest, config_digest, config
|
|
|
|
|
|
class _RegistryOpener:
|
|
def __init__(self, repository, manifest_digest, manifest, config_digest, config):
|
|
self.repository = repository
|
|
self.manifest_digest = manifest_digest
|
|
self.manifest = manifest
|
|
self.config_digest = config_digest
|
|
self.config = config
|
|
self.requests = []
|
|
|
|
def open(self, request, timeout=None):
|
|
self.requests.append((request, timeout))
|
|
path = urlparse(request.full_url).path
|
|
method = request.get_method()
|
|
root = f"/v2/{self.repository}"
|
|
if path == f"{root}/manifests/{self.manifest_digest}" and method == "GET":
|
|
return _Response(
|
|
self.manifest,
|
|
headers={"Docker-Content-Digest": self.manifest_digest},
|
|
)
|
|
if path == f"{root}/blobs/{self.config_digest}":
|
|
return _Response(self.config if method == "GET" else b"", status=200)
|
|
raise AssertionError(f"unexpected registry request {method} {path}")
|
|
|
|
|
|
def _challenge_error(repository, *, scope=None):
|
|
scope = scope or f"repository:{repository}:pull"
|
|
headers = Message()
|
|
headers["WWW-Authenticate"] = (
|
|
'Bearer realm="https://gitea.example.com/v2/token",'
|
|
f'service="container_registry",scope="{scope}"'
|
|
)
|
|
return HTTPError("https://gitea.example.com/v2/", 401, "Unauthorized", headers, io.BytesIO())
|
|
|
|
|
|
def test_registry_requires_anonymous_get_denial_with_exact_repository_scope(
|
|
status_module, monkeypatch
|
|
):
|
|
repository = "test-owner/test-image"
|
|
digest = "sha256:" + "a" * 64
|
|
public_opener = _StaticOpener(_Response(b"public manifest", status=200))
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: public_opener)
|
|
assert status_module._forge_registry_challenge(repository, f"manifests/{digest}") is None
|
|
assert public_opener.requests[0][0].get_method() == "GET"
|
|
|
|
private_opener = _StaticOpener(_challenge_error(repository))
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: private_opener)
|
|
token_url = status_module._forge_registry_challenge(repository, f"manifests/{digest}")
|
|
assert token_url is not None
|
|
assert urlparse(token_url).hostname == "gitea.example.com"
|
|
assert urlparse(token_url).path == "/v2/token"
|
|
assert "repository%3Atest-owner%2Ftest-image%3Apull" in token_url
|
|
|
|
wrong_scope = _StaticOpener(
|
|
_challenge_error(repository, scope="repository:someone-else/image:pull")
|
|
)
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: wrong_scope)
|
|
assert status_module._forge_registry_challenge(repository, f"blobs/{digest}") is None
|
|
|
|
|
|
def test_registry_token_exchange_uses_fixed_gitea_token_and_scoped_challenge(
|
|
status_module, monkeypatch
|
|
):
|
|
repository = f"{_FORGE_PRIVATE_OWNER}/test-image"
|
|
digest = "sha256:" + "a" * 64
|
|
monkeypatch.setattr(
|
|
status_module,
|
|
"_forge_registry_challenge",
|
|
lambda _repository, _path: "https://gitea.example.com/v2/token?scope=repository%3Atest-owner%2Ftest-image%3Apull",
|
|
)
|
|
opener = _StaticOpener(_Response(json.dumps({"token": "t" * 40}).encode()))
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: opener)
|
|
assert status_module._forge_registry_auth(repository, digest, b"fixed-gitea-token") == "t" * 40
|
|
request = opener.requests[0][0]
|
|
assert request.get_method() == "GET"
|
|
assert request.get_header("Authorization") == "Basic " + base64.b64encode(
|
|
b"jens:fixed-gitea-token"
|
|
).decode()
|
|
|
|
|
|
def test_registry_attestation_uses_protected_owner_and_binds_both_image_roles(
|
|
status_module, monkeypatch
|
|
):
|
|
config = _forge_config()
|
|
monkeypatch.setattr(status_module, "_load_forge_bootstrap_config", lambda _path: config)
|
|
monkeypatch.setattr(
|
|
status_module,
|
|
"_gitea_validation_settings",
|
|
lambda: ("https://gitea.example.com/api/v1", b"fixed-token"),
|
|
)
|
|
owner_checks = []
|
|
image_checks = []
|
|
|
|
def owner_is_private(owner, *_args):
|
|
owner_checks.append(owner)
|
|
return True
|
|
|
|
def image_pullable(repository, digest, _token, **kwargs):
|
|
image_checks.append((repository, digest, kwargs))
|
|
return True
|
|
|
|
profile = {
|
|
"revision": "a" * 40,
|
|
"workspace_image_digest": "sha256:" + "a" * 64,
|
|
"frontdoor_image_digest": "sha256:" + "b" * 64,
|
|
}
|
|
result, proof = status_module._forge_image_registry_attestation(
|
|
profile,
|
|
owner_is_private=owner_is_private,
|
|
image_pullable=image_pullable,
|
|
)
|
|
assert result == "passed"
|
|
assert owner_checks == [_FORGE_PRIVATE_OWNER]
|
|
assert [item[:2] for item in image_checks] == [
|
|
(f"{_FORGE_PRIVATE_OWNER}/itworx-forge", profile["workspace_image_digest"]),
|
|
(f"{_FORGE_PRIVATE_OWNER}/itworx-forge-frontdoor", profile["frontdoor_image_digest"]),
|
|
]
|
|
assert all(
|
|
item[2]["expected_revision"] == profile["revision"]
|
|
and item[2]["approved_owner"] == _FORGE_PRIVATE_OWNER
|
|
for item in image_checks
|
|
)
|
|
assert [item[2]["expected_package"] for item in image_checks] == [
|
|
"itworx-forge",
|
|
"itworx-forge-frontdoor",
|
|
]
|
|
assert proof == {
|
|
"owner": _FORGE_PRIVATE_OWNER,
|
|
"ownerVisibility": "private",
|
|
"workspaceDigest": profile["workspace_image_digest"],
|
|
"frontdoorDigest": profile["frontdoor_image_digest"],
|
|
"workspacePullable": True,
|
|
"frontdoorPullable": True,
|
|
"workspaceSource": _FORGE_SOURCE_URL,
|
|
"workspaceRevision": profile["revision"],
|
|
"frontdoorSource": _FORGE_SOURCE_URL,
|
|
"frontdoorRevision": profile["revision"],
|
|
}
|
|
|
|
|
|
def test_registry_attestation_stays_blocked_without_protected_owner(status_module, monkeypatch):
|
|
monkeypatch.setattr(status_module, "_load_forge_bootstrap_config", lambda _path: None)
|
|
result, proof = status_module._forge_image_registry_attestation(
|
|
{
|
|
"revision": "a" * 40,
|
|
"workspace_image_digest": "sha256:" + "a" * 64,
|
|
"frontdoor_image_digest": "sha256:" + "b" * 64,
|
|
}
|
|
)
|
|
assert result == "blocked"
|
|
assert proof is None
|
|
|
|
|
|
def test_registry_owner_limited_visibility_is_not_private(status_module, monkeypatch):
|
|
def owner_result(visibility):
|
|
return _StaticOpener(
|
|
_Response(
|
|
json.dumps({"login": "jens", "visibility": visibility}).encode()
|
|
)
|
|
)
|
|
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: owner_result("limited"))
|
|
assert not status_module._forge_registry_owner_is_private(
|
|
"jens", "https://gitea.example.com/api/v1", b"gitea-token"
|
|
)
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: owner_result("private"))
|
|
assert status_module._forge_registry_owner_is_private(
|
|
"jens", "https://gitea.example.com/api/v1", b"gitea-token"
|
|
)
|
|
assert not status_module._forge_registry_owner_is_private(
|
|
"Owner.With.Dot", "https://gitea.example.com/api/v1", b"gitea-token"
|
|
)
|
|
|
|
|
|
def test_registry_config_blob_must_match_digest_source_and_revision(status_module, monkeypatch):
|
|
repository = f"{_FORGE_PRIVATE_OWNER}/itworx-forge"
|
|
revision = "a" * 40
|
|
matching_labels = {
|
|
"org.opencontainers.image.source": _FORGE_SOURCE_URL,
|
|
"org.opencontainers.image.revision": revision,
|
|
}
|
|
|
|
def verify(labels, *, tampered_config=False):
|
|
manifest_digest, manifest, config_digest, config = _registry_artifact(labels)
|
|
opener = _RegistryOpener(
|
|
repository,
|
|
manifest_digest,
|
|
manifest,
|
|
config_digest,
|
|
(b" " + config[1:]) if tampered_config else config,
|
|
)
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: opener)
|
|
monkeypatch.setattr(
|
|
status_module,
|
|
"_forge_registry_challenge",
|
|
lambda _repository, _path: "https://gitea.example.com/v2/token",
|
|
)
|
|
monkeypatch.setattr(
|
|
status_module, "_forge_registry_auth", lambda *_args, **_kwargs: "b" * 40
|
|
)
|
|
result = status_module._forge_registry_image_pullable(
|
|
repository,
|
|
manifest_digest,
|
|
b"gitea-token",
|
|
approved_owner=_FORGE_PRIVATE_OWNER,
|
|
expected_package="itworx-forge",
|
|
expected_revision=revision,
|
|
)
|
|
return result, opener
|
|
|
|
accepted, opener = verify(matching_labels)
|
|
assert accepted
|
|
config_gets = [
|
|
request for request, _timeout in opener.requests
|
|
if request.get_method() == "GET" and "/blobs/" in request.full_url
|
|
]
|
|
assert len(config_gets) == 1
|
|
assert config_gets[0].get_header("Authorization") == "Bearer " + "b" * 40
|
|
|
|
wrong_source = {**matching_labels, "org.opencontainers.image.source": "https://example.invalid/source"}
|
|
wrong_revision = {**matching_labels, "org.opencontainers.image.revision": "d" * 40}
|
|
assert not verify(wrong_source)[0]
|
|
assert not verify(wrong_revision)[0]
|
|
assert not verify(matching_labels, tampered_config=True)[0]
|
|
|
|
|
|
def test_registry_pullability_rejects_wrong_owner_and_package_role(status_module, monkeypatch):
|
|
owner = _FORGE_PRIVATE_OWNER
|
|
repository = f"{owner}/itworx-forge"
|
|
labels = {
|
|
"org.opencontainers.image.source": _FORGE_SOURCE_URL,
|
|
"org.opencontainers.image.revision": "a" * 40,
|
|
}
|
|
manifest_digest, manifest, config_digest, config = _registry_artifact(labels)
|
|
opener = _RegistryOpener(repository, manifest_digest, manifest, config_digest, config)
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: opener)
|
|
monkeypatch.setattr(
|
|
status_module,
|
|
"_forge_registry_challenge",
|
|
lambda *_args: "https://gitea.example.com/v2/token",
|
|
)
|
|
monkeypatch.setattr(status_module, "_forge_registry_auth", lambda *_args, **_kwargs: "b" * 40)
|
|
|
|
assert not status_module._forge_registry_image_pullable(
|
|
"other-owner/itworx-forge",
|
|
manifest_digest,
|
|
b"gitea-token",
|
|
approved_owner=owner,
|
|
expected_package="itworx-forge",
|
|
expected_revision="a" * 40,
|
|
)
|
|
wrong_role_repository = f"{owner}/unexpected-image"
|
|
wrong_role_opener = _RegistryOpener(
|
|
wrong_role_repository, manifest_digest, manifest, config_digest, config
|
|
)
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: wrong_role_opener)
|
|
assert not status_module._forge_registry_image_pullable(
|
|
wrong_role_repository,
|
|
manifest_digest,
|
|
b"gitea-token",
|
|
approved_owner=owner,
|
|
expected_package="itworx-forge",
|
|
expected_revision="a" * 40,
|
|
)
|
|
|
|
|
|
def test_authentik_readiness_requires_exact_mapping_key_and_redirect_contract(
|
|
status_module, monkeypatch
|
|
):
|
|
auth_base = "https://auth.example-host.com"
|
|
issuer = status_module._FORGE_OIDC_ISSUER
|
|
mapping_ids = {
|
|
"openid": "01234567-89ab-4cde-8fab-0123456789ab",
|
|
"email": "21234567-89ab-4cde-8fab-0123456789ab",
|
|
"profile": "31234567-89ab-4cde-8fab-0123456789ab",
|
|
}
|
|
provider_id = "1234567890"
|
|
signing_key_id = "11234567-89ab-4cde-8fab-0123456789ab"
|
|
config = {
|
|
"issuer": issuer,
|
|
"audience": "itworx-forge",
|
|
"client_id": "itworx-forge",
|
|
"authentik_api_url": auth_base,
|
|
"launch_url": "https://forge.example.com",
|
|
"authorization_flow": "default-provider-authorization-implicit-consent",
|
|
"invalidation_flow": "default-provider-invalidation-flow",
|
|
"authentik_openid_mapping_id": mapping_ids["openid"],
|
|
"authentik_email_mapping_id": mapping_ids["email"],
|
|
"authentik_profile_mapping_id": mapping_ids["profile"],
|
|
"authentik_provider_id": provider_id,
|
|
"authentik_signing_key_id": signing_key_id,
|
|
}
|
|
certificate = _forge_oidc_test_certificate()
|
|
fingerprint = certificate["fingerprint"]
|
|
jwk = certificate["jwk"]
|
|
assert isinstance(jwk, dict)
|
|
kid = jwk["kid"]
|
|
payloads = {
|
|
issuer + ".well-known/openid-configuration": {
|
|
"issuer": issuer,
|
|
"jwks_uri": issuer + "jwks/",
|
|
"id_token_signing_alg_values_supported": ["RS256"],
|
|
"scopes_supported": ["openid", "email", "profile"],
|
|
"response_types_supported": ["code"],
|
|
"grant_types_supported": ["authorization_code"],
|
|
},
|
|
f"{auth_base}/api/v3/core/applications/itworx-forge/": {
|
|
"pk": "41234567-89ab-4cde-8fab-0123456789ab",
|
|
"slug": "itworx-forge",
|
|
"provider": int(provider_id),
|
|
"meta_launch_url": "https://forge.example.com",
|
|
"meta_description": "Managed by AppOps Forge OIDC v1.",
|
|
},
|
|
f"{auth_base}/api/v3/providers/oauth2/{provider_id}/": {
|
|
"pk": int(provider_id),
|
|
"client_secret": "a" * 40,
|
|
"client_id": "itworx-forge",
|
|
"client_type": "confidential",
|
|
"redirect_uris": [{
|
|
"matching_mode": "strict",
|
|
"url": "https://forge.example.com/auth/callback",
|
|
}],
|
|
"issuer_mode": "per_provider",
|
|
"include_claims_in_id_token": True,
|
|
"sub_mode": "hashed_user_id",
|
|
"authorization_flow": config["authorization_flow"],
|
|
"invalidation_flow": config["invalidation_flow"],
|
|
"property_mappings": [mapping_ids["openid"], mapping_ids["email"], mapping_ids["profile"]],
|
|
"signing_key": signing_key_id,
|
|
},
|
|
f"{auth_base}/api/v3/propertymappings/provider/scope/{mapping_ids['openid']}/": {
|
|
"pk": mapping_ids["openid"],
|
|
"managed": status_module._FORGE_OPENID_MAPPING_MANAGED,
|
|
"scope_name": "openid",
|
|
"expression": (
|
|
"# This scope is required by the OpenID-spec, and must as such exist in authentik.\n"
|
|
"# The scope by itself does not grant any information\nreturn {}"
|
|
),
|
|
},
|
|
f"{auth_base}/api/v3/propertymappings/provider/scope/{mapping_ids['email']}/": {
|
|
"pk": mapping_ids["email"],
|
|
"managed": status_module._FORGE_EMAIL_MAPPING_MANAGED,
|
|
"scope_name": "email",
|
|
"expression": 'return {\n "email": request.user.email,\n "email_verified": False\n}',
|
|
},
|
|
f"{auth_base}/api/v3/propertymappings/provider/scope/{mapping_ids['profile']}/": {
|
|
"pk": mapping_ids["profile"],
|
|
"name": status_module._FORGE_PROFILE_MAPPING_NAME,
|
|
"scope_name": "profile",
|
|
"expression": status_module._FORGE_PROFILE_MAPPING_EXPRESSION,
|
|
},
|
|
f"{auth_base}/api/v3/crypto/certificatekeypairs/{signing_key_id}/": {
|
|
"pk": signing_key_id,
|
|
"key_type": "RSA",
|
|
"private_key_available": True,
|
|
"fingerprint_sha256": fingerprint,
|
|
},
|
|
f"{auth_base}/api/v3/crypto/certificatekeypairs/{signing_key_id}/view_certificate/": {
|
|
"data": certificate["data"],
|
|
},
|
|
issuer + "jwks/": {
|
|
"keys": [jwk]
|
|
},
|
|
}
|
|
group_ids = {
|
|
"forge-viewers": "51234567-89ab-4cde-8fab-0123456789ab",
|
|
"forge-operators": "52234567-89ab-4cde-8fab-0123456789ab",
|
|
"forge-owners": "53234567-89ab-4cde-8fab-0123456789ab",
|
|
}
|
|
group_rows = {}
|
|
for index, name in enumerate(status_module._FORGE_GROUP_NAMES, start=1):
|
|
group = {
|
|
"pk": group_ids[name],
|
|
"name": name,
|
|
"is_superuser": False,
|
|
"attributes": {},
|
|
"roles": [],
|
|
"parents": [],
|
|
"children": [],
|
|
# Membership is preserved and is not a privilege field on this gate.
|
|
"users": [17000 + index, 18000 + index],
|
|
}
|
|
group_rows[name] = group
|
|
payloads[f"{auth_base}/api/v3/core/groups/?name={name}&page_size=2"] = {
|
|
"results": [group]
|
|
}
|
|
|
|
class _MappedOpener:
|
|
def __init__(self):
|
|
self.requests = []
|
|
self.responses = {}
|
|
|
|
def open(self, request, timeout=None):
|
|
self.requests.append((request, timeout))
|
|
value = payloads.get(request.full_url)
|
|
if value is None:
|
|
raise AssertionError(f"unexpected Authentik request {request.full_url}")
|
|
response = _Response(json.dumps(value).encode())
|
|
self.responses[request.full_url] = response
|
|
return response
|
|
|
|
opener = _MappedOpener()
|
|
monkeypatch.setattr(status_module, "_load_forge_bootstrap_config", lambda _path: config)
|
|
delivered_secret = {"value": b"a" * 40}
|
|
secret_reads = {"token": 0, "client": 0}
|
|
|
|
def private_secret(path):
|
|
assert path.name == "authentik_read_token.secret"
|
|
secret_reads["token"] += 1
|
|
return b"r" * 40
|
|
|
|
def forge_secret(_root, name):
|
|
assert name == "oidc_client_secret"
|
|
secret_reads["client"] += 1
|
|
return delivered_secret["value"]
|
|
|
|
monkeypatch.setattr(status_module, "_private_secret_file", private_secret)
|
|
monkeypatch.setattr(status_module, "_read_forge_printable_secret", forge_secret)
|
|
monkeypatch.setattr(status_module, "build_opener", lambda *_args: opener)
|
|
status, proof = status_module._forge_oidc_attestation()
|
|
assert status == "passed"
|
|
assert proof == {
|
|
"issuer": issuer,
|
|
"audience": "itworx-forge",
|
|
"clientId": "itworx-forge",
|
|
"clientType": "confidential",
|
|
"callback": "https://forge.example.com/auth/callback",
|
|
"applicationId": "41234567-89ab-4cde-8fab-0123456789ab",
|
|
"providerId": provider_id,
|
|
"clientSecretMatched": True,
|
|
"authorizationFlowId": config["authorization_flow"],
|
|
"invalidationFlowId": config["invalidation_flow"],
|
|
"roleGroups": ["forge-operators", "forge-owners", "forge-viewers"],
|
|
"scopeMappingIds": mapping_ids,
|
|
"scopeMappingExpressionSha256": status_module._FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256,
|
|
"profileMappingName": status_module._FORGE_PROFILE_MAPPING_NAME,
|
|
"standardMappingsManaged": True,
|
|
"supportedScopes": ["email", "openid", "profile"],
|
|
"authorizationCodeFlowSupported": True,
|
|
"signingKeyId": signing_key_id,
|
|
"signingKeyType": "RSA",
|
|
"signingKeyPrivateAvailable": True,
|
|
"signingKeyFingerprintSha256": fingerprint,
|
|
"signingKeyKid": kid,
|
|
"tokenSigningAlgorithm": "RS256",
|
|
"jwksKid": kid,
|
|
}
|
|
assert group_rows["forge-operators"]["users"] == [17002, 18002]
|
|
assert "a" * 40 not in json.dumps(proof)
|
|
|
|
disk_reads_before_held_check = dict(secret_reads)
|
|
held_candidate = {
|
|
"config": config,
|
|
"oidc_client_secret": b"a" * 40,
|
|
"authentik_read_token": b"r" * 40,
|
|
}
|
|
assert status_module._forge_oidc_attestation(held_candidate=held_candidate) == (
|
|
"passed",
|
|
proof,
|
|
)
|
|
assert secret_reads == disk_reads_before_held_check
|
|
|
|
def assert_group_mutation_is_blocked(
|
|
field: str, value: object = None, *, remove: bool = False
|
|
):
|
|
group = group_rows["forge-operators"]
|
|
original = group.get(field)
|
|
if remove:
|
|
group.pop(field, None)
|
|
else:
|
|
group[field] = value
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
if original is not None or field in group:
|
|
group[field] = original
|
|
else:
|
|
group.pop(field, None)
|
|
|
|
# Group identity must be a canonical, nonnil RFC UUID string. False and
|
|
# integer IDs must not impersonate one.
|
|
for invalid_pk in (
|
|
17,
|
|
False,
|
|
"00000000-0000-0000-0000-000000000000",
|
|
group_ids["forge-operators"].upper(),
|
|
):
|
|
assert_group_mutation_is_blocked("pk", invalid_pk)
|
|
for missing_field in ("is_superuser", "attributes", "roles", "parents", "children"):
|
|
assert_group_mutation_is_blocked(missing_field, remove=True)
|
|
for invalid_superuser in (True, 0, "false"):
|
|
assert_group_mutation_is_blocked("is_superuser", invalid_superuser)
|
|
for invalid_attributes in ({"owner": "unmanaged"}, []):
|
|
assert_group_mutation_is_blocked("attributes", invalid_attributes)
|
|
for relation in ("roles", "parents", "children"):
|
|
assert_group_mutation_is_blocked(relation, [group_ids["forge-viewers"]])
|
|
|
|
import builtins
|
|
|
|
original_import = builtins.__import__
|
|
|
|
def without_cryptography(name, *args, **kwargs):
|
|
if name == "cryptography" or name.startswith("cryptography."):
|
|
raise ImportError("test dependency is unavailable")
|
|
return original_import(name, *args, **kwargs)
|
|
|
|
with monkeypatch.context() as crypto_patch:
|
|
crypto_patch.setattr(builtins, "__import__", without_cryptography)
|
|
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
|
|
|
public_urls = {
|
|
issuer + ".well-known/openid-configuration",
|
|
issuer + "jwks/",
|
|
}
|
|
assert all(
|
|
request.get_header("Authorization") is None
|
|
if request.full_url in public_urls
|
|
else request.get_header("Authorization") == "Bearer " + "r" * 40
|
|
for request, _timeout in opener.requests
|
|
)
|
|
requested_urls = [request.full_url for request, _timeout in opener.requests]
|
|
assert f"{auth_base}/api/v3/core/applications/itworx-forge/" in requested_urls
|
|
assert not any(
|
|
"/api/v3/core/applications/?" in url for url in requested_urls
|
|
)
|
|
certificate_url = f"{auth_base}/api/v3/crypto/certificatekeypairs/{signing_key_id}/view_certificate/"
|
|
assert opener.responses[certificate_url].read_sizes == [65537]
|
|
assert kid == "8cf5f32a6c6d9a876fd0284980a25ce0"
|
|
|
|
provider_url = f"{auth_base}/api/v3/providers/oauth2/{provider_id}/"
|
|
# Authentik exposes property_mappings as an unordered many-to-many list.
|
|
# Exact membership matters; response order does not.
|
|
payloads[provider_url]["property_mappings"] = [
|
|
mapping_ids["profile"], mapping_ids["openid"], mapping_ids["email"]
|
|
]
|
|
assert status_module._forge_oidc_attestation()[0] == "passed"
|
|
payloads[provider_url]["property_mappings"] = [
|
|
mapping_ids["openid"], mapping_ids["email"], mapping_ids["profile"]
|
|
]
|
|
profile_mapping_url = f"{auth_base}/api/v3/propertymappings/provider/scope/{mapping_ids['profile']}/"
|
|
payloads[profile_mapping_url]["expression"] = status_module._FORGE_PROFILE_MAPPING_EXPRESSION.replace("\n", "\r\n")
|
|
assert status_module._forge_oidc_attestation()[0] == "passed"
|
|
payloads[profile_mapping_url]["expression"] = status_module._FORGE_PROFILE_MAPPING_EXPRESSION
|
|
payloads[provider_url]["redirect_uris"][0]["redirect_uri_type"] = "authorization"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
del payloads[provider_url]["redirect_uris"][0]["redirect_uri_type"]
|
|
|
|
# A UUID application primary key and its separately pinned numeric provider
|
|
# identity are different Authentik resource types.
|
|
application_url = f"{auth_base}/api/v3/core/applications/itworx-forge/"
|
|
payloads[application_url]["pk"] = 8
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[application_url]["pk"] = "41234567-89ab-4cde-8fab-0123456789ab"
|
|
payloads[application_url]["provider"] = 8
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[application_url]["provider"] = int(provider_id)
|
|
config["authentik_provider_id"] = "8"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
config["authentik_provider_id"] = provider_id
|
|
config["authentik_provider_id"] = "1" * 11
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
config["authentik_provider_id"] = provider_id
|
|
|
|
# Reject legacy group-only, stock-profile/incorrect profile mapping, extra
|
|
# provider scopes, and any scope-to-ID swaps.
|
|
payloads[provider_url]["property_mappings"] = [mapping_ids["profile"]]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[provider_url]["property_mappings"] = [
|
|
mapping_ids["openid"], mapping_ids["email"], mapping_ids["profile"],
|
|
"41234567-89ab-4cde-8fab-0123456789ab",
|
|
]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[provider_url]["property_mappings"] = [
|
|
mapping_ids["openid"], mapping_ids["email"], mapping_ids["profile"]
|
|
]
|
|
payloads[profile_mapping_url]["name"] = "User profile"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[profile_mapping_url]["name"] = status_module._FORGE_PROFILE_MAPPING_NAME
|
|
payloads[profile_mapping_url]["managed"] = ""
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
del payloads[profile_mapping_url]["managed"]
|
|
original_openid_id = config["authentik_openid_mapping_id"]
|
|
config["authentik_openid_mapping_id"] = mapping_ids["email"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
config["authentik_openid_mapping_id"] = original_openid_id
|
|
config["authentik_email_mapping_id"] = config["authentik_openid_mapping_id"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
config["authentik_email_mapping_id"] = mapping_ids["email"]
|
|
|
|
discovery_url = issuer + ".well-known/openid-configuration"
|
|
payloads[discovery_url]["scopes_supported"] = ["openid", "email"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[discovery_url]["scopes_supported"] = ["openid", "email", "profile"]
|
|
payloads[discovery_url]["response_types_supported"] = []
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[discovery_url]["response_types_supported"] = ["code"]
|
|
payloads[discovery_url]["grant_types_supported"] = []
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[discovery_url]["grant_types_supported"] = ["authorization_code"]
|
|
payloads[discovery_url].pop("scopes_supported")
|
|
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
|
payloads[discovery_url]["scopes_supported"] = ["openid", "email", "profile"]
|
|
|
|
# Certificate, authenticated fingerprint and RSA JWKS public numbers form
|
|
# one binding; metadata booleans alone cannot open this check.
|
|
keypair_url = f"{auth_base}/api/v3/crypto/certificatekeypairs/{signing_key_id}/"
|
|
jwks_url = issuer + "jwks/"
|
|
modern_kid = base64.urlsafe_b64encode(bytes(range(64))).decode("ascii").rstrip("=")
|
|
payloads[jwks_url]["keys"][0]["kid"] = modern_kid
|
|
modern_status, modern_proof = status_module._forge_oidc_attestation()
|
|
assert modern_status == "passed"
|
|
assert modern_proof is not None
|
|
assert modern_proof["signingKeyKid"] == modern_kid
|
|
assert modern_proof["jwksKid"] == modern_kid
|
|
payloads[jwks_url]["keys"][0]["kid"] = "A" * 85 + "B"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[jwks_url]["keys"][0]["kid"] = kid
|
|
|
|
alternate = _forge_oidc_test_certificate()
|
|
payloads[certificate_url]["data"] = alternate["data"]
|
|
payloads[keypair_url]["fingerprint_sha256"] = alternate["fingerprint"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[certificate_url]["data"] = certificate["data"]
|
|
payloads[keypair_url]["fingerprint_sha256"] = fingerprint
|
|
|
|
payloads[keypair_url]["fingerprint_sha256"] = alternate["fingerprint"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[keypair_url]["fingerprint_sha256"] = fingerprint
|
|
|
|
payloads[jwks_url]["keys"][0]["n"] = alternate["jwk"]["n"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[jwks_url]["keys"][0] = jwk
|
|
|
|
payloads[jwks_url]["keys"][0]["e"] = "Aw"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[jwks_url]["keys"][0] = jwk
|
|
|
|
payloads[keypair_url]["key_type"] = "EC"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[keypair_url]["key_type"] = "RSA"
|
|
|
|
ec_certificate = _forge_oidc_test_certificate(elliptic_curve=True)
|
|
payloads[certificate_url]["data"] = ec_certificate["data"]
|
|
payloads[keypair_url]["fingerprint_sha256"] = ec_certificate["fingerprint"]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[certificate_url]["data"] = certificate["data"]
|
|
payloads[keypair_url]["fingerprint_sha256"] = fingerprint
|
|
|
|
payloads[jwks_url]["keys"] = [jwk, {**jwk, "kid": "0" * 32}]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[jwks_url]["keys"] = [jwk]
|
|
|
|
# Duplicate KIDs are ambiguous even when only one RSA key matches the
|
|
# certificate; relying party key selection is by KID, not modulus.
|
|
payloads[jwks_url]["keys"] = [jwk, {**alternate["jwk"], "kid": kid}]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[jwks_url]["keys"] = [jwk]
|
|
|
|
weak_certificate = _forge_oidc_test_certificate(rsa_key_size=1024)
|
|
payloads[certificate_url]["data"] = weak_certificate["data"]
|
|
payloads[keypair_url]["fingerprint_sha256"] = weak_certificate["fingerprint"]
|
|
payloads[jwks_url]["keys"] = [weak_certificate["jwk"]]
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[certificate_url]["data"] = certificate["data"]
|
|
payloads[keypair_url]["fingerprint_sha256"] = fingerprint
|
|
payloads[jwks_url]["keys"] = [jwk]
|
|
|
|
payloads[provider_url]["client_secret"] = "mismatched-client-secret"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[provider_url]["client_secret"] = "a" * 40
|
|
del payloads[provider_url]["client_secret"]
|
|
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
|
payloads[provider_url]["client_secret"] = "a" * 40
|
|
delivered_secret["value"] = None
|
|
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
|
delivered_secret["value"] = b"a" * 40
|
|
profile_mapping_url = f"{auth_base}/api/v3/propertymappings/provider/scope/{mapping_ids['profile']}/"
|
|
payloads[profile_mapping_url]["expression"] = "return {'groups': ['admin']}"
|
|
assert status_module._forge_oidc_attestation() == ("blocked", None)
|
|
payloads[profile_mapping_url]["expression"] = status_module._FORGE_PROFILE_MAPPING_EXPRESSION
|
|
payloads[certificate_url]["data"] = "x" * (64 * 1024)
|
|
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
|
assert opener.responses[certificate_url].read_sizes[-1] == 65537
|