import hashlib
import json
from collections.abc import Iterator
from datetime import UTC, datetime, timedelta
from uuid import uuid4

import pytest
from pydantic import ValidationError
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session

from modelforge_api.domain.dispatch_context import DispatchContext
from modelforge_api.domain.lab import LabRerankingInput
from modelforge_api.domain.serving import (
    CapabilityExperimentCreate,
    EmbeddingInvokeRequest,
    RerankingInvokeRequest,
    ServiceClientCreate,
)
from modelforge_api.persistence.models import (
    Base,
    DeploymentCandidate,
    GatewayRequest,
    SchedulerPolicyRevision,
    ServiceClient,
    ServingJob,
)
from modelforge_api.services.manifest_registry import ManifestRegistry
from modelforge_api.services.serving import ServingError
from modelforge_api.services.transient_payloads import MemoryPayloadStore
from tests.test_lab_preflight import setup_lab
from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5


@pytest.fixture
def session() -> Iterator[Session]:
    engine = create_engine("sqlite://")
    Base.metadata.create_all(engine)
    with Session(engine) as value:
        yield value
    engine.dispose()


@pytest.mark.parametrize("priority", ["production", "interactive", "background", "benchmark"])
def test_persisted_service_priority_and_lease_owner_are_preserved(priority: str) -> None:
    client_id = uuid4()
    context = DispatchContext.for_service_client(client_id, "project-client", priority)
    assert context.service_client_id == client_id
    assert context.workload_priority == priority
    assert context.lease_owner == "project-client"
    assert context.provenance()["actor_id"] == str(client_id)
    assert context.operation_id is None
    with pytest.raises(ValidationError, match="frozen"):
        context.workload_priority = "lab"


@pytest.mark.parametrize("priority", ["lab", "maintenance", "admin", "", "PRODUCTION"])
def test_service_dispatch_rejects_unconfigured_priority_domain(priority: str) -> None:
    with pytest.raises(ValidationError):
        DispatchContext.for_service_client(uuid4(), "project-client", priority)


def test_operator_identity_is_separate_and_cannot_elevate_priority() -> None:
    context = DispatchContext.for_operator("jens", uuid4())
    assert context.service_client_id is None
    assert context.workload_priority == "lab"
    assert context.lease_owner == "operator:jens"
    for changes in (
        {"workload_priority": "production"}, {"service_client_id": uuid4()},
        {"operation_id": None}, {"actor_kind": "service_client"},
        {"lease_owner": "a-different-owner"}, {"token": "not-accepted"},
    ):
        with pytest.raises(ValidationError):
            DispatchContext.model_validate({**context.model_dump(), **changes})
    assert set(context.provenance()) == {
        "actor_kind", "actor_id", "service_client_id", "operation_id", "workload_priority",
    }


def test_operator_scheduler_admission_requires_no_fabricated_service_client(session: Session) -> None:
    service, candidate, _ = setup_m5(session)
    _, response = approve_and_promote(service, candidate.id)
    deployment = service.repo.deployment(response.id)
    assert deployment is not None
    context = DispatchContext.for_operator("operator-test", uuid4())
    request = GatewayRequest(request_id=uuid4(), service_client_id=None,
        capability_key="rag.embedding", capability_version=1,
        capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
        status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
    session.add(request)
    session.commit()
    lease = service._acquire_request_lease(deployment, request, context)
    assert lease.priority == "lab"
    assert lease.owner == "operator:operator-test"
    assert lease.request_id == request.request_id
    assert request.decision_evidence["dispatch_context"] == context.provenance()
    assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0


def test_operator_context_cannot_bypass_paused_lab_policy(session: Session) -> None:
    service, candidate, _ = setup_m5(session)
    _, response = approve_and_promote(service, candidate.id)
    deployment = service.repo.deployment(response.id)
    assert deployment is not None
    session.add(SchedulerPolicyRevision(revision="lab-paused-test", active=True,
                                        configuration={"lab_paused": True}))
    context = DispatchContext.for_operator("operator-test", uuid4())
    request = GatewayRequest(request_id=uuid4(), service_client_id=None,
        capability_key="rag.embedding", capability_version=1,
        capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
        status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
    session.add(request)
    session.commit()
    with pytest.raises(ServingError) as raised:
        service._acquire_request_lease(deployment, request, context)
    assert raised.value.code == "POLICY_BLOCKED"
    assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []


@pytest.mark.parametrize("mismatch", ["client", "priority", "deployment"])
def test_context_mismatch_fails_before_scheduler_mutation(session: Session, mismatch: str) -> None:
    service, candidate, _ = setup_m5(session)
    _, response = approve_and_promote(service, candidate.id)
    deployment = service.repo.deployment(response.id)
    assert deployment is not None
    context = DispatchContext.for_operator("operator-test", uuid4())
    request = GatewayRequest(request_id=uuid4(), service_client_id=uuid4() if mismatch == "client" else None,
        capability_key="rag.embedding", capability_version=1,
        capability_deployment_id=uuid4() if mismatch == "deployment" else deployment.id,
        compute_node_id=deployment.compute_node_id, status="queued",
        priority="production" if mismatch == "priority" else "lab", input_sha256="a" * 64, input_count=1)
    with pytest.raises(ServingError) as raised:
        service._acquire_request_lease(deployment, request, context)
    assert raised.value.code == "DISPATCH_CONTEXT_MISMATCH"
    assert request.status == "queued"
    assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []


def operator_fixture(session: Session):
    preflight, request = setup_lab(session)
    service = InlineWorkerServingService(session, preflight.settings,
        ManifestRegistry(), MemoryPayloadStore())
    experiment = service.create_experiment(request.target.deployment_candidate_id,
        CapabilityExperimentCreate(route_key="operator-selected", purpose="Bounded exact-target unit test",
                                  execution_approval_id=request.target.execution_approval_id))
    return service, request, experiment


def test_operator_embedding_executes_exact_selected_experiment_without_client(session: Session) -> None:
    service, request, selected = operator_fixture(session)
    service.synthetic_worker_metrics = {
        "gpu_uuid": service.repo.accelerator(selected.deployment.accelerator_id).device_uuid,
        "used_vram_bytes": 4_096,
        "process_used_vram_bytes": 2_048,
        "free_vram_bytes": 8_192,
        "gpu_utilization_percent": 32,
        "temperature_c": 63,
        "power_draw_w": 101.25,
        "private_metric_canary": "must-not-be-persisted",
    }
    other = service.create_experiment(request.target.deployment_candidate_id,
        CapabilityExperimentCreate(route_key="operator-other", purpose="Unselected comparison fixture",
                                  execution_approval_id=request.target.execution_approval_id))
    service.authenticated_credential_id = uuid4()  # stale service-instance state must not cross domains
    operation_id = uuid4()
    context = DispatchContext.for_operator("operator-test", operation_id)
    result = service.invoke_operator_lab(request.input, context,
        deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
    assert record is not None
    assert record.capability_deployment_id == selected.capability_deployment_id != other.capability_deployment_id
    assert record.service_client_id is None and record.priority == "lab"
    assert record.decision_evidence["service_credential_id"] is None
    assert record.decision_evidence["dispatch_context"] == context.provenance()
    assert record.decision_evidence["operator_route_id"] == str(selected.id)
    assert result.embedding_space_id == selected.deployment.embedding_space.id
    evidence = record.decision_evidence["lab_run_evidence"]
    candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
    profile = service.repo.profile(request.target.runtime_profile_id)
    deployment = service.repo.deployment(selected.capability_deployment_id)
    assert candidate is not None and profile is not None and deployment is not None
    accelerator = service.repo.accelerator(deployment.accelerator_id)
    assert accelerator is not None
    assert evidence["schema_version"] == 1
    assert evidence["status"] == "completed"
    assert evidence["capability"] == "rag.embedding@1"
    assert evidence["operation_id"] == str(operation_id)
    assert evidence["input"] == {
        "sha256": record.input_sha256,
        "item_count": 1,
        "content_logged": False,
    }
    assert evidence["target"] == {
        "deployment_id": str(selected.capability_deployment_id),
        "candidate_id": str(candidate.id),
        "experiment_route_id": str(selected.id),
        "capability_contract_id": str(deployment.capability_contract_id),
        "contract_version": 1,
        "artifact_set_id": str(deployment.artifact_set_id),
        "artifact_sha256": profile.artifact_sha256,
        "runtime_profile_id": str(profile.id),
        "runtime_profile_fingerprint": profile.fingerprint,
        "compatibility_assessment_id": str(candidate.compatibility_assessment_id),
        "runtime_probe_id": str(candidate.runtime_probe_id),
        "compute_node_id": str(deployment.compute_node_id),
        "accelerator_id": str(deployment.accelerator_id),
        "accelerator_uuid": accelerator.device_uuid,
        "execution_approval_id": str(request.target.execution_approval_id),
        "embedding_space_id": str(deployment.embedding_space_id),
        "deployment_fingerprint": deployment.config_fingerprint,
    }
    expected_output_hash = hashlib.sha256(
        json.dumps(result.data, sort_keys=True, separators=(",", ":")).encode()
    ).hexdigest()
    assert evidence["output"] == {
        "type": "float_vector",
        "dimension": result.dimension,
        "item_count": len(result.data),
        "embedding_space_id": str(result.embedding_space_id),
        "sha256": expected_output_hash,
        "content_logged": False,
    }
    serving_job = session.scalar(
        select(ServingJob).where(
            ServingJob.gateway_request_id == record.id,
            ServingJob.operation == "invoke",
        )
    )
    expected_observations = {
        "source": "runtime-worker",
        "phase": "after_inference",
        "availability": "complete",
        "device_matches_target": True,
        "metrics": {
            "gpu_uuid": accelerator.device_uuid,
            "used_vram_bytes": 4_096,
            "process_used_vram_bytes": 2_048,
            "free_vram_bytes": 8_192,
            "gpu_utilization_percent": 32,
            "temperature_c": 63,
            "power_draw_w": 101.25,
        },
    }
    assert evidence["worker_observations"] == expected_observations
    assert serving_job is not None
    assert serving_job.result_summary["resource_observation"] == expected_observations
    assert "vectors" not in evidence["output"] and "data" not in evidence["output"]
    assert "private input content" not in json.dumps(evidence)
    assert "must-not-be-persisted" not in json.dumps(evidence)
    assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0


def test_operator_lab_without_worker_resource_metrics_does_not_invent_gpu_evidence(
    session: Session,
) -> None:
    service, request, selected = operator_fixture(session)
    result = service.invoke_operator_lab(
        request.input,
        DispatchContext.for_operator("operator-test", uuid4()),
        deployment_id=selected.capability_deployment_id,
        experiment_route_id=selected.id,
    )
    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
    assert record is not None
    observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
    assert observations == {
        "source": "runtime-worker",
        "phase": "after_inference",
        "availability": "unavailable",
        "device_matches_target": None,
        "metrics": {},
    }


def test_operator_lab_rejects_unbounded_worker_resource_metrics(session: Session) -> None:
    service, request, selected = operator_fixture(session)
    deployment = service.repo.deployment(selected.capability_deployment_id)
    assert deployment is not None
    accelerator = service.repo.accelerator(deployment.accelerator_id)
    assert accelerator is not None
    service.synthetic_worker_metrics = {
        "gpu_uuid": accelerator.device_uuid,
        "used_vram_bytes": True,
        "process_used_vram_bytes": 2**100,
        "free_vram_bytes": 8_192,
        "gpu_utilization_percent": 101,
        "temperature_c": 10**1000,
        "power_draw_w": 10**1000,
        "private_metric_canary": "must-not-be-persisted",
    }
    result = service.invoke_operator_lab(
        request.input,
        DispatchContext.for_operator("operator-test", uuid4()),
        deployment_id=selected.capability_deployment_id,
        experiment_route_id=selected.id,
    )
    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
    assert record is not None
    observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
    assert observations == {
        "source": "runtime-worker",
        "phase": "after_inference",
        "availability": "partial",
        "device_matches_target": True,
        "metrics": {"gpu_uuid": accelerator.device_uuid, "free_vram_bytes": 8_192},
    }


def test_operator_lab_scheduler_failure_is_durable_and_redacted(
    session: Session, monkeypatch: pytest.MonkeyPatch
) -> None:
    service, request, selected = operator_fixture(session)

    def reject_admission(*_args, **_kwargs):
        raise ServingError(429, "LAB_QUEUE_REJECTED", "private scheduler detail")

    monkeypatch.setattr(service, "_acquire_request_lease", reject_admission)
    with pytest.raises(ServingError, match="private scheduler detail"):
        service.invoke_operator_lab(
            request.input,
            DispatchContext.for_operator("operator-test", uuid4()),
            deployment_id=selected.capability_deployment_id,
            experiment_route_id=selected.id,
        )

    record = session.scalar(select(GatewayRequest).where(GatewayRequest.priority == "lab"))
    assert record is not None and record.status == "failed"
    assert record.failure_code == "LAB_QUEUE_REJECTED"
    evidence = record.decision_evidence["lab_run_evidence"]
    assert evidence["status"] == "failed"
    assert evidence["failure_code"] == "LAB_QUEUE_REJECTED"
    assert "output" not in evidence and "worker_observations" not in evidence
    assert "private input content" not in json.dumps(evidence)
    assert "private scheduler detail" not in json.dumps(evidence)


@pytest.mark.parametrize("bad_target", ["missing_deployment", "missing_route", "inactive_route", "stable"])
def test_operator_embedding_rejects_non_exact_or_non_lab_target(session: Session, bad_target: str) -> None:
    service, request, selected = operator_fixture(session)
    deployment_id, route_id = selected.capability_deployment_id, selected.id
    if bad_target == "missing_deployment":
        deployment_id = uuid4()
    elif bad_target == "missing_route":
        route_id = uuid4()
    elif bad_target == "inactive_route":
        service.deactivate_experiment(route_id)
    else:
        _, stable = approve_and_promote(service, request.target.deployment_candidate_id)
        deployment_id = stable.id
    with pytest.raises(ServingError) as raised:
        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
            deployment_id=deployment_id, experiment_route_id=route_id)
    assert raised.value.code == "LAB_TARGET_NOT_ROUTABLE"
    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0


def test_operator_entry_rejects_service_context_and_unimplemented_family(session: Session) -> None:
    service, request, selected = operator_fixture(session)
    with pytest.raises(ServingError) as denied:
        service.invoke_operator_lab(request.input, DispatchContext.for_service_client(uuid4(), "test-client", "production"),
            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
    assert denied.value.code == "OPERATOR_CONTEXT_REQUIRED"
    reranking = LabRerankingInput(capability="rag.reranking", payload=RerankingInvokeRequest(
        query="test", documents=[{"id": "a", "text": "test"}], top_n=1))
    with pytest.raises(ServingError) as unsupported:
        service.invoke_operator_lab(reranking, DispatchContext.for_operator("operator-test", uuid4()),
            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
    assert unsupported.value.code == "LAB_INPUT_NOT_SUPPORTED"
    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0


@pytest.mark.parametrize("invalid", ["approval_expired", "wrong_contract"])
def test_operator_entry_rechecks_exact_approval_and_contract(session: Session, invalid: str) -> None:
    service, request, selected = operator_fixture(session)
    if invalid == "approval_expired":
        approval = service.repo.execution_approval(request.target.execution_approval_id)
        assert approval is not None
        approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
    else:
        route = service.repo.experiment_route_by_id(selected.id)
        assert route is not None
        route.capability_contract_id = service.ensure_contract("rag.reranking", 1).id
    session.commit()
    with pytest.raises(ServingError) as denied:
        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
    assert denied.value.code == ("LAB_ADMISSION_BLOCKED" if invalid == "approval_expired" else "LAB_TARGET_NOT_ROUTABLE")
    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0


def test_route_deactivation_after_load_prevents_operator_inference(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
    service, request, selected = operator_fixture(session)
    original = service._ensure_loaded
    original_put = service.payload_store.put
    input_references = []
    def track_put(payload, ttl_seconds):
        reference = original_put(payload, ttl_seconds)
        if "input" in payload:
            input_references.append(reference)
        return reference
    monkeypatch.setattr(service.payload_store, "put", track_put)
    def load_then_deactivate(deployment, deadline, *, priority):
        loaded = original(deployment, deadline, priority=priority)
        route = service.repo.experiment_route_by_id(selected.id)
        assert route is not None
        route.status = "inactive"
        session.commit()
        return loaded
    monkeypatch.setattr(service, "_ensure_loaded", load_then_deactivate)
    with pytest.raises(ServingError) as denied:
        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
    assert denied.value.code == "LAB_TARGET_NOT_ROUTABLE"
    assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
    record = session.scalar(select(GatewayRequest))
    assert record is not None and record.status == "failed"
    assert len(input_references) == 1
    assert service.payload_store.get(input_references[0]) is None


def test_gateway_times_validation_once_before_stable_resolution(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
    service, request, _ = operator_fixture(session)
    approve_and_promote(service, request.target.deployment_candidate_id)
    created = service.create_client(ServiceClientCreate(name="timing-client", allowed_capabilities=["rag.embedding@1"]))
    client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
    events = []
    clock = [1000.0]
    original_validate = service._validate_inputs
    original_resolve = service._stable_embedding_deployment
    def validate(value):
        events.append("validate")
        clock[0] += 0.125
        return original_validate(value)
    def resolve():
        events.append("resolve")
        clock[0] += 0.375
        return original_resolve()
    monkeypatch.setattr(service, "_validate_inputs", validate)
    monkeypatch.setattr(service, "_stable_embedding_deployment", resolve)
    monkeypatch.setattr("modelforge_api.services.serving.time.perf_counter", lambda: clock[0])
    response = service.invoke(EmbeddingInvokeRequest(input="timed request"), client)
    assert events == ["validate", "resolve"]
    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == response.request_id))
    assert record is not None
    latency = record.decision_evidence["latency_ms"]
    assert latency["validation"] == 125.0
    assert latency["resolution"] == 375.0
    assert latency["total"] >= 500.0


def test_failed_enqueue_deletes_input_without_a_job_owner(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
    service, request, selected = operator_fixture(session)
    original_enqueue = service._enqueue_job
    original_put = service.payload_store.put
    input_references = []
    def track_put(payload, ttl_seconds):
        reference = original_put(payload, ttl_seconds)
        if "input" in payload:
            input_references.append(reference)
        return reference
    def enqueue(deployment, operation, **kwargs):
        if operation == "invoke":
            raise ServingError(409, "ENQUEUE_REJECTED", "isolated enqueue failure")
        return original_enqueue(deployment, operation, **kwargs)
    monkeypatch.setattr(service.payload_store, "put", track_put)
    monkeypatch.setattr(service, "_enqueue_job", enqueue)
    with pytest.raises(ServingError, match="isolated enqueue failure"):
        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
    assert len(input_references) == 1
    assert service.payload_store.get(input_references[0]) is None
    assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0