Publish curated Unraid-AppOps-Gateway source from a506550ef83ef2d81651e37c98a89f7ee330c730

This commit is contained in:
Public Source Publisher committed 2026-10-03 13:35:10 +00:00
1 parent 2facb8de34
commit 767e66921e
68 files changed
+15924 -313

No files matched your search

+7 -11
View File
@@ -217,7 +217,9 @@ class GiteaActionsConnector(BaseConnector):
"order": "asc",
},
)
pending_payload = self._request(
# Keep pending inventory in the all-or-error fetch, but it is not
# runnable queue pressure and must not drive runner diagnostics.
self._request(
"/api/v1/admin/actions/jobs",
{
"status": "pending",
@@ -267,9 +269,10 @@ class GiteaActionsConnector(BaseConnector):
]
queued_jobs = [
item
for payload in (queued_payload, pending_payload)
for item in (
payload.get("jobs") if isinstance(payload.get("jobs"), list) else []
queued_payload.get("jobs")
if isinstance(queued_payload.get("jobs"), list)
else []
)
if isinstance(item, dict)
]
@@ -462,8 +465,7 @@ class GiteaActionsConnector(BaseConnector):
"offline": sum(item["status"] != "online" for item in runners),
},
"queue": {
"queued": int(queued_payload.get("total_count", 0) or 0)
+ int(pending_payload.get("total_count", 0) or 0),
"queued": int(queued_payload.get("total_count", 0) or 0),
"inProgress": int(
active_payload.get("total_count", len(active_jobs)) or 0
),
@@ -476,12 +478,6 @@ class GiteaActionsConnector(BaseConnector):
if isinstance(queued_payload.get("jobs"), list)
else []
)
or int(pending_payload.get("total_count", 0) or 0)
> len(
pending_payload.get("jobs")
if isinstance(pending_payload.get("jobs"), list)
else []
)
),
"blockedByRunnerLabels": sorted(set(blocked_labels))[:20],
"staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20],
+19 -1
View File
@@ -11,8 +11,22 @@ from ..core.config import settings
from ..core.redaction import redact_message
from .base import BaseConnector
_PORTAINER_DEFAULT_TIMEOUT_SECONDS = 8.0
_CONTROLLER_OWNED_CI_CONTAINER_NAMES = frozenset({
'itworx-ci-checks',
'itworx-ci-backend',
'itworx-ci-frontend',
'itworx-ci-security',
'itworx-ci-browser',
'itworx-ci-build',
'itworx-security-runner',
'itworx-validation-runner-2',
'itworx-validation-runner-3',
'itworx-docker-runner',
'itworx-browser-runner',
'itworx-validation-runner-burst-1',
'gitea-runner-dind',
})
@dataclass
@@ -213,6 +227,10 @@ class PortainerConnector(BaseConnector):
if not selected:
return {'status': 'not_found', 'reason': f'Container {target} not found'}
selected_name = str(selected.get('name', '')).strip().strip('/').lower()
if selected_name in _CONTROLLER_OWNED_CI_CONTAINER_NAMES:
return {'status': 'blocked', 'reason': 'controller_owned_ci_lifecycle'}
container_name = selected.get('id') or selected.get('name')
if not container_name:
return {'status': 'error', 'reason': 'Unable to resolve container identifier'}
+46
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
import stat
from typing import Any
@@ -34,6 +36,38 @@ def _secret_value(name: str, file_name: str) -> str | None:
return None
def _private_secret_file(path: str | None) -> str | None:
"""Read a small owner-private regular secret file without following links."""
if not path:
return None
descriptor: int | None = None
try:
source = Path(path)
before = source.lstat()
if not source.is_absolute() or source.is_symlink() or not stat.S_ISREG(before.st_mode) or before.st_size > 512:
return None
descriptor = os.open(
source,
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0),
)
opened = os.fstat(descriptor)
if not stat.S_ISREG(opened.st_mode) or opened.st_size > 512:
return None
if os.name != "nt" and (
opened.st_uid != os.geteuid() or stat.S_IMODE(opened.st_mode) & 0o077
):
return None
value = os.read(descriptor, 513)
if len(value) > 512:
return None
return value.decode("utf-8").strip() or None
except (OSError, UnicodeDecodeError):
return None
finally:
if descriptor is not None:
os.close(descriptor)
def _env_list(name: str, default: list[str] | None = None) -> list[str]:
value = os.getenv(name)
if value is None:
@@ -86,6 +120,7 @@ class Settings:
mcp_token: str | None = None
mcp_token_file: str | None = None
mcp_allowed_hosts: list[str] | None = None
runner_statistics_token_file: str | None = None
n8n_enabled: bool = False
n8n_token: str | None = None
n8n_token_file: str | None = None
@@ -120,6 +155,11 @@ class Settings:
self.mcp_token = token_file.read().strip() or None
except OSError:
self.mcp_token = None
default_runner_statistics_token_file = str(Path(self.data_dir) / "runner-statistics-token")
self.runner_statistics_token_file = _env_value(
"APPOPS_RUNNER_STATISTICS_TOKEN_FILE",
self.runner_statistics_token_file or default_runner_statistics_token_file,
)
allowed_hosts = _env_value("APPOPS_MCP_ALLOWED_HOSTS", "localhost:*,127.0.0.1:*,198.18.10.150:*") or ""
self.mcp_allowed_hosts = [item.strip() for item in allowed_hosts.split(",") if item.strip()]
self.n8n_enabled = _env_bool("APPOPS_ENABLE_N8N", self.n8n_enabled)
@@ -267,6 +307,10 @@ class Settings:
"APPOPS_VERIFY_TLS": self.verify_tls,
}
def read_runner_statistics_token(self) -> str | None:
"""Read the fixed private credential at request time for no-restart provisioning."""
return _private_secret_file(self.runner_statistics_token_file)
def redact_env_preview(self) -> dict[str, Any]:
return {
"APPOPS_BIND_HOST": self.bind_host,
@@ -291,6 +335,8 @@ class Settings:
"APPOPS_MCP_TOKEN_FILE": bool(self.mcp_token_file),
"APPOPS_MCP_TOKEN_CONFIGURED": bool(self.mcp_token),
"APPOPS_MCP_ALLOWED_HOSTS": list(self.mcp_allowed_hosts or []),
"APPOPS_RUNNER_STATISTICS_TOKEN_FILE": bool(self.runner_statistics_token_file),
"APPOPS_RUNNER_STATISTICS_TOKEN_CONFIGURED": bool(self.read_runner_statistics_token()),
"APPOPS_ENABLE_N8N": self.n8n_enabled,
"APPOPS_N8N_TOKEN_FILE": bool(self.n8n_token_file),
"APPOPS_N8N_TOKEN_CONFIGURED": bool(self.n8n_token),
+9
View File
@@ -53,6 +53,15 @@ def require_service_request(request: Request, service: str) -> str:
enabled, expected = settings.mcp_enabled, settings.mcp_token
elif service == "n8n":
enabled, expected = settings.n8n_enabled, settings.n8n_token
elif service == "mcp_runner_statistics":
expected = settings.read_runner_statistics_token()
if not expected:
raise HTTPException(404, "runner statistics MCP integration is disabled")
if not token_is_strong(expected):
raise HTTPException(503, "runner statistics MCP authentication is not safely configured")
if settings.mcp_token and hmac.compare_digest(expected, settings.mcp_token):
raise HTTPException(503, "runner statistics MCP token must be separate from the governed MCP token")
enabled = True
elif service == "voice":
enabled, expected = True, settings.voice_token or os.getenv("APP_OPS_VOICE_TOKEN")
else:
+13 -3
View File
@@ -73,6 +73,7 @@ from .mcp_server import (
restore_tool_metrics,
set_session_running,
)
from .runner_statistics_mcp import runner_statistics_mcp_http_app, run_runner_statistics_mcp
from .services import knowledge_service
from .services.discovery_scheduler import run_periodic_discovery
from .services.maintenance_scheduler import (
@@ -130,6 +131,7 @@ async def lifespan(_: FastAPI):
if settings.mcp_enabled:
await stack.enter_async_context(appops_mcp.session_manager.run())
set_session_running(True)
await stack.enter_async_context(run_runner_statistics_mcp())
try:
yield
finally:
@@ -182,6 +184,7 @@ class MachineBodyLimitMiddleware:
method = str(scope.get("method", "GET")).upper()
is_machine_post = method not in _SAFE_METHODS and (
path == "/mcp" or path.startswith("/mcp/")
or path == "/mcp-runner-statistics" or path.startswith("/mcp-runner-statistics/")
or path == "/api/integrations/n8n" or path.startswith("/api/integrations/n8n/")
)
if scope["type"] != "http" or not is_machine_post:
@@ -243,17 +246,23 @@ async def private_operator_boundary(request, call_next):
is_api = path == "/api" or path.startswith("/api/")
is_voice = path.startswith("/api/voice/")
is_mcp = path == "/mcp" or path.startswith("/mcp/")
is_runner_statistics_mcp = path == "/mcp-runner-statistics" or path.startswith("/mcp-runner-statistics/")
is_n8n = path == "/api/integrations/n8n" or path.startswith("/api/integrations/n8n/")
is_public_auth = request.method in _SAFE_METHODS and (
path in {"/api/auth/session", "/api/auth/providers", "/api/health"}
or path.startswith(("/api/auth/login/", "/api/auth/callback/"))
)
if is_mcp or is_n8n:
if is_runner_statistics_mcp:
try:
require_service_request(request, "mcp_runner_statistics")
except HTTPException as exc:
return finalize_response(JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}))
elif is_mcp or is_n8n:
try:
require_service_request(request, "mcp" if is_mcp else "n8n")
except HTTPException as exc:
return finalize_response(JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}))
if is_mcp or is_n8n:
if is_mcp or is_runner_statistics_mcp or is_n8n:
content_length = request.headers.get("content-length")
if content_length and content_length.isdigit() and int(content_length) > 1_048_576:
return finalize_response(JSONResponse(status_code=413, content={"detail": "Request body exceeds 1 MiB limit"}))
@@ -297,7 +306,7 @@ async def private_operator_boundary(request, call_next):
response = await call_next(request)
finally:
actor_identity.reset(actor_token)
if is_api:
if is_api or is_runner_statistics_mcp:
response.headers.setdefault("Cache-Control", "no-store")
return finalize_response(response)
@@ -356,6 +365,7 @@ app.include_router(integrations_router)
app.include_router(n8n_router)
app.include_router(knowledge_router)
app.mount("/mcp", mcp_http_app)
app.mount("/mcp-runner-statistics", runner_statistics_mcp_http_app)
@app.get('/api')
+8
View File
@@ -46,6 +46,7 @@ from .services.privatemuse_artifact_installer import ArtifactInstallError
from .services.privatemuse_artifact_installer import install as install_artifact_file
from .services.privatemuse_artifact_installer import preflight as artifact_preflight
from .services.runtime_service import runtime_info
from .services.runner_statistics import RunnerStatisticsService
_SESSION_RUNNING = False
_SESSION_VERIFIED_AT: str | None = None
@@ -233,6 +234,13 @@ def gitea_actions_runners_status() -> dict[str, Any]:
return _json(GiteaActionsConnector().snapshot())
@appops_mcp.tool(name="gitea.actions.runners.statistics", annotations=_READ_ONLY_TOOL)
@_tracked("gitea.actions.runners.statistics")
def gitea_actions_runner_statistics() -> dict[str, Any]:
"""Read bounded CI profile budgets, measured usage and job statistics."""
return _json(RunnerStatisticsService().snapshot())
def _json(value: Any) -> Any:
if hasattr(value, "model_dump"):
return _json(value.model_dump(mode="json"))
+164
View File
@@ -0,0 +1,164 @@
"""Dedicated one-tool MCP surface for bounded, read-only CI statistics."""
from __future__ import annotations
from collections.abc import Callable, Sequence
from contextlib import asynccontextmanager
import ipaddress
import re
from typing import Any
from urllib.parse import urlsplit
from mcp.server import MCPServer
from mcp.server.transport_security import TransportSecuritySettings
from mcp.types import ToolAnnotations
from starlette.applications import Starlette
from starlette.responses import Response
from starlette.types import ASGIApp, Receive, Scope, Send
from .core.config import settings
from .core.version import APP_VERSION
from .services.runner_statistics import RunnerStatisticsService, sanitize_statistics, unavailable_statistics
RUNNER_STATISTICS_TOOL = "gitea.actions.runners.statistics"
_READ_ONLY_TOOL = ToolAnnotations(
read_only_hint=True,
destructive_hint=False,
idempotent_hint=True,
open_world_hint=False,
)
_DNS_HOST_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)(?:\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?))*$")
def _validated_auth_public_host() -> tuple[str, int | None] | None:
try:
parsed = urlsplit(settings.auth_public_url)
hostname = parsed.hostname
port = parsed.port
except ValueError:
return None
if (
parsed.scheme.lower() != "https"
or not hostname
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
or (port is not None and not 1 <= port <= 65535)
):
return None
try:
normalized = hostname.rstrip(".").encode("idna").decode("ascii").lower()
except UnicodeError:
return None
if not normalized:
return None
if ":" in normalized:
try:
ipaddress.IPv6Address(normalized)
except ValueError:
return None
elif len(normalized) > 253 or not _DNS_HOST_PATTERN.fullmatch(normalized):
return None
return normalized, port
def _readonly_allowed_hosts() -> list[str]:
"""Add only the validated public auth host to this read-only transport."""
hosts = list(settings.mcp_allowed_hosts or [])
validated = _validated_auth_public_host()
if validated is None:
return hosts
normalized, port = validated
host_value = f"[{normalized}]" if ":" in normalized else normalized
candidates = [f"{host_value}:{port}" if port is not None else host_value]
for candidate in candidates:
if candidate not in hosts:
hosts.append(candidate)
return hosts
def _readonly_allowed_origins() -> list[str]:
"""Keep existing frontend origin and admit only the validated auth origin."""
origins = [settings.frontend_origin] if settings.frontend_origin else []
validated = _validated_auth_public_host()
if validated is None:
return origins
normalized, port = validated
host_value = f"[{normalized}]" if ":" in normalized else normalized
netloc = f"{host_value}:{port}" if port is not None else host_value
origin = f"https://{netloc}"
if origin not in origins:
origins.append(origin)
return origins
def create_runner_statistics_mcp(
*,
snapshot_provider: Callable[[], object] | None = None,
allowed_hosts: Sequence[str] | None = None,
allowed_origins: Sequence[str] | None = None,
) -> tuple[MCPServer, Starlette]:
"""Build an isolated MCP server exposing only the sanitized statistics tool."""
provider = snapshot_provider or RunnerStatisticsService().snapshot
server = MCPServer(
"AppOps CI Runner Statistics",
version=APP_VERSION,
instructions=(
"Read-only CI runner profile statistics. This server exposes no operational, "
"administrative, deployment, filesystem, or mutation tools."
),
)
@server.tool(name=RUNNER_STATISTICS_TOOL, annotations=_READ_ONLY_TOOL)
def gitea_actions_runner_statistics() -> dict[str, Any]:
"""Read bounded runner budgets, measured usage, and job statistics."""
try:
return sanitize_statistics(provider())
except Exception: # noqa: BLE001 - keep internal broker details off the MCP surface.
return unavailable_statistics()
transport = server.streamable_http_app(
streamable_http_path="/",
stateless_http=True,
json_response=True,
max_request_body_size=1_048_576,
transport_security=TransportSecuritySettings(
enable_dns_rebinding_protection=True,
allowed_hosts=_readonly_allowed_hosts() if allowed_hosts is None else list(allowed_hosts),
allowed_origins=_readonly_allowed_origins() if allowed_origins is None else list(allowed_origins),
),
)
return server, transport
class _ReplaceableASGIApp:
"""Keep the mounted path stable while giving each app lifespan a fresh manager."""
def __init__(self) -> None:
self._app: ASGIApp | None = None
def replace(self, app: ASGIApp | None) -> None:
self._app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if self._app is None:
response = Response("Runner statistics MCP is unavailable", status_code=503)
await response(scope, receive, send)
return
await self._app(scope, receive, send)
runner_statistics_mcp_http_app = _ReplaceableASGIApp()
@asynccontextmanager
async def run_runner_statistics_mcp():
"""Run a fresh stateless transport manager for this FastAPI lifespan."""
server, transport = create_runner_statistics_mcp()
runner_statistics_mcp_http_app.replace(transport)
try:
async with server.session_manager.run():
yield
finally:
runner_statistics_mcp_http_app.replace(None)
+355 -10
View File
@@ -53,6 +53,9 @@ APPROVAL_EXECUTOR_ACTIONS = {
"container.restart",
"controller.artifact.promote",
"runner.fleet.upgrade_v2",
"runner.fleet.specialize",
"runner.statistics.configure",
"hub.migration.prepare_permissions",
"deployment.targets.sync",
"deployment.runnerless.provision",
"deployment.target.deploy_workmesh",
@@ -1648,18 +1651,27 @@ def _execute_runner_fleet_upgrade_v2(
risk: str,
reason: str,
original_policy_decision: str,
specialized: bool = False,
statistics_configuration: bool = False,
) -> dict[str, Any]:
from .deployment_broker import DeploymentBrokerClient
action = "runner.fleet.upgrade_v2"
target = "tower/runner-fleet-v2"
action = ("runner.statistics.configure" if statistics_configuration else
"runner.fleet.specialize" if specialized else "runner.fleet.upgrade_v2")
target = ("tower/projectbrain-runner-statistics" if statistics_configuration else
"tower/runner-fleet-specialized" if specialized else "tower/runner-fleet-v2")
steps = [
"Validate exact fixed runner-fleet target",
"Call HMAC-signed root controller endpoint",
"Prove protected main and successful push validation",
"Execute exact-revision fleet-v2 migration script",
"Execute exact-revision specialized migration script" if specialized
else "Execute exact-revision fleet-v2 migration script",
"Verify specialized runner topology",
]
if statistics_configuration:
steps = ["Verify fixed read-only statistics target", "Prove protected controller/runtime source",
"Lock production configuration", "Provision private independent credential with durable CAS",
"Configure ProjectBrain for its governed delivery"]
job = create_job(
session,
action_id=action,
@@ -1670,24 +1682,31 @@ def _execute_runner_fleet_upgrade_v2(
steps=steps,
)
start_job(session, job)
set_job_progress(session, job, 25, "Requesting fixed Tower runner-fleet v2 migration.")
set_job_progress(session, job, 25, f"Requesting {action} through the fixed controller endpoint.")
try:
result = DeploymentBrokerClient().runner_fleet_upgrade_v2(reason=reason)
broker = DeploymentBrokerClient()
result = (broker.runner_statistics_configure(reason=reason) if statistics_configuration else
broker.runner_fleet_specialize(reason=reason) if specialized else broker.runner_fleet_upgrade_v2(reason=reason))
status = str(result.get("status") or "error")
success = status in {"upgraded", "already_applied"}
success = status in ({"configured", "already_configured"} if statistics_configuration else
{"specialized", "already_applied"} if specialized else {"upgraded", "already_applied"})
message = str(
result.get("reason")
or (
"Tower runner-fleet v2 migration verified."
if success
else "Tower runner-fleet v2 migration was rejected or failed."
("ProjectBrain read-only statistics configuration verified." if statistics_configuration else
"Tower runner specialization verified." if specialized
else "Tower runner-fleet v2 migration verified.")
if success else
("ProjectBrain statistics configuration was rejected or failed." if statistics_configuration else
"Tower runner specialization was rejected or failed." if specialized
else "Tower runner-fleet v2 migration was rejected or failed.")
)
)
except Exception as exc:
result = {"status": "error"}
status = "error"
success = False
message = f"{type(exc).__name__}: runner fleet v2 migration failed"
message = f"{type(exc).__name__}: fixed controller operation failed"
completed = complete_job(
session,
@@ -1711,6 +1730,7 @@ def _execute_runner_fleet_upgrade_v2(
"status": status,
"revision": result.get("revision"),
"verifiedRunners": result.get("verifiedRunners"),
"projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"),
},
redacted_metadata={
"executor": action,
@@ -1732,6 +1752,253 @@ def _execute_runner_fleet_upgrade_v2(
"controllerStatus": status,
"revision": result.get("revision"),
"verifiedRunners": result.get("verifiedRunners"),
"projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"),
}
def _execute_hub_migration_prepare_permissions(
session: Session,
*,
source: str,
risk: str,
reason: str,
original_policy_decision: str,
) -> dict[str, Any]:
from .deployment_broker import DeploymentBrokerClient
action = "hub.migration.prepare_permissions"
target = "tower/itworx-mcp-hub-permissions"
state_file = "/mnt/cache/appdata/gitea-deploy-runners/state/itworx-mcp-hub.json"
fixed_targets = {state_file}
steps = [
"Validate the exact fixed MCP Hub permission target",
"Authenticate the fixed root-controller request with HMAC and replay protection",
"Verify the running controller is the protected current main release with full-push validation",
"Reinspect the fixed Hub state file and shared deployment lock",
"Apply only the fixed state-file mode correction and verify its final metadata",
]
job = create_job(
session,
action_id=action,
target=target,
source=source,
risk=risk,
policy_decision="allowed",
steps=steps,
)
start_job(session, job)
set_job_progress(session, job, 25, "Requesting fixed MCP Hub permission preparation through the controller.")
try:
result = DeploymentBrokerClient().hub_migration_prepare_permissions(reason=reason)
status = str(result.get("status") or "blocked")
revision_value = result.get("revision")
revision = (
revision_value
if isinstance(revision_value, str)
and len(revision_value) == 40
and all(character in "0123456789abcdef" for character in revision_value)
else None
)
changed_value = result.get("changedTargets", [])
changed_targets = (
changed_value
if isinstance(changed_value, list)
and all(isinstance(path, str) and path in fixed_targets for path in changed_value)
and len(changed_value) == len(set(changed_value))
else None
)
changed_details_value = result.get("changedTargetDetails", [])
target_names = {
state_file: "state_file",
}
changed_details = (
changed_details_value
if isinstance(changed_details_value, list)
and all(
isinstance(item, dict)
and set(item) == {"name", "path", "observedMode", "sameIdentityAtPath"}
and isinstance(item.get("path"), str)
and item.get("path") in fixed_targets
and item.get("name") == target_names.get(item.get("path"))
and isinstance(item.get("observedMode"), str)
and len(item["observedMode"]) == 4
and item["observedMode"].startswith("0")
and all(character in "01234567" for character in item["observedMode"])
and isinstance(item.get("sameIdentityAtPath"), bool)
and changed_targets is not None
and item["path"] in changed_targets
for item in changed_details_value
)
and len({item["path"] for item in changed_details_value}) == len(changed_details_value)
else None
)
uncertain_value = result.get("uncertainTargets", [])
uncertain_targets = (
uncertain_value
if isinstance(uncertain_value, list)
and all(isinstance(path, str) and path in fixed_targets for path in uncertain_value)
and len(uncertain_value) == len(set(uncertain_value))
else None
)
expected_verified = {
"state_file": {
"path": state_file,
"uid": 0,
"gid": 0,
"mode": "0600",
"aclPresent": False,
},
}
verified_value = result.get("verifiedTargets")
verified_targets = None
if isinstance(verified_value, list) and len(verified_value) == len(expected_verified):
by_name = {
item.get("name"): item
for item in verified_value
if isinstance(item, dict) and isinstance(item.get("name"), str)
}
if set(by_name) == set(expected_verified) and all(
all(by_name[name].get(key) == value for key, value in expected.items())
for name, expected in expected_verified.items()
):
verified_targets = [
{"name": name, **expected_verified[name]}
for name in ("state_file",)
]
has_controller_projection = all(
key in result
for key in (
"revision",
"changedTargets",
"changedTargetDetails",
"uncertainTargets",
)
)
success = (
status in {"prepared", "already_compliant"}
and has_controller_projection
and revision is not None
and changed_targets is not None
and changed_details is not None
and uncertain_targets == []
and verified_targets is not None
and (bool(changed_targets) if status == "prepared" else not changed_targets)
)
if success:
message = (
"The fixed MCP Hub state-file permission was prepared and verified."
if status == "prepared"
else "The fixed MCP Hub state-file permission was already compliant and verified."
)
elif (
status == "partial"
and has_controller_projection
and revision is not None
and changed_targets is not None
and changed_details is not None
and uncertain_targets is not None
and (changed_targets or uncertain_targets)
and len(changed_details) == len(changed_targets)
and {item["path"] for item in changed_details} == set(changed_targets)
):
message = "Permission preparation stopped partway; inspect the exact changed and uncertain fixed targets before retrying."
elif (
status in {
"blocked",
"unauthorized",
"missing_config",
"missing_secret",
"runtime_identity_unavailable",
"controller_source_mismatch",
"preflight_rejected",
"configuration_unavailable",
"exact_source_unavailable",
"invalid_request",
"invalid_body_size",
"authentication_rejected",
"forbidden",
}
and changed_targets == []
and changed_details == []
and uncertain_targets == []
):
status = "blocked"
message = "The fixed MCP Hub permission request was rejected before any target change."
else:
status = "outcome_unknown"
changed_targets = []
changed_details = []
uncertain_targets = sorted(fixed_targets)
verified_targets = None
message = (
"The controller request outcome is unknown; the fixed Hub state file may have changed. "
"Inspect it before retrying."
)
except Exception:
result = {"status": "outcome_unknown"}
status = "outcome_unknown"
revision = None
changed_targets = []
changed_details = []
uncertain_targets = sorted(fixed_targets)
verified_targets = None
success = False
message = (
"The controller request outcome is unknown; the fixed Hub state file may have changed. "
"Inspect it before retrying."
)
completed = complete_job(
session,
job,
status="succeeded" if success else "failed",
result_summary=message,
)
audit = log_event(
session,
source="action",
identity=source,
action_id=action,
target=target,
target_type="mcp-hub-permissions",
risk=risk,
policy_decision="allowed",
reason=reason,
result="success" if success else "outcome_unknown" if status == "outcome_unknown" else "failed",
execution_plan={"steps": steps, "jobId": completed.id},
after_state={
"status": status,
"revision": revision,
"changedTargets": changed_targets,
"changedTargetDetails": changed_details,
"uncertainTargets": uncertain_targets,
"verifiedTargets": verified_targets,
},
redacted_metadata={
"executor": action,
"integration": "mcp" if source == "chatgpt-mcp" else "operator",
"originalPolicyDecision": original_policy_decision,
"targetSelectionCallerSelectable": False,
"pathOrCommandCallerSelectable": False,
},
)
return {
"success": success,
"status": "success" if success else status,
"result": "success" if success else "outcome_unknown" if status == "outcome_unknown" else "failed",
"message": message,
"jobId": completed.id,
"auditId": audit.id,
"policyDecision": "allowed",
"originalPolicyDecision": original_policy_decision,
"controllerStatus": status,
"revision": revision,
"changedTargets": changed_targets,
"changedTargetDetails": changed_details,
"uncertainTargets": uncertain_targets,
"verifiedTargets": verified_targets,
}
@@ -1809,6 +2076,24 @@ def execute_governed_action(
original_policy_decision=original_policy_decision,
)
if action == "hub.migration.prepare_permissions":
if target.strip().lower() != "tower/itworx-mcp-hub-permissions":
return {
"success": False,
"status": "failed",
"result": "blocked",
"message": "MCP Hub permission preparation accepts only tower/itworx-mcp-hub-permissions.",
"policyDecision": "blocked",
"originalPolicyDecision": original_policy_decision,
}
return _execute_hub_migration_prepare_permissions(
session,
source=source,
risk=risk,
reason=reason,
original_policy_decision=original_policy_decision,
)
if action == "runner.fleet.upgrade_v2":
if target.strip().lower() != "tower/runner-fleet-v2":
return {
@@ -1827,6 +2112,28 @@ def execute_governed_action(
original_policy_decision=original_policy_decision,
)
if action == "runner.statistics.configure":
if target.strip().lower() != "tower/projectbrain-runner-statistics":
return {"success": False, "status": "failed", "result": "blocked",
"message": "Statistics configuration accepts only tower/projectbrain-runner-statistics.",
"policyDecision": "blocked", "originalPolicyDecision": original_policy_decision}
return _execute_runner_fleet_upgrade_v2(
session, source=source, risk=risk, reason=reason,
original_policy_decision=original_policy_decision, statistics_configuration=True,
)
if action == "runner.fleet.specialize":
if target.strip().lower() != "tower/runner-fleet-specialized":
return {
"success": False, "status": "failed", "result": "blocked",
"message": "Runner specialization accepts only tower/runner-fleet-specialized.",
"policyDecision": "blocked", "originalPolicyDecision": original_policy_decision,
}
return _execute_runner_fleet_upgrade_v2(
session, source=source, risk=risk, reason=reason,
original_policy_decision=original_policy_decision, specialized=True,
)
if action == "deployment.runnerless.provision":
return _execute_runnerless_target_provisioning(
session,
@@ -2287,6 +2594,44 @@ def execute_approved_action(session: Session, approval: Approval) -> dict[str, A
original_policy_decision="approval_required",
)
if approval.action == "runner.statistics.configure":
if approval.target.strip().lower() != "tower/projectbrain-runner-statistics":
return {"success": False, "status": "failed", "result": "blocked",
"message": "Statistics configuration accepts only tower/projectbrain-runner-statistics."}
return _execute_runner_fleet_upgrade_v2(
session, source="approved_operator", risk=approval.risk,
reason=approval.reason or f"Approved through {approval.id}.",
original_policy_decision="approval_required", statistics_configuration=True,
)
if approval.action == "runner.fleet.specialize":
if approval.target.strip().lower() != "tower/runner-fleet-specialized":
return {
"success": False, "status": "failed", "result": "blocked",
"message": "Runner specialization accepts only tower/runner-fleet-specialized.",
}
return _execute_runner_fleet_upgrade_v2(
session, source="approved_operator", risk=approval.risk,
reason=approval.reason or f"Approved through {approval.id}.",
original_policy_decision="approval_required", specialized=True,
)
if approval.action == "hub.migration.prepare_permissions":
if approval.target.strip().lower() != "tower/itworx-mcp-hub-permissions":
return {
"success": False,
"status": "failed",
"result": "blocked",
"message": "MCP Hub permission preparation accepts only tower/itworx-mcp-hub-permissions.",
}
return _execute_hub_migration_prepare_permissions(
session,
source="approved_operator",
risk=approval.risk,
reason=approval.reason or f"Approved through {approval.id}.",
original_policy_decision="approval_required",
)
if approval.action in {"unraid.vm.start", "unraid.vm.resume", "unraid.vm.pause"}:
from .unraid_vm_service import execute_approved_vm_action
+12
View File
@@ -7,8 +7,11 @@ ACTIONS = [
'container.restart',
'container.stop',
'runner.fleet.upgrade_v2',
'runner.fleet.specialize',
'runner.statistics.configure',
'controller.artifact.promote',
'controller.managed-storage.initialize',
'hub.migration.prepare_permissions',
'gateway.clear_cache',
'stack.status',
'stack.redeploy',
@@ -214,8 +217,11 @@ ACTION_RISK = {
'container.restart': 'medium',
'container.stop': 'high',
'runner.fleet.upgrade_v2': 'high',
'runner.fleet.specialize': 'high',
'runner.statistics.configure': 'high',
'controller.artifact.promote': 'high',
'controller.managed-storage.initialize': 'high',
'hub.migration.prepare_permissions': 'high',
'gateway.clear_cache': 'high',
'stack.status': 'low',
'stack.redeploy': 'high',
@@ -529,8 +535,14 @@ def action_default_target(action: str) -> str:
return 'tower/deployment-controller'
if action == 'controller.managed-storage.initialize':
return 'tower/deployment-controller'
if action == 'hub.migration.prepare_permissions':
return 'tower/itworx-mcp-hub-permissions'
if action == 'runner.fleet.upgrade_v2':
return 'tower/runner-fleet-v2'
if action == 'runner.fleet.specialize':
return 'tower/runner-fleet-specialized'
if action == 'runner.statistics.configure':
return 'tower/projectbrain-runner-statistics'
if action == 'deployment.targets.sync':
return 'jens/*'
if action == 'deployment.target.deploy_workmesh':
@@ -0,0 +1,128 @@
from __future__ import annotations
import re
from typing import Any
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
HELPER_NAME = "itworx-controller-artifact-promoter-modern"
CONTROLLER_VERSION = "2.1.14"
HELPER_LABEL = "itworx.controller.promoter"
_REVISION = re.compile(r"^[0-9a-f]{40}$")
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
_CONTAINER_ID = re.compile(r"^[0-9a-f]{64}$")
_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$")
def helper_container_spec(
*,
revision: str,
image_id: str,
current_controller_id: str | None = None,
current_controller_revision: str | None = None,
current_controller_tree: str | None = None,
current_controller_version: str | None = None,
) -> dict[str, Any]:
if not _REVISION.fullmatch(revision):
raise ValueError("controller_promoter_revision_invalid")
if not _IMAGE_ID.fullmatch(image_id):
raise ValueError("controller_promoter_image_invalid")
bootstrap_values = (
current_controller_id,
current_controller_revision,
current_controller_tree,
current_controller_version,
)
if any(value is not None for value in bootstrap_values):
if (
not _CONTAINER_ID.fullmatch(current_controller_id or "")
or not _REVISION.fullmatch(current_controller_revision or "")
or not _REVISION.fullmatch(current_controller_tree or "")
or not _VERSION.fullmatch(current_controller_version or "")
):
raise ValueError("controller_promoter_bootstrap_binding_invalid")
command = f"""set -Eeuo pipefail
umask 077
revision='{revision}'
version='{CONTROLLER_VERSION}'
image_id='{image_id}'
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
[[ \"$(stat -c '%a:%u:%g' /tmp)\" == '1777:0:0' ]]
work=\"$(mktemp -d -p /tmp itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX)\"
[[ -d \"$work\" && ! -L \"$work\" && \"$(stat -c '%u:%a' \"$work\")\" == '0:700' ]]
cleanup() {{ rm -rf -- \"$work\"; }}
trap cleanup EXIT
command -v git >/dev/null
command -v docker >/dev/null
docker compose version >/dev/null
command -v python3 >/dev/null
command -v tar >/dev/null
command -v curl >/dev/null
git --git-dir=\"$repo\" cat-file -e \"$revision^{{commit}}\"
tree=\"$(git --git-dir=\"$repo\" rev-parse \"$revision^{{tree}}\")\"
git --git-dir=\"$repo\" archive --format=tar \"$revision\" \\
ops/gitea-deploy-broker \\
scripts/prepare_controller_artifact.py \\
scripts/reconcile_controller_state.py \\
scripts/prepare_controller_hub_release_volume.py \\
scripts/vendor \\
scripts/run_repopulse_adoption.py \
scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"
chmod 0600 \"$work/controller.tar\"
archive_sha=\"$(sha256sum \"$work/controller.tar\" | cut -d ' ' -f 1)\"
git --git-dir=\"$repo\" show \"$revision:scripts/upgrade_deployment_controller.sh\" > \"$work/upgrade.sh\"
chmod 0700 \"$work/upgrade.sh\"
bash \"$work/upgrade.sh\" \\
--archive \"$work/controller.tar\" \\
--archive-sha256 \"$archive_sha\" \\
--source-repository '{SOURCE_REPOSITORY}' \\
--source-revision \"$revision\" \\
--source-tree \"$tree\" \\
--controller-version \"$version\" \\
--image-id \"$image_id\" \\
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
"""
labels = {
HELPER_LABEL: "true",
"itworx.source.repository": SOURCE_REPOSITORY,
"itworx.source.revision": revision,
"itworx.controller.version": CONTROLLER_VERSION,
}
if current_controller_id is not None:
labels.update(
{
"itworx.controller.bootstrap.target.id": current_controller_id,
"itworx.controller.bootstrap.target.image": image_id,
"itworx.controller.bootstrap.target.revision": current_controller_revision or "",
"itworx.controller.bootstrap.target.tree": current_controller_tree or "",
"itworx.controller.bootstrap.target.version": current_controller_version or "",
}
)
return {
"Image": image_id,
"User": "0:0",
"Cmd": ["/bin/bash", "-lc", command],
"Labels": labels,
"HostConfig": {
"Binds": [
"/mnt/cache:/mnt/cache",
"/mnt/user:/mnt/user",
"/var/lib/docker:/var/lib/docker:ro",
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
"/var/run/docker.sock:/var/run/docker.sock",
],
"NetworkMode": "host",
"Privileged": False,
"ReadonlyRootfs": True,
"CpuPeriod": 100000,
"CpuQuota": 100000,
"Memory": 1024 * 1024 * 1024,
"PidsLimit": 128,
"Tmpfs": {"/tmp": "rw,nosuid,nodev,size=268435456,mode=1777"},
"SecurityOpt": ["no-new-privileges:true"],
"RestartPolicy": {"Name": "no"},
},
}
@@ -11,19 +11,22 @@ import httpx
from ..connectors.portainer import PortainerConnector
from ..core.config import settings
from ..core.redaction import redact_message
from .controller_artifact_helper_spec import (
CONTROLLER_VERSION,
HELPER_LABEL,
HELPER_NAME,
SOURCE_REPOSITORY as SOURCE_REPOSITORY,
_IMAGE_ID,
_REVISION,
helper_container_spec,
)
from .deployment_broker import DeploymentBrokerClient
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
GITEA_OWNER = "Jens"
GITEA_REPOSITORY = "Unraid-AppOps-Gateway"
REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"
HELPER_NAME = "itworx-controller-artifact-promoter"
CONTROLLER_CONTAINER = "gitea-deploy-control"
CONTROLLER_VERSION = "2.1.13"
HELPER_LABEL = "itworx.controller.promoter"
_HELPER_TIMEOUT_SECONDS = 240
_REVISION = re.compile(r"^[0-9a-f]{40}$")
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$")
@@ -31,78 +34,6 @@ class ControllerArtifactPromotionError(RuntimeError):
pass
def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]:
if not _REVISION.fullmatch(revision):
raise ValueError("controller_promoter_revision_invalid")
if not _IMAGE_ID.fullmatch(image_id):
raise ValueError("controller_promoter_image_invalid")
command = f"""set -Eeuo pipefail
umask 077
revision='{revision}'
version='{CONTROLLER_VERSION}'
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
work="$(mktemp -d -p /tmp 'itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX')"
[[ -d "$work" && ! -L "$work" && "$(stat -c '%u:%a' "$work")" == '0:700' ]]
cleanup() {{ rm -rf -- "$work"; }}
trap cleanup EXIT
command -v git >/dev/null
command -v docker >/dev/null
docker compose version >/dev/null
command -v python3 >/dev/null
command -v tar >/dev/null
command -v curl >/dev/null
git --git-dir="$repo" cat-file -e "$revision^{{commit}}"
tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")"
git --git-dir="$repo" archive --format=tar "$revision" \
ops/gitea-deploy-broker \
scripts/prepare_controller_artifact.py \
scripts/reconcile_controller_state.py \
scripts/run_repopulse_adoption.py \
scripts/prepare_controller_hub_release_volume.py \
scripts/vendor \
scripts/upgrade_deployment_controller.sh > "$work/controller.tar"
chmod 0600 "$work/controller.tar"
archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)"
git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh"
chmod 0700 "$work/upgrade.sh"
bash "$work/upgrade.sh" \
--archive "$work/controller.tar" \
--archive-sha256 "$archive_sha" \
--source-repository '{SOURCE_REPOSITORY}' \
--source-revision "$revision" \
--source-tree "$tree" \
--controller-version "$version" \
--image-id '{image_id}' \
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
"""
return {
"Image": image_id,
"User": "0:0",
"Cmd": ["/bin/bash", "-lc", command],
"Labels": {
HELPER_LABEL: "true",
"itworx.source.repository": SOURCE_REPOSITORY,
"itworx.source.revision": revision,
"itworx.controller.version": CONTROLLER_VERSION,
},
"HostConfig": {
"Binds": [
"/mnt/cache:/mnt/cache",
"/mnt/user:/mnt/user",
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
"/var/lib/docker:/var/lib/docker:ro",
"/var/run/docker.sock:/var/run/docker.sock",
],
"NetworkMode": "host",
"Privileged": False,
"SecurityOpt": ["no-new-privileges:true"],
"RestartPolicy": {"Name": "no"},
},
}
class ControllerArtifactPromoter:
def __init__(
self,
+32 -2
View File
@@ -551,6 +551,27 @@ class DeploymentBrokerClient:
payload={"reason": reason[:1000]},
)
def runner_fleet_specialize(self, *, reason: str) -> dict[str, Any]:
"""Request the fixed protected-source CI specialization migration."""
return self._request(
"POST", "/v1/appops/runner-fleet/specialize",
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
)
def runner_statistics_configure(self, *, reason: str) -> dict[str, Any]:
"""Provision the fixed independent read-only ProjectBrain capability."""
return self._request(
"POST", "/v1/appops/runner-statistics/configure",
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
)
def hub_migration_prepare_permissions(self, *, reason: str) -> dict[str, Any]:
"""Prepare only the fixed Hub backup and controller-state permissions."""
return self._request(
"POST", "/v1/appops/hub-migration/prepare-permissions",
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
)
def runner_fleet_status(self) -> dict[str, Any]:
"""Read the controller's private, bounded CI sandbox projection.
@@ -558,16 +579,25 @@ class DeploymentBrokerClient:
derives the payload from fixed runner identities and its private DinD
daemon; AppOps never receives a Docker socket or a cleanup selector.
"""
return self._runner_fleet_read("/v1/runner-fleet")
def runner_fleet_statistics(self) -> dict[str, Any]:
"""Read the persisted profile projection without scheduling work."""
return self._runner_fleet_read("/v1/runner-fleet/statistics", timeout_seconds=min(self.timeout_seconds, 5))
def _runner_fleet_read(self, path: str, *, timeout_seconds: float | None = None) -> dict[str, Any]:
if path not in {"/v1/runner-fleet", "/v1/runner-fleet/statistics"}:
return {"status": "unavailable", "reason": "runner_fleet_path_invalid"}
if not self.is_configured():
return {"status": "missing_config", "reason": self.configuration_error()}
try:
with httpx.Client(
timeout=self.timeout_seconds,
timeout=self.timeout_seconds if timeout_seconds is None else timeout_seconds,
verify=self._verify_option(),
follow_redirects=False,
) as client, client.stream(
"GET",
f"{self.base_url}/v1/runner-fleet",
f"{self.base_url}{path}",
headers={"Accept": "application/json"},
) as response:
if response.status_code != 200:
@@ -32,8 +32,8 @@ import subprocess
import sys
import time
path = Path("/opt/gitea-deploy/projects.json")
host_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
path = Path("/controller-config/projects.json")
host_path = Path("/controller-hub-releases/controller-config/projects.json")
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
repository, revision, branch, raw_project = sys.argv[1:5]
project = json.loads(raw_project)
@@ -29,8 +29,8 @@ import sys
import time
repository, expected_container = sys.argv[1:3]
registry_path = Path("/opt/gitea-deploy/projects.json")
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "media-codec-dashboard.json"
project_lock_path = state_root / "locks" / "media-codec-dashboard.lock"
@@ -50,7 +50,7 @@ from pathlib import Path
import sys
repository = sys.argv[1]
path = Path("/opt/gitea-deploy/projects.json")
path = Path("/controller-config/projects.json")
try:
projects = json.loads(path.read_text(encoding="utf-8"))
except Exception:
@@ -145,8 +145,8 @@ import subprocess
import sys
import time
registry_path = Path("/opt/gitea-deploy/projects.json")
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "portfolio.json"
project_lock_path = state_root / "locks" / "portfolio.lock"
@@ -46,7 +46,7 @@ if (
raise SystemExit(40)
controller_path = Path("/opt/gitea-deploy/deploy.py")
registry_path = Path("/opt/gitea-deploy/projects.json")
registry_path = Path("/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "portfolio.json"
project_lock_path = state_root / "locks" / "portfolio.lock"
@@ -46,8 +46,8 @@ import subprocess
import sys
import time
registry_path = Path("/opt/gitea-deploy/projects.json")
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "privatemuse.json"
project_lock_path = state_root / "locks" / "privatemuse.lock"
@@ -64,7 +64,7 @@ if (
raise SystemExit(40)
controller_path = Path("/opt/gitea-deploy/deploy.py")
registry_path = Path("/opt/gitea-deploy/projects.json")
registry_path = Path("/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "privatemuse.json"
project_lock_path = state_root / "locks" / "privatemuse.lock"
@@ -26,8 +26,8 @@ repository, endpoint = sys.argv[1:3]
if repository != "jens/privatemuse" or endpoint != "http://198.19.0.1:1299/v1/gitea/push/jens/privatemuse":
raise SystemExit(40)
projects_path = Path("/opt/gitea-deploy/projects.json")
secret_root = Path("/opt/gitea-deploy/webhook-secrets")
projects_path = Path("/controller-config/projects.json")
secret_root = Path("/controller-config/webhook-secrets")
secret_path = secret_root / "jens__privatemuse"
admin_token_path = Path("/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token")
expected_validation = {
@@ -75,8 +75,8 @@ CONTAINER = "repopulse"
VOLUME = "repopulse_data"
APPDATA_ROOT = Path("/srv/example/appdata/repopulse")
SCAN_ROOT = Path("/mnt/cache/appdata/gitea-deploy-runners/sources")
REGISTRY = Path("/opt/gitea-deploy/projects.json")
HOST_REGISTRY = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
REGISTRY = Path("/controller-config/projects.json")
HOST_REGISTRY = Path("/controller-hub-releases/controller-config/projects.json")
STATE = Path("/mnt/cache/appdata/gitea-deploy-runners/state/repopulse.json")
BARE_REPO = Path("/gitea-repositories/jens/repopulse.git")
SHA = re.compile(r"^[0-9a-f]{40}$")
+181
View File
@@ -0,0 +1,181 @@
"""Bounded read-only CI statistics; absent measurements remain unknown."""
from __future__ import annotations
import copy
import re
import threading
import time
from datetime import UTC, datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from .deployment_broker import DeploymentBrokerClient
class _StatisticsModel(BaseModel):
model_config = ConfigDict(extra="ignore", allow_inf_nan=False, strict=True)
class _Budget(_StatisticsModel):
cpuLimit: float | int | None = Field(default=None, ge=0, le=20)
memoryLimitBytes: int | None = Field(default=None, ge=0)
activeJobLimit: int | None = Field(default=None, ge=0, le=6)
heavyJobLimit: int | None = Field(default=None, ge=0, le=6)
reservedCpu: float | int | None = Field(default=None, ge=0)
reservedMemoryBytes: int | None = Field(default=None, ge=0)
activeJobs: int | None = Field(default=None, ge=0)
heavyJobs: int | None = Field(default=None, ge=0)
class _Host(_StatisticsModel):
cpuBusyPercent: float | int | None = Field(default=None, ge=0, le=100)
loadRatio: float | int | None = Field(default=None, ge=0)
availableMemoryBytes: int | None = Field(default=None, ge=0)
cacheFreeBytes: int | None = Field(default=None, ge=0)
class _Profile(_StatisticsModel):
id: Literal["checks", "backend", "frontend", "security", "browser", "build"]
displayName: str = Field(min_length=1, max_length=80)
labels: list[str] = Field(max_length=8)
runnerName: str = Field(min_length=1, max_length=80)
status: str = Field(min_length=1, max_length=80)
queueDepth: int | None = Field(default=None, ge=0)
activeJobs: int | None = Field(default=None, ge=0)
oldestQueuedSeconds: float | int | None = Field(default=None, ge=0)
reason: str | None = Field(default=None, max_length=120)
cpuLimit: float | int | None = Field(default=None, ge=0, le=8)
memoryLimitBytes: int | None = Field(default=None, ge=0)
cpuUsed: float | int | None = Field(default=None, ge=0)
memoryUsedBytes: int | None = Field(default=None, ge=0)
completedJobs: int = Field(default=0, ge=0)
p50WaitSeconds: float | int | None = Field(default=None, ge=0)
p95WaitSeconds: float | int | None = Field(default=None, ge=0)
p50DurationSeconds: float | int | None = Field(default=None, ge=0)
p95DurationSeconds: float | int | None = Field(default=None, ge=0)
lastDurationSeconds: float | int | None = Field(default=None, ge=0)
class _Window(_StatisticsModel):
startAt: str | None = Field(default=None, max_length=40)
endAt: str | None = Field(default=None, max_length=40)
completedJobs: int = Field(default=0, ge=0)
samples: int = Field(default=0, ge=0)
coverage: Literal["bounded", "complete", "unavailable"] = "unavailable"
class _History(_StatisticsModel):
at: str = Field(min_length=1, max_length=40)
activeJobs: int | None = Field(default=None, ge=0)
queuedJobs: int | None = Field(default=None, ge=0)
cpuUsed: float | int | None = Field(default=None, ge=0)
memoryUsedBytes: int | None = Field(default=None, ge=0)
class RunnerStatistics(_StatisticsModel):
schemaVersion: Literal[1]
status: Literal["healthy", "degraded", "unavailable", "not_configured"]
scope: Literal["ci-fleet"]
generatedAt: str = Field(min_length=1, max_length=40)
budget: _Budget
host: _Host
profiles: list[_Profile] = Field(max_length=6)
window: _Window
history: list[_History] = Field(max_length=288)
def _valid_timestamp(value: str) -> bool:
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return False
return parsed.tzinfo is not None and 0 < parsed.timestamp() <= time.time() + 15
def unavailable_statistics(*, configured: bool = True) -> dict[str, object]:
return RunnerStatistics(
schemaVersion=1,
status="unavailable" if configured else "not_configured",
scope="ci-fleet",
generatedAt=datetime.now(UTC).isoformat(),
budget=_Budget(),
host=_Host(),
profiles=[],
window=_Window(),
history=[],
).model_dump(mode="json")
def sanitize_statistics(payload: object) -> dict[str, object]:
"""Accept only the public contract, never raw paths, credentials or jobs."""
try:
snapshot = RunnerStatistics.model_validate(payload)
except (ValidationError, TypeError, ValueError):
return unavailable_statistics()
timestamps = [snapshot.generatedAt, *(sample.at for sample in snapshot.history)]
timestamps.extend(value for value in (snapshot.window.startAt, snapshot.window.endAt) if value)
if any(not _valid_timestamp(value) for value in timestamps):
return unavailable_statistics()
if len({profile.id for profile in snapshot.profiles}) != len(snapshot.profiles):
return unavailable_statistics()
for profile in snapshot.profiles:
allowed_labels = {f"itworx-ci-{profile.id}"}
if profile.id in {"backend", "frontend"}:
allowed_labels.update({"linux-validation", "ubuntu-latest"})
elif profile.id == "build":
allowed_labels.add("docker-validation")
if (
profile.displayName != f"ITWorx CI · {profile.id.title()}"
or profile.runnerName != f"itworx-ci-{profile.id}"
or not profile.labels
or len(set(profile.labels)) != len(profile.labels)
or not set(profile.labels).issubset(allowed_labels)
or re.fullmatch(r"[a-z][a-z0-9_-]{0,79}", profile.status) is None
or (profile.reason is not None and re.fullmatch(r"[a-z][a-z0-9_-]{0,119}", profile.reason) is None)
):
return unavailable_statistics()
sample_time = datetime.fromisoformat(snapshot.generatedAt.replace("Z", "+00:00")).timestamp()
history_times = [datetime.fromisoformat(sample.at.replace("Z", "+00:00")).timestamp() for sample in snapshot.history]
if history_times != sorted(set(history_times)) or any(at > sample_time + 15 for at in history_times):
return unavailable_statistics()
window = snapshot.window
if (window.startAt is None) != (window.endAt is None):
return unavailable_statistics()
if window.startAt is not None and window.endAt is not None:
start = datetime.fromisoformat(window.startAt.replace("Z", "+00:00")).timestamp()
end = datetime.fromisoformat(window.endAt.replace("Z", "+00:00")).timestamp()
if not start <= end <= sample_time + 15:
return unavailable_statistics()
elif window.coverage != "unavailable":
return unavailable_statistics()
if time.time() - sample_time > 90 and snapshot.status == "healthy":
snapshot.status = "degraded"
return snapshot.model_dump(mode="json")
_lock = threading.Lock()
_cached: dict[str, object] | None = None
_cached_at = 0.0
_CACHE_SECONDS = 30.0
class RunnerStatisticsService:
def snapshot(self) -> dict[str, object]:
global _cached, _cached_at
with _lock:
now = time.monotonic()
if _cached is not None and now - _cached_at < _CACHE_SECONDS:
return copy.deepcopy(_cached)
broker = DeploymentBrokerClient()
if not broker.is_configured():
snapshot = unavailable_statistics(configured=False)
else:
try:
snapshot = sanitize_statistics(broker.runner_fleet_statistics())
except Exception:
snapshot = unavailable_statistics()
_cached = snapshot
_cached_at = time.monotonic()
return copy.deepcopy(snapshot)
@@ -36,8 +36,8 @@ if (
):
raise SystemExit(40)
projects_path = Path("/opt/gitea-deploy/projects.json")
secret_root = Path("/opt/gitea-deploy/webhook-secrets")
projects_path = Path("/controller-hub-releases/controller-config/projects.json")
secret_root = Path("/controller-hub-releases/controller-config/webhook-secrets")
admin_token_path = Path(
"/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token"
)
@@ -53,8 +53,8 @@ containers = [
]
ports = {1255, 1256, 1257, 1258}
controller_root = Path("/opt/gitea-deploy")
registry_path = controller_root / "projects.json"
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
env_dir = Path("/srv/example/appdata/workmesh")
env_path = env_dir / ".env"