647 lines
32 KiB
Python
647 lines
32 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from collections.abc import Callable
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from threading import Lock
|
|
from time import monotonic
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func
|
|
from sqlmodel import Session, col, select
|
|
|
|
from ..connectors.deluge import DelugeConnector
|
|
from ..connectors.plex import PlexConnector
|
|
from ..connectors.prowlarr import ProwlarrConnector
|
|
from ..connectors.sonarr import SonarrConnector
|
|
from ..connectors.tautulli import TautulliConnector
|
|
from ..connectors.widefrog import WidefrogConnector
|
|
from ..core.redaction import redact_message
|
|
from ..core.time import utc_now
|
|
from ..db.models import App, AppSetting, Connector, Secret
|
|
from .audit_logger import log_event
|
|
from .job_runner import complete_job, create_job, set_job_progress, start_job
|
|
from .policy_engine import evaluate_policy
|
|
|
|
CONNECTOR_IDS = ('sonarr', 'prowlarr', 'deluge', 'plex', 'tautulli', 'widefrog')
|
|
BLOCKED_ACTION_TOKENS = ('delete', 'remove', 'kill_session', 'terminate', 'empty_trash', 'download_database', 'download_config')
|
|
SNAPSHOT_CACHE_TTL_SECONDS = 15.0
|
|
SNAPSHOT_MAX_WORKERS = 8
|
|
SUMMARY_CACHE_SCHEMA_VERSION = 1
|
|
SUMMARY_CACHE_SETTING_KEY = 'media_pipeline:summary_cache'
|
|
SUMMARY_CACHE_TTL_SECONDS = 60
|
|
SUMMARY_CACHE_MAX_STALE_SECONDS = 300
|
|
_SNAPSHOT_CACHE: dict[tuple[int, ...], tuple[float, dict[str, Any]]] = {}
|
|
_SNAPSHOT_CACHE_LOCK = Lock()
|
|
|
|
|
|
def _now() -> str:
|
|
return utc_now().isoformat()
|
|
|
|
|
|
def _parse_utc(value: str) -> datetime:
|
|
return datetime.fromisoformat(value.replace('Z', '+00:00'))
|
|
|
|
|
|
def _summary_cache_ttl_seconds() -> int:
|
|
raw = os.getenv('MEDIA_PIPELINE_SUMMARY_CACHE_TTL_SECONDS', str(SUMMARY_CACHE_TTL_SECONDS))
|
|
try:
|
|
return max(1, int(raw))
|
|
except (TypeError, ValueError):
|
|
return SUMMARY_CACHE_TTL_SECONDS
|
|
|
|
|
|
def _summary_cache_max_stale_seconds() -> int:
|
|
raw = os.getenv('MEDIA_PIPELINE_SUMMARY_CACHE_MAX_STALE_SECONDS', str(SUMMARY_CACHE_MAX_STALE_SECONDS))
|
|
try:
|
|
return max(0, int(raw))
|
|
except (TypeError, ValueError):
|
|
return SUMMARY_CACHE_MAX_STALE_SECONDS
|
|
|
|
|
|
def _redact(value: Any) -> Any:
|
|
if isinstance(value, str):
|
|
return redact_message(value)
|
|
if isinstance(value, list):
|
|
return [_redact(item) for item in value]
|
|
if isinstance(value, dict):
|
|
return {
|
|
str(key): _redact(item)
|
|
for key, item in value.items()
|
|
if 'secret' not in str(key).lower() and 'token' not in str(key).lower() and 'password' not in str(key).lower()
|
|
}
|
|
return value
|
|
|
|
|
|
def _int(value: Any, default: int = 0) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _safe_call(call: Callable[[], dict[str, Any]], fallback_status: str = 'error') -> dict[str, Any]:
|
|
try:
|
|
payload = call()
|
|
return payload if isinstance(payload, dict) else {'success': False, 'status': fallback_status, 'message': 'unexpected_payload'}
|
|
except Exception as exc: # noqa: BLE001 - connector failures must not crash the dashboard.
|
|
return {'success': False, 'status': fallback_status, 'message': str(exc)}
|
|
|
|
|
|
def _snapshot_cache_key() -> tuple[int, ...]:
|
|
return (
|
|
id(SonarrConnector.health),
|
|
id(SonarrConnector.wanted_missing),
|
|
id(SonarrConnector.queue_status),
|
|
id(ProwlarrConnector.health),
|
|
id(ProwlarrConnector.indexer_status),
|
|
id(ProwlarrConnector.application_status),
|
|
id(ProwlarrConnector.sync_status),
|
|
id(DelugeConnector.health),
|
|
id(DelugeConnector.session_status),
|
|
id(DelugeConnector.active_downloads),
|
|
id(DelugeConnector.stalled_downloads),
|
|
id(DelugeConnector.paused_downloads),
|
|
id(DelugeConnector.speed_status),
|
|
id(PlexConnector.health),
|
|
id(PlexConnector.library_status),
|
|
id(PlexConnector.active_sessions),
|
|
id(PlexConnector.recent_media),
|
|
id(PlexConnector.stream_summary),
|
|
id(TautulliConnector.health),
|
|
id(TautulliConnector.activity),
|
|
id(TautulliConnector.history),
|
|
id(TautulliConnector.watch_stats),
|
|
id(TautulliConnector.stream_summary),
|
|
id(TautulliConnector.libraries),
|
|
id(WidefrogConnector.health),
|
|
id(WidefrogConnector.queue_status),
|
|
id(WidefrogConnector.failed_jobs),
|
|
id(WidefrogConnector.activity),
|
|
id(WidefrogConnector.download_status),
|
|
)
|
|
|
|
|
|
def _summary_signature(session: Session) -> str:
|
|
connectors = [
|
|
{
|
|
'id': item.id,
|
|
'status': item.status,
|
|
'lastTest': item.last_test,
|
|
'secretRef': item.secret_ref,
|
|
'capabilities': item.capabilities,
|
|
}
|
|
for item in session.exec(select(Connector)).all()
|
|
if item.id in CONNECTOR_IDS
|
|
]
|
|
secret_rows = [
|
|
{
|
|
'id': item.id,
|
|
'name': item.name,
|
|
'updatedAt': item.updated_at.isoformat() if item.updated_at else None,
|
|
'usedBy': item.used_by_json,
|
|
}
|
|
for item in session.exec(select(Secret)).all()
|
|
if any(connector_id in item.used_by_json for connector_id in CONNECTOR_IDS)
|
|
]
|
|
media_app_count = session.exec(select(func.count(App.id)).where(App.category.in_(['media', 'downloads']))).one()
|
|
signature_payload = {
|
|
'schemaVersion': SUMMARY_CACHE_SCHEMA_VERSION,
|
|
'connectors': sorted(connectors, key=lambda item: item['id']),
|
|
'secrets': sorted(secret_rows, key=lambda item: item['id']),
|
|
'mediaAppCount': media_app_count,
|
|
}
|
|
serialized = json.dumps(signature_payload, sort_keys=True, default=str, separators=(',', ':'))
|
|
return hashlib.sha256(serialized.encode('utf-8')).hexdigest()
|
|
|
|
|
|
def _persistent_summary(session: Session, signature: str) -> dict[str, Any] | None:
|
|
row = session.get(AppSetting, SUMMARY_CACHE_SETTING_KEY)
|
|
if not row or not isinstance(row.value_json, dict):
|
|
return None
|
|
value = row.value_json
|
|
if value.get('schemaVersion') != SUMMARY_CACHE_SCHEMA_VERSION or value.get('signature') != signature:
|
|
return None
|
|
payload = value.get('payload')
|
|
expires_at_raw = value.get('expiresAt')
|
|
if not isinstance(payload, dict) or not isinstance(expires_at_raw, str):
|
|
return None
|
|
expires_at = _parse_utc(expires_at_raw)
|
|
now_dt = datetime.now(UTC)
|
|
stale_until = expires_at.timestamp() + _summary_cache_max_stale_seconds()
|
|
if now_dt >= expires_at and now_dt.timestamp() >= stale_until:
|
|
return None
|
|
return dict(payload)
|
|
|
|
|
|
def _store_persistent_summary(session: Session, signature: str, payload: dict[str, Any]) -> None:
|
|
ttl_seconds = _summary_cache_ttl_seconds()
|
|
expires_at = datetime.fromtimestamp(datetime.now(UTC).replace(microsecond=0).timestamp() + ttl_seconds, UTC).isoformat().replace('+00:00', 'Z')
|
|
row = session.get(AppSetting, SUMMARY_CACHE_SETTING_KEY)
|
|
if not row:
|
|
row = AppSetting(key=SUMMARY_CACHE_SETTING_KEY, is_secret=False)
|
|
row.value_json = {
|
|
'schemaVersion': SUMMARY_CACHE_SCHEMA_VERSION,
|
|
'signature': signature,
|
|
'payload': _redact(payload),
|
|
'generatedAt': payload.get('lastUpdated'),
|
|
'expiresAt': expires_at,
|
|
}
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
|
|
|
|
def _snapshot_cached() -> dict[str, Any]:
|
|
key = _snapshot_cache_key()
|
|
now = monotonic()
|
|
cached = _SNAPSHOT_CACHE.get(key)
|
|
if cached and now - cached[0] <= SNAPSHOT_CACHE_TTL_SECONDS:
|
|
return cached[1]
|
|
with _SNAPSHOT_CACHE_LOCK:
|
|
now = monotonic()
|
|
cached = _SNAPSHOT_CACHE.get(key)
|
|
if cached and now - cached[0] <= SNAPSHOT_CACHE_TTL_SECONDS:
|
|
return cached[1]
|
|
data = _build_snapshot()
|
|
_SNAPSHOT_CACHE.clear()
|
|
_SNAPSHOT_CACHE[key] = (now, data)
|
|
return data
|
|
|
|
|
|
def _connector_status(payload: dict[str, Any]) -> str:
|
|
status = str(payload.get('status') or ('connected' if payload.get('success') else 'error')).lower()
|
|
if status in {'ok', 'healthy'}:
|
|
return 'connected'
|
|
if status in {'connected', 'missing_config', 'auth_failed', 'unreachable', 'unsupported', 'unavailable', 'error'}:
|
|
return status
|
|
return 'error'
|
|
|
|
|
|
def _ui_status(connector_status: str, warning: bool = False, critical: bool = False) -> str:
|
|
if connector_status == 'missing_config':
|
|
return 'missing_config'
|
|
if connector_status in {'auth_failed', 'unreachable', 'error'} or critical:
|
|
return 'critical'
|
|
if connector_status in {'unsupported', 'unavailable'} or warning:
|
|
return 'warning'
|
|
return 'healthy'
|
|
|
|
|
|
def _flow_edge_status(source_health: dict[str, Any], target_health: dict[str, Any], *, warning: bool = False) -> str:
|
|
statuses = [str(health.get('status') or ('connected' if health.get('success') else 'unknown')).lower() for health in (source_health, target_health)]
|
|
known_statuses = {'connected', 'ok', 'healthy', 'auth_failed', 'unreachable', 'error'}
|
|
if any(status not in known_statuses for status in statuses):
|
|
return 'unknown'
|
|
if any(status in {'auth_failed', 'unreachable', 'error'} for status in statuses):
|
|
return 'broken'
|
|
return 'warning' if warning else 'healthy'
|
|
|
|
|
|
def _policy_action(session: Session, *, label: str, action_id: str, target: str, source: str = 'manual') -> dict[str, Any]:
|
|
policy = evaluate_policy(session=session, action=action_id, target=target, source=source)
|
|
return {
|
|
'label': label,
|
|
'actionId': action_id,
|
|
'target': target,
|
|
'policyDecision': policy['decision'],
|
|
'risk': policy.get('risk', 'medium'),
|
|
'reason': policy.get('reason'),
|
|
'link': _action_link(action_id, target),
|
|
}
|
|
|
|
|
|
def _action_link(action_id: str, target: str) -> str | None:
|
|
if action_id == 'media_pipeline.diagnose':
|
|
return '/media-pipeline'
|
|
if target in CONNECTOR_IDS:
|
|
return f'/apps/{target}'
|
|
return None
|
|
|
|
|
|
def _filter_safe_actions(actions: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [action for action in actions if not any(token in action.get('actionId', '') for token in BLOCKED_ACTION_TOKENS)]
|
|
|
|
|
|
def _build_snapshot() -> dict[str, Any]:
|
|
calls: dict[str, Callable[[], dict[str, Any]]] = {
|
|
'sonarrHealth': lambda: SonarrConnector().health(),
|
|
'sonarrWanted': lambda: SonarrConnector().wanted_missing(),
|
|
'sonarrQueue': lambda: SonarrConnector().queue_status(),
|
|
'prowlarrHealth': lambda: ProwlarrConnector().health(),
|
|
'prowlarrIndexers': lambda: ProwlarrConnector().indexer_status(),
|
|
'prowlarrApplications': lambda: ProwlarrConnector().application_status(),
|
|
'prowlarrSync': lambda: ProwlarrConnector().sync_status(),
|
|
'delugeHealth': lambda: DelugeConnector().health(),
|
|
'delugeSession': lambda: DelugeConnector().session_status(),
|
|
'delugeActive': lambda: DelugeConnector().active_downloads(),
|
|
'delugeStalled': lambda: DelugeConnector().stalled_downloads(),
|
|
'delugePaused': lambda: DelugeConnector().paused_downloads(),
|
|
'delugeSpeeds': lambda: DelugeConnector().speed_status(),
|
|
'plexHealth': lambda: PlexConnector().health(),
|
|
'plexLibraries': lambda: PlexConnector().library_status(),
|
|
'plexSessions': lambda: PlexConnector().active_sessions(),
|
|
'plexRecent': lambda: PlexConnector().recent_media(),
|
|
'plexStreams': lambda: PlexConnector().stream_summary(),
|
|
'tautulliHealth': lambda: TautulliConnector().health(),
|
|
'tautulliActivity': lambda: TautulliConnector().activity(),
|
|
'tautulliHistory': lambda: TautulliConnector().history(),
|
|
'tautulliStats': lambda: TautulliConnector().watch_stats(),
|
|
'tautulliStreams': lambda: TautulliConnector().stream_summary(),
|
|
'tautulliLibraries': lambda: TautulliConnector().libraries(),
|
|
'widefrogHealth': lambda: WidefrogConnector().health(),
|
|
'widefrogQueue': lambda: WidefrogConnector().queue_status(),
|
|
'widefrogFailed': lambda: WidefrogConnector().failed_jobs(),
|
|
'widefrogActivity': lambda: WidefrogConnector().activity(),
|
|
'widefrogDownloads': lambda: WidefrogConnector().download_status(),
|
|
}
|
|
|
|
with ThreadPoolExecutor(max_workers=min(SNAPSHOT_MAX_WORKERS, len(calls))) as executor:
|
|
futures = {key: executor.submit(_safe_call, call) for key, call in calls.items()}
|
|
return {key: future.result() for key, future in futures.items()}
|
|
|
|
|
|
def _connector_map(data: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
return {
|
|
'sonarr': {'status': _connector_status(data['sonarrHealth']), 'enabled': True},
|
|
'prowlarr': {'status': _connector_status(data['prowlarrHealth']), 'enabled': True},
|
|
'deluge': {'status': _connector_status(data['delugeHealth']), 'enabled': True},
|
|
'plex': {'status': _connector_status(data['plexHealth']), 'enabled': True},
|
|
'tautulli': {'status': _connector_status(data['tautulliHealth']), 'enabled': True},
|
|
'widefrog': {'status': _connector_status(data['widefrogHealth']), 'enabled': True},
|
|
}
|
|
|
|
|
|
def _counts(data: dict[str, Any]) -> dict[str, int]:
|
|
return {
|
|
'wantedSeries': _int(data['sonarrWanted'].get('total')),
|
|
'missingEpisodes': _int(data['sonarrWanted'].get('total')),
|
|
'activeDownloads': _int(data['delugeActive'].get('total', data['delugeSession'].get('active'))),
|
|
'stalledDownloads': _int(data['delugeStalled'].get('total', data['delugeSession'].get('stalled'))),
|
|
'failedWidefrogJobs': _int(data['widefrogFailed'].get('failed')),
|
|
'plexLibraries': _int(data['plexLibraries'].get('total')),
|
|
'activeStreams': max(_int(data['plexStreams'].get('total')), _int(data['tautulliStreams'].get('activeStreams')), _int(data['tautulliActivity'].get('streamCount'))),
|
|
'transcodes': max(_int(data['plexStreams'].get('transcoding')), _int(data['tautulliStreams'].get('transcoding'))),
|
|
'indexers': _int(data['prowlarrIndexers'].get('total')),
|
|
'failingIndexers': _int(data['prowlarrIndexers'].get('failing'), _int(data['prowlarrSync'].get('indexersFailing'))),
|
|
}
|
|
|
|
|
|
def _summary_from_snapshot(data: dict[str, Any]) -> dict[str, Any]:
|
|
connectors = _connector_map(data)
|
|
counts = _counts(data)
|
|
missing = [key for key, value in connectors.items() if value['status'] == 'missing_config']
|
|
hard_failures = [key for key, value in connectors.items() if value['status'] in {'auth_failed', 'unreachable', 'error'}]
|
|
warnings: list[str] = []
|
|
warnings.extend(f'{name} missing_config' for name in missing)
|
|
warnings.extend(f'{name} {connectors[name]["status"]}' for name in hard_failures)
|
|
if counts['failingIndexers']:
|
|
warnings.append('Prowlarr has failing indexers')
|
|
if counts['stalledDownloads']:
|
|
warnings.append('Deluge has stalled downloads')
|
|
if counts['failedWidefrogJobs']:
|
|
warnings.append('Widefrog has failed jobs')
|
|
if counts['transcodes'] >= 3:
|
|
warnings.append('Plex has high active transcodes')
|
|
|
|
if hard_failures:
|
|
status = 'critical'
|
|
risk_level = 'high'
|
|
elif missing:
|
|
status = 'partial' if len(missing) < len(CONNECTOR_IDS) else 'missing_config'
|
|
risk_level = 'medium'
|
|
elif warnings:
|
|
status = 'warning'
|
|
risk_level = 'medium'
|
|
else:
|
|
status = 'healthy'
|
|
risk_level = 'low'
|
|
|
|
return {
|
|
'success': True,
|
|
'status': status,
|
|
'message': 'Media pipeline ready.' if status == 'healthy' else 'Media pipeline has items requiring attention.',
|
|
'connectors': connectors,
|
|
'counts': counts,
|
|
'risk': {'level': risk_level, 'reasons': warnings},
|
|
'lastUpdated': _now(),
|
|
}
|
|
|
|
|
|
def summary(session: Session | None = None, *, force_refresh: bool = False) -> dict[str, Any]:
|
|
signature = _summary_signature(session) if session else None
|
|
if session and signature and not force_refresh:
|
|
persisted = _persistent_summary(session, signature)
|
|
if persisted:
|
|
return persisted
|
|
|
|
data = _snapshot_cached()
|
|
payload = _summary_from_snapshot(data)
|
|
if session and signature:
|
|
_store_persistent_summary(session, signature, payload)
|
|
return payload
|
|
|
|
|
|
def flow(session: Session) -> dict[str, Any]:
|
|
data = _snapshot_cached()
|
|
connectors = _connector_map(data)
|
|
counts = _counts(data)
|
|
nodes = [
|
|
{'id': 'sonarr', 'label': 'Sonarr', 'type': 'wanted', 'status': _ui_status(connectors['sonarr']['status'], counts['wantedSeries'] > 0), 'metrics': {'wanted': counts['wantedSeries'], 'queue': _int(data['sonarrQueue'].get('total'))}, 'actions': []},
|
|
{'id': 'prowlarr', 'label': 'Prowlarr', 'type': 'indexers', 'status': _ui_status(connectors['prowlarr']['status'], counts['failingIndexers'] > 0), 'metrics': {'indexers': counts['indexers'], 'failing': counts['failingIndexers'], 'apps': _int(data['prowlarrApplications'].get('total'))}, 'actions': [_policy_action(session, label='Run search test', action_id='prowlarr.search_test', target='prowlarr')]},
|
|
{'id': 'deluge', 'label': 'Deluge', 'type': 'downloads', 'status': _ui_status(connectors['deluge']['status'], counts['stalledDownloads'] > 0), 'metrics': {'active': counts['activeDownloads'], 'stalled': counts['stalledDownloads'], 'paused': _int(data['delugePaused'].get('total'))}, 'actions': [_policy_action(session, label='Pause all', action_id='deluge.pause_all', target='deluge')]},
|
|
{'id': 'widefrog', 'label': 'Widefrog', 'type': 'processing', 'status': _ui_status(connectors['widefrog']['status'], counts['failedWidefrogJobs'] > 0), 'metrics': {'queue': _int(data['widefrogQueue'].get('total')), 'failed': counts['failedWidefrogJobs']}, 'actions': [_policy_action(session, label='Retry failed', action_id='widefrog.retry_failed', target='widefrog')]},
|
|
{'id': 'plex', 'label': 'Plex', 'type': 'library', 'status': _ui_status(connectors['plex']['status'], counts['transcodes'] >= 3), 'metrics': {'libraries': counts['plexLibraries'], 'streams': counts['activeStreams'], 'transcodes': counts['transcodes']}, 'actions': [_policy_action(session, label='Scan all libraries', action_id='plex.scan_all_libraries', target='plex')]},
|
|
{'id': 'tautulli', 'label': 'Tautulli', 'type': 'analytics', 'status': _ui_status(connectors['tautulli']['status']), 'metrics': {'streams': _int(data['tautulliStreams'].get('activeStreams')), 'plays': _int(data['tautulliStats'].get('totalPlays'))}, 'actions': []},
|
|
]
|
|
edges = [
|
|
{'from': 'sonarr', 'to': 'prowlarr', 'status': _flow_edge_status(data['sonarrHealth'], data['prowlarrHealth']), 'label': 'indexer sync'},
|
|
{'from': 'prowlarr', 'to': 'deluge', 'status': _flow_edge_status(data['prowlarrHealth'], data['delugeHealth'], warning=counts['failingIndexers'] > 0), 'label': 'release search'},
|
|
{'from': 'widefrog', 'to': 'plex', 'status': _flow_edge_status(data['widefrogHealth'], data['plexHealth'], warning=counts['failedWidefrogJobs'] > 0), 'label': 'media processing'},
|
|
{'from': 'plex', 'to': 'tautulli', 'status': _flow_edge_status(data['plexHealth'], data['tautulliHealth']), 'label': 'playback analytics'},
|
|
]
|
|
return {'success': True, 'nodes': nodes, 'edges': edges, 'lastUpdated': _now()}
|
|
|
|
|
|
def issues(session: Session) -> dict[str, Any]:
|
|
data = _snapshot_cached()
|
|
connectors = _connector_map(data)
|
|
counts = _counts(data)
|
|
result: list[dict[str, Any]] = []
|
|
observed_at = _now()
|
|
|
|
for name, connector in connectors.items():
|
|
if connector['status'] == 'missing_config':
|
|
result.append(_issue('connector-missing-' + name, 'warning', f'{name.title()} missing_config', f'{name} connector configuration is incomplete.', [name], [connector['status']], [_configure_action(name)]))
|
|
elif connector['status'] in {'auth_failed', 'unreachable', 'error'}:
|
|
result.append(_issue('connector-error-' + name, 'critical', f'{name.title()} connector unavailable', f'{name} returned {connector["status"]}.', [name], [connector['status']], [_configure_action(name)]))
|
|
|
|
if connectors['sonarr']['status'] == 'connected' and counts['failingIndexers']:
|
|
result.append(_issue('prowlarr-failing-indexers', 'warning', 'Prowlarr failing indexers', 'Sonarr is connected but one or more indexers are failing.', ['sonarr', 'prowlarr'], [f'{counts["failingIndexers"]} failing indexers'], [_policy_action(session, label='Run search test', action_id='prowlarr.search_test', target='prowlarr')]))
|
|
if counts['wantedSeries'] and counts['activeDownloads'] == 0:
|
|
result.append(_issue('wanted-no-downloads', 'info', 'Wanted items without active downloads', 'Sonarr has wanted items but Deluge has no active downloads.', ['sonarr', 'deluge'], [f'{counts["wantedSeries"]} wanted', '0 active downloads'], [_policy_action(session, label='Run Prowlarr search test', action_id='prowlarr.search_test', target='prowlarr')]))
|
|
if counts['stalledDownloads']:
|
|
source = 'deluge.stalled_downloads'
|
|
result.append(_issue(
|
|
'deluge-stalled',
|
|
'warning',
|
|
'Deluge stalled downloads',
|
|
'One or more Deluge downloads are stalled.',
|
|
['deluge'],
|
|
[f'{counts["stalledDownloads"]} stalled downloads', f'source={source}', f'observedAt={observed_at}'],
|
|
[_policy_action(session, label='Resume stalled torrent', action_id='deluge.resume_torrent', target='deluge')],
|
|
source=source,
|
|
observed_at=observed_at,
|
|
))
|
|
if connectors['plex']['status'] == 'connected' and connectors['tautulli']['status'] == 'missing_config':
|
|
result.append(_issue('tautulli-missing-with-plex', 'warning', 'Tautulli analytics incomplete', 'Plex is connected but Tautulli is missing configuration.', ['plex', 'tautulli'], ['Plex connected', 'Tautulli missing_config'], [_configure_action('tautulli')]))
|
|
if counts['transcodes'] >= 3:
|
|
result.append(_issue('plex-high-transcodes', 'warning', 'Plex transcodes active', 'Multiple active transcodes may indicate client or server load.', ['plex', 'tautulli'], [f'{counts["transcodes"]} active transcodes'], []))
|
|
if counts['failedWidefrogJobs']:
|
|
result.append(_issue('widefrog-failed-jobs', 'warning', 'Widefrog failed jobs', 'Widefrog reports failed processing jobs.', ['widefrog'], [f'{counts["failedWidefrogJobs"]} failed jobs'], [_policy_action(session, label='Retry Widefrog failed jobs', action_id='widefrog.retry_failed', target='widefrog')]))
|
|
|
|
result.extend(_needs_review_issues(session))
|
|
for issue in result:
|
|
issue['suggestedActions'] = _filter_safe_actions(issue['suggestedActions'])
|
|
return {'success': True, 'issues': result, 'lastUpdated': _now()}
|
|
|
|
|
|
def _issue(issue_id: str, severity: str, title: str, description: str, affected: list[str], evidence: list[str], actions: list[dict[str, Any]], *, source: str = 'media_pipeline', observed_at: str | None = None) -> dict[str, Any]:
|
|
return {'id': issue_id, 'severity': severity, 'title': title, 'description': description, 'affectedConnectors': affected, 'evidence': evidence, 'suggestedActions': actions, 'source': source, 'observedAt': observed_at or _now()}
|
|
|
|
|
|
def _configure_action(connector: str) -> dict[str, Any]:
|
|
link = f'/setup?step=connectors&connector={connector}' if connector in CONNECTOR_IDS else f'/apps/{connector}'
|
|
return {'label': f'Configure {connector}', 'actionId': 'connector.configure', 'target': connector, 'policyDecision': 'allowed', 'risk': 'low', 'link': link}
|
|
|
|
|
|
def _needs_review_issues(session: Session) -> list[dict[str, Any]]:
|
|
apps = session.exec(select(App).where(col(App.needs_review).is_(True))).all()
|
|
result: list[dict[str, Any]] = []
|
|
for app in apps:
|
|
if (app.category or '').lower() not in {'media', 'downloads'}:
|
|
continue
|
|
target = str(getattr(app, 'connector_type', None) or app.id)
|
|
result.append(
|
|
_issue(
|
|
f'app-needs-review-{app.id}',
|
|
'info',
|
|
f'{app.name} needs review',
|
|
'Imported media app needs operator review before enabling broader actions.',
|
|
[target],
|
|
['needsReview=true'],
|
|
[_configure_action(target)],
|
|
)
|
|
)
|
|
return result
|
|
|
|
|
|
def downloads(session: Session) -> dict[str, Any]:
|
|
data = _snapshot_cached()
|
|
actions = [
|
|
{'label': 'Open Deluge detail', 'actionId': 'app.open', 'target': 'deluge', 'policyDecision': 'allowed', 'risk': 'low', 'link': '/apps/deluge'},
|
|
_policy_action(session, label='Resume stalled torrent', action_id='deluge.resume_torrent', target='deluge'),
|
|
_policy_action(session, label='Retry Widefrog failed', action_id='widefrog.retry_failed', target='widefrog'),
|
|
_policy_action(session, label='Pause all Deluge', action_id='deluge.pause_all', target='deluge'),
|
|
]
|
|
warnings = []
|
|
if _int(data['delugeStalled'].get('total')):
|
|
warnings.append('Deluge has stalled downloads')
|
|
if _int(data['widefrogFailed'].get('failed')):
|
|
warnings.append('Widefrog has failed jobs')
|
|
return {
|
|
'success': True,
|
|
'active': data['delugeActive'],
|
|
'stalled': data['delugeStalled'],
|
|
'paused': data['delugePaused'],
|
|
'failedWidefrogJobs': data['widefrogFailed'],
|
|
'speedSummary': data['delugeSpeeds'],
|
|
'queueSummary': data['widefrogQueue'],
|
|
'warnings': warnings,
|
|
'suggestedActions': _filter_safe_actions(actions),
|
|
'lastUpdated': _now(),
|
|
}
|
|
|
|
|
|
def library(session: Session) -> dict[str, Any]:
|
|
data = _snapshot_cached()
|
|
actions = [
|
|
_policy_action(session, label='Scan Plex library', action_id='plex.scan_library', target='plex'),
|
|
_policy_action(session, label='Scan all Plex libraries', action_id='plex.scan_all_libraries', target='plex'),
|
|
_policy_action(session, label='Scan Widefrog library', action_id='widefrog.scan_library', target='widefrog'),
|
|
]
|
|
return {
|
|
'success': True,
|
|
'plexLibraries': data['plexLibraries'],
|
|
'recentMedia': data['plexRecent'],
|
|
'activeSessions': data['plexSessions'],
|
|
'transcodeSummary': data['plexStreams'],
|
|
'tautulliStats': data['tautulliStats'],
|
|
'tautulliLibraries': data['tautulliLibraries'],
|
|
'widefrogQueue': data['widefrogQueue'],
|
|
'scanSuggestions': _filter_safe_actions(actions),
|
|
'suggestedActions': _filter_safe_actions(actions),
|
|
'lastUpdated': _now(),
|
|
}
|
|
|
|
|
|
def activity() -> dict[str, Any]:
|
|
data = _snapshot_cached()
|
|
return {
|
|
'success': True,
|
|
'tautulliActivity': data['tautulliActivity'],
|
|
'plexSessions': data['plexSessions'],
|
|
'tautulliHistory': data['tautulliHistory'],
|
|
'plexRecent': data['plexRecent'],
|
|
'streamSummary': data['tautulliStreams'],
|
|
'transcodeSummary': data['plexStreams'],
|
|
'privacy': {'sensitive': True, 'actions': ['tautulli.history', 'tautulli.users', 'tautulli.watch_stats'], 'voicePolicy': 'approval_required'},
|
|
'lastUpdated': _now(),
|
|
}
|
|
|
|
|
|
def recommendations(session: Session) -> dict[str, Any]:
|
|
issue_payload = issues(session)
|
|
items = [
|
|
{
|
|
'id': f'rec-{issue["id"]}',
|
|
'severity': issue['severity'],
|
|
'confidence': 0.86 if issue['severity'] == 'warning' else 0.72,
|
|
'title': issue['title'],
|
|
'evidence': issue['evidence'],
|
|
'affectedConnectors': issue['affectedConnectors'],
|
|
'action': issue['suggestedActions'][0] if issue['suggestedActions'] else {'label': 'Review', 'actionId': 'dashboard.review', 'target': 'media-pipeline', 'policyDecision': 'allowed', 'risk': 'low'},
|
|
}
|
|
for issue in issue_payload['issues']
|
|
]
|
|
if not items:
|
|
items.append({'id': 'rec-healthy', 'severity': 'info', 'confidence': 0.7, 'title': 'Media pipeline healthy', 'evidence': ['No active cross-connector issues detected'], 'affectedConnectors': list(CONNECTOR_IDS), 'action': {'label': 'Review dashboard', 'actionId': 'dashboard.review', 'target': 'media-pipeline', 'policyDecision': 'allowed', 'risk': 'low'}})
|
|
return {'success': True, 'recommendations': items, 'lastUpdated': _now()}
|
|
|
|
|
|
def check(session: Session) -> dict[str, Any]:
|
|
steps = ['Check connector statuses', 'Pull summaries', 'Detect issues', 'Write audit event', 'Return media pipeline summary']
|
|
job = create_job(session, action_id='media_pipeline.check', target='media-pipeline', source='manual', risk='low', policy_decision='allowed', steps=steps)
|
|
start_job(session, job)
|
|
set_job_progress(session, job, 45, 'Collected connector summaries.')
|
|
summary_payload = summary(session, force_refresh=True)
|
|
issue_payload = issues(session)
|
|
completed = complete_job(session, job, status='succeeded', result_summary=summary_payload['status'])
|
|
audit = log_event(
|
|
session,
|
|
source='action',
|
|
identity='manual',
|
|
action_id='media_pipeline.check',
|
|
target='media-pipeline',
|
|
risk='low',
|
|
policy_decision='allowed',
|
|
reason='Read-only media pipeline check.',
|
|
result='success',
|
|
execution_plan={'plan': steps, 'jobId': completed.id},
|
|
redacted_metadata={'connectors': summary_payload['connectors'], 'issueCount': len(issue_payload['issues'])},
|
|
)
|
|
return {'success': True, 'summary': summary_payload, 'issues': issue_payload['issues'], 'jobId': completed.id, 'auditId': audit.id}
|
|
|
|
|
|
def diagnose(session: Session, payload: dict[str, Any]) -> dict[str, Any]:
|
|
query = str(payload.get('query') or '').strip()
|
|
scope = str(payload.get('scope') or 'all').strip() or 'all'
|
|
data = _snapshot_cached()
|
|
counts = _counts(data)
|
|
likely_causes: list[str] = []
|
|
evidence: list[str] = []
|
|
actions: list[dict[str, Any]] = []
|
|
|
|
if counts['wantedSeries'] and counts['activeDownloads'] == 0:
|
|
likely_causes.append('Wanted media exists but no download is currently active.')
|
|
evidence.extend([f'{counts["wantedSeries"]} wanted items', '0 active Deluge downloads'])
|
|
actions.append(_policy_action(session, label='Run Prowlarr search test', action_id='prowlarr.search_test', target='prowlarr'))
|
|
if counts['failingIndexers']:
|
|
likely_causes.append('One or more Prowlarr indexers are failing.')
|
|
evidence.append(f'{counts["failingIndexers"]} failing indexers')
|
|
actions.append(_policy_action(session, label='Run Prowlarr search test', action_id='prowlarr.search_test', target='prowlarr'))
|
|
if counts['stalledDownloads']:
|
|
likely_causes.append('Deluge has stalled downloads.')
|
|
evidence.append(f'{counts["stalledDownloads"]} stalled downloads')
|
|
actions.append(_policy_action(session, label='Resume stalled torrent', action_id='deluge.resume_torrent', target='deluge'))
|
|
if counts['failedWidefrogJobs']:
|
|
likely_causes.append('Widefrog has failed processing jobs.')
|
|
evidence.append(f'{counts["failedWidefrogJobs"]} failed Widefrog jobs')
|
|
actions.append(_policy_action(session, label='Retry Widefrog failed', action_id='widefrog.retry_failed', target='widefrog'))
|
|
if counts['transcodes'] >= 3:
|
|
likely_causes.append('Plex has several active transcodes.')
|
|
evidence.append(f'{counts["transcodes"]} transcodes')
|
|
|
|
if not likely_causes:
|
|
likely_causes.append('No obvious rule-based issue found.')
|
|
evidence.append('Connector summaries did not cross configured thresholds')
|
|
|
|
audit = log_event(
|
|
session,
|
|
source='action',
|
|
identity='manual',
|
|
action_id='media_pipeline.diagnose',
|
|
target='media-pipeline',
|
|
risk='low',
|
|
policy_decision='allowed',
|
|
reason='Read-only media pipeline diagnosis.',
|
|
result='success',
|
|
execution_plan={'query': query, 'scope': scope},
|
|
redacted_metadata={'scope': scope},
|
|
)
|
|
return {
|
|
'success': True,
|
|
'diagnosis': {
|
|
'query': query,
|
|
'scope': scope,
|
|
'likelyCauses': likely_causes,
|
|
'evidence': evidence,
|
|
'suggestedActions': _filter_safe_actions(actions),
|
|
'executedActions': [],
|
|
'auditId': audit.id,
|
|
},
|
|
}
|