Files
Unraid-AppOps-Gateway-Public/backend/app/services/inventory_discovery.py
T

1716 lines
68 KiB
Python

from __future__ import annotations
import hashlib
import json
import re
import threading
import time
from datetime import datetime
from typing import Any
from uuid import uuid4
from weakref import WeakKeyDictionary
from sqlalchemy import func
from sqlmodel import Session, delete, select
from ..connectors.portainer import PortainerConnector
from ..core.config import settings
from ..core.redaction import redact_message
from ..core.time import utc_now
from ..db.models import App, AppContainerMapping, AppSetting, DiscoveredContainer, DiscoveryRun
from ..db.session import get_session
from .audit_logger import log_event
from . import safety_center_service
from .app_classifier import classify_container
_SECRET_KEYS = {'env', 'Env', 'environment', 'Config', 'Secrets'}
_SENSITIVE_KEY_TOKENS = {'key', 'token', 'password', 'secret'}
_DRIFT_ACK_SETTING = 'inventory:drift_acknowledged'
_DRIFT_INCIDENTS_SETTING = 'inventory:drift_incidents'
_DRIFT_BASELINE_SETTING = 'inventory:drift_baseline'
_IGNORE_RULES_SETTING = 'inventory:ignore_rules'
_LATEST_BY_CONTAINER_CACHE_TTL_SECONDS = 5.0
_latest_by_container_cache_lock = threading.Lock()
_latest_by_container_cache: WeakKeyDictionary[Session, tuple[float, dict[str, DiscoveredContainer]]] = WeakKeyDictionary()
_CONNECTOR_REQUIRED_SECRETS = {
'portainer': ['PORTAINER_URL', 'PORTAINER_API_KEY', 'PORTAINER_ENDPOINT_ID'],
'unraid': ['UNRAID_URL', 'UNRAID_API_KEY'],
'sonarr': ['SONARR_URL', 'SONARR_API_KEY'],
'radarr': ['RADARR_URL', 'RADARR_API_KEY'],
'widefrog': ['WIDEFROG_URL'],
'plex': ['PLEX_URL', 'PLEX_TOKEN'],
'tautulli': ['TAUTULLI_URL', 'TAUTULLI_API_KEY'],
'deluge': ['DELUGE_URL', 'DELUGE_PASSWORD'],
'prowlarr': ['PROWLARR_URL', 'PROWLARR_API_KEY'],
'nginx_proxy_manager': ['NPM_URL', 'NPM_IDENTITY', 'NPM_SECRET'],
'authentik': ['AUTHENTIK_URL', 'AUTHENTIK_TOKEN'],
'home_assistant': ['HOME_ASSISTANT_URL'],
}
_GENERIC_ACTIONS = [
'container.status',
'container.logs',
'container.restart',
'health.basic',
'portainer.open',
'container.mounts',
'container.ports',
'container.networks',
'container.image',
'audit.recent',
'inventory.import',
'inventory.ignore',
'inventory.map',
]
def _image_repository(image: str) -> str:
value = str(image or '').strip().casefold().split('@', 1)[0]
last_slash = value.rfind('/')
last_colon = value.rfind(':')
return value[:last_colon] if last_colon > last_slash else value
def _ignore_signature(name: str, image: str) -> dict[str, str]:
return {'name': str(name or '').strip().casefold(), 'imageRepository': _image_repository(image)}
def _is_ephemeral_container(name: str, image: str) -> bool:
value = f'{name} {image}'.casefold()
if image.casefold().startswith('mcr.microsoft.com/dotnet/sdk:'):
return True
return bool(re.search(
r'(^|\s)(gitea-actions-task-\d+-workflow-|buildx_buildkit_|testcontainers[-_ ]ryuk)'
r'|(?:^|[-_ ])candidate[-_]\d+'
r'|(?:^|[-_ ])phase\d+[a-z]?(?:[-_ ]|$)',
value,
))
def _ignore_signatures(session: Session) -> set[tuple[str, str]]:
signatures: set[tuple[str, str]] = set()
row = session.get(AppSetting, _IGNORE_RULES_SETTING)
rules = row.value_json.get('rules', []) if row and isinstance(row.value_json, dict) else []
for rule in rules:
if isinstance(rule, dict):
signatures.add((str(rule.get('name') or '').casefold(), str(rule.get('imageRepository') or '').casefold()))
ignored = session.exec(select(DiscoveredContainer).where(DiscoveredContainer.import_status == 'ignored')).all()
for item in ignored:
signature = _ignore_signature(item.name, item.image)
signatures.add((signature['name'], signature['imageRepository']))
return signatures
def _store_ignore_signature(session: Session, item: DiscoveredContainer) -> None:
signature = _ignore_signature(item.name, item.image)
row = session.get(AppSetting, _IGNORE_RULES_SETTING) or AppSetting(key=_IGNORE_RULES_SETTING, is_secret=False)
existing = row.value_json.get('rules', []) if isinstance(row.value_json, dict) else []
rules = [rule for rule in existing if isinstance(rule, dict)]
if signature not in rules:
rules.append(signature)
row.value_json = {'version': 1, 'rules': rules[-500:]}
row.updated_at = utc_now()
session.add(row)
_DEFAULT_SETUP_CHECKLIST = [
'Confirm app type',
'Confirm category',
'Confirm criticality',
'Confirm container mapping',
'Select connector if available',
'Configure secret if needed',
'Add healthcheck',
'Review policies',
'Enable voice actions, optional',
]
def _id(prefix: str) -> str:
return f'{prefix}-{uuid4().hex[:12]}'
def _redact_value(value: Any) -> Any:
if isinstance(value, dict):
return {
str(key): '[REDACTED]' if any(token in str(key).lower() for token in _SENSITIVE_KEY_TOKENS) else _redact_value(nested)
for key, nested in value.items()
if str(key) not in _SECRET_KEYS and 'env' not in str(key).lower()
}
if isinstance(value, list):
return [_redact_value(item) for item in value if not (isinstance(item, str) and '=' in item and any(token in item.lower() for token in ['key', 'token', 'password', 'secret']))]
if isinstance(value, str):
return redact_message(value)
return value
def _json_clone(value: Any, fallback: Any) -> Any:
try:
return json.loads(json.dumps(_redact_value(value)))
except (TypeError, ValueError):
return fallback
def _canonical_list(value: Any) -> list[Any]:
cloned = _json_clone(value, [])
if not isinstance(cloned, list):
return []
return sorted(
cloned,
key=lambda item: json.dumps(item, sort_keys=True, separators=(',', ':'), default=str),
)
def _normalize_container(raw: dict[str, Any]) -> dict[str, Any]:
container_id = str(raw.get('container_id') or raw.get('id') or raw.get('Id') or raw.get('name') or 'unknown')
name = str(raw.get('name') or raw.get('Name') or container_id).strip('/').strip() or container_id
labels = raw.get('labels') or raw.get('Labels') or {}
ports = raw.get('ports') or raw.get('Ports') or []
mounts = raw.get('mounts') or raw.get('Mounts') or []
networks = raw.get('networks') or raw.get('Networks') or []
if isinstance(networks, dict):
networks = list(networks.keys())
return {
'container_id': container_id,
'name': name,
'image': str(raw.get('image') or raw.get('Image') or 'unknown'),
'state': str(raw.get('state') or raw.get('State') or 'unknown').lower(),
'status': str(raw.get('status') or raw.get('Status') or raw.get('health') or 'unknown'),
'ports': _canonical_list(ports),
'mounts': _canonical_list(mounts),
'networks': _canonical_list(networks),
'labels': _json_clone(labels if isinstance(labels, dict) else {}, {}),
'created': raw.get('created') or raw.get('Created'),
'restart_count': int(raw.get('restartCount') or raw.get('RestartCount') or 0),
'raw_metadata': _json_clone(raw, {}),
}
def _serialize_run(run: DiscoveryRun) -> dict[str, Any]:
return {
'id': run.id,
'source': run.source,
'startedAt': run.started_at.isoformat(),
'finishedAt': run.finished_at.isoformat() if run.finished_at else None,
'status': run.status,
'containersFound': run.containers_found,
'appsMatched': run.apps_matched,
'newItems': run.new_items,
'changedItems': run.changed_items,
'errors': run.errors_json,
}
def serialize_discovered(item: DiscoveredContainer) -> dict[str, Any]:
raw = item.raw_metadata_json or {}
classifier = raw.get('classification') if isinstance(raw.get('classification'), dict) else {}
return {
'id': item.id,
'discoveryRunId': item.discovery_run_id,
'containerId': item.container_id,
'name': item.name,
'image': item.image,
'state': item.state,
'status': item.status,
'ports': item.ports_json,
'mounts': item.mounts_json,
'networks': item.networks_json,
'labels': item.labels_json,
'suggestedAppId': classifier.get('suggested_app_id') or item.suggested_app_type,
'displayName': classifier.get('display_name') or item.name,
'suggestedAppType': item.suggested_app_type,
'suggestedCategory': item.suggested_category,
'suggestedCriticality': item.suggested_criticality,
'suggestedConnector': item.suggested_connector,
'suggestedPolicyProfile': item.suggested_policy_profile,
'protected': bool(classifier.get('protected', False)),
'confidence': float(classifier.get('confidence', 0)),
'reasons': classifier.get('reasons', []),
'importStatus': item.import_status,
'matchedAppId': item.matched_app_id,
'source': 'portainer',
'lifecycleState': item.import_status,
'appType': item.suggested_app_type,
'connectorType': _connector_type(item),
'connectorStatus': _connector_status(item),
'connectorRequiredSecrets': _connector_required_secrets(item),
'isManaged': item.import_status == 'managed',
'isImported': item.import_status == 'managed',
'isProtected': bool(classifier.get('protected', False)),
'policyProfile': item.suggested_policy_profile,
'privacySensitive': bool(classifier.get('privacy_sensitive', False)),
'actionsAvailable': _actions_available(item),
'setupChecklist': _setup_checklist(item),
}
def _connector_type(item: DiscoveredContainer) -> str:
connector = str(item.suggested_connector or 'none')
return 'none' if connector in {'', 'none', 'prowlarr_future'} else connector
def _connector_required_secrets(item: DiscoveredContainer) -> list[str]:
return _CONNECTOR_REQUIRED_SECRETS.get(_connector_type(item), [])
def connector_required_secrets(connector_id: str) -> list[str]:
return _CONNECTOR_REQUIRED_SECRETS.get(connector_id, [])
def connector_missing_secrets(connector_id: str) -> list[str]:
return [name for name in connector_required_secrets(connector_id) if not settings.apis.get(name)]
def connector_runtime_status(session: Session | None, connector_id: str) -> str:
connector = str(connector_id or 'none')
if connector in {'', 'none', 'prowlarr_future'}:
return 'none'
if session is not None:
from .setup_service import apply_stored_connector_settings
apply_stored_connector_settings(session)
return 'missing_config' if connector_missing_secrets(connector) else 'connected'
def _connector_status(item: DiscoveredContainer) -> str:
connector = _connector_type(item)
if connector == 'none':
return 'none'
missing = connector_missing_secrets(connector)
return 'missing_config' if missing else 'connected'
def _actions_available(item: DiscoveredContainer) -> list[str]:
actions = list(_GENERIC_ACTIONS)
connector = _connector_type(item)
if connector != 'none':
actions.append(f'{connector}.capabilities')
return actions
def _setup_checklist(item: DiscoveredContainer) -> list[str]:
connector = _connector_type(item)
if connector == 'none':
return list(_DEFAULT_SETUP_CHECKLIST)
if connector == 'nginx_proxy_manager':
return [
'Confirm NPM URL',
'Configure NPM_IDENTITY',
'Configure NPM_SECRET',
'Test connector',
'Review exposure policies',
'Bind connector',
]
if connector == 'authentik':
return [
'Confirm Authentik URL',
'Configure AUTHENTIK_TOKEN',
'Test connector',
'Review outpost health',
'Review auth coverage',
'Bind connector',
]
return [
'Confirm app type',
'Confirm category',
'Confirm criticality',
'Confirm container mapping',
f'Select {connector} connector',
'Configure connector secret if needed',
'Test connector',
'Review policies',
'Enable voice actions, optional',
]
def serialize_inventory_app(item: DiscoveredContainer) -> dict[str, Any]:
discovered = serialize_discovered(item)
display_name = str(discovered['displayName'])
connector = str(discovered['connectorType'])
lifecycle = str(discovered['lifecycleState'])
health = 'missing' if lifecycle == 'missing' else ('healthy' if item.state == 'running' else 'warning')
return {
'id': f'discovered-{item.id}',
'display_name': display_name,
'name': display_name,
'app_type': item.suggested_app_type,
'category': item.suggested_category,
'criticality': item.suggested_criticality,
'lifecycle_state': lifecycle,
'container_names': [item.name],
'primary_container': item.name,
'connector_type': connector,
'connector_status': discovered['connectorStatus'],
'connector_required_secrets': discovered['connectorRequiredSecrets'],
'is_protected': discovered['isProtected'],
'policy_profile': item.suggested_policy_profile,
'health_status': health,
'source': 'portainer',
'actions_available': discovered['actionsAvailable'],
'setup_checklist': discovered['setupChecklist'],
'discovery_confidence': discovered['confidence'],
'discovery_reasons': discovered['reasons'],
'discovery': discovered,
# Legacy frontend-compatible fields.
'status': 'warning' if lifecycle in {'unmapped', 'changed'} else health,
'connectorStatus': discovered['connectorStatus'],
'containerStatus': item.state,
'voiceEnabled': False,
'protected': discovered['isProtected'],
'discoveryStatus': lifecycle,
'discoverySource': 'portainer',
'needsReview': lifecycle != 'managed',
'suggestedAppId': discovered['suggestedAppId'],
'containerId': item.container_id,
}
def _discovery_run_order(session: Session) -> dict[str, datetime]:
return {run.id: run.started_at for run in session.exec(select(DiscoveryRun)).all()}
def invalidate_discovery_cache() -> None:
with _latest_by_container_cache_lock:
_latest_by_container_cache.clear()
def _latest_by_container(session: Session) -> dict[str, DiscoveredContainer]:
now = time.monotonic()
with _latest_by_container_cache_lock:
cached = _latest_by_container_cache.get(session)
if cached is not None and now < cached[0]:
return dict(cached[1])
ranked = (
select(
DiscoveredContainer.id.label('discovered_id'),
func.row_number().over(
partition_by=DiscoveredContainer.container_id,
order_by=(DiscoveryRun.started_at.desc(), DiscoveredContainer.id.desc()),
).label('latest_rank'),
)
.join(DiscoveryRun, DiscoveryRun.id == DiscoveredContainer.discovery_run_id, isouter=True)
.subquery()
)
rows = session.exec(
select(DiscoveredContainer)
.join(ranked, DiscoveredContainer.id == ranked.c.discovered_id)
.where(ranked.c.latest_rank == 1)
).all()
latest = {row.container_id: row for row in rows}
with _latest_by_container_cache_lock:
_latest_by_container_cache[session] = (
time.monotonic() + _LATEST_BY_CONTAINER_CACHE_TTL_SECONDS,
dict(latest),
)
return latest
def latest_discovered_by_container(session: Session) -> dict[str, DiscoveredContainer]:
"""Return the current inventory snapshot without exposing history to consumers."""
return _latest_by_container(session)
def _recent_by_container(session: Session, limit: int = 2) -> dict[str, list[DiscoveredContainer]]:
ranked = (
select(
DiscoveredContainer.id.label('discovered_id'),
DiscoveredContainer.container_id.label('container_id'),
func.row_number().over(
partition_by=DiscoveredContainer.container_id,
order_by=(DiscoveryRun.started_at.desc(), DiscoveredContainer.id.desc()),
).label('recent_rank'),
)
.join(DiscoveryRun, DiscoveryRun.id == DiscoveredContainer.discovery_run_id, isouter=True)
.subquery()
)
rows = session.exec(
select(DiscoveredContainer)
.join(ranked, DiscoveredContainer.id == ranked.c.discovered_id)
.where(ranked.c.recent_rank <= max(1, limit))
.order_by(ranked.c.container_id, ranked.c.recent_rank.desc())
).all()
grouped: dict[str, list[DiscoveredContainer]] = {}
for row in rows:
grouped.setdefault(row.container_id, []).append(row)
return grouped
def _has_changed(previous: DiscoveredContainer | None, normalized: dict[str, Any]) -> bool:
if not previous:
return False
return (
previous.image != normalized['image']
or _canonical_list(previous.ports_json) != _canonical_list(normalized['ports'])
or _canonical_list(previous.mounts_json) != _canonical_list(normalized['mounts'])
or _canonical_list(previous.networks_json) != _canonical_list(normalized['networks'])
or previous.status != normalized['status']
)
def run_discovery(source: str = 'portainer', session: Session | None = None) -> dict[str, Any]:
owns_session = session is None
if session is None:
session = next(get_session())
run = DiscoveryRun(id=_id('disc'), source=source, status='running')
session.add(run)
session.commit()
session.refresh(run)
connector = PortainerConnector()
connection = connector.test_connection()
if connection.get('status') == 'missing_config':
run.status = 'missing_config'
run.finished_at = utc_now()
run.errors_json = [{'status': 'missing_config', 'reason': 'missing_config'}]
session.add(run)
session.commit()
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
result = {'success': False, 'runId': run.id, 'status': 'missing_config', 'containersFound': 0, 'appsMatched': 0, 'newItems': 0, 'changedItems': 0, 'errors': run.errors_json}
if owns_session:
session.close()
return result
containers_result = connector.list_containers()
if containers_result.get('status') != 'ok':
run.status = 'failed'
run.finished_at = utc_now()
run.errors_json = [{'status': containers_result.get('status'), 'reason': redact_message(str(containers_result.get('reason', 'discovery_failed')))}]
session.add(run)
session.commit()
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
result = {'success': False, 'runId': run.id, 'status': 'failed', 'containersFound': 0, 'appsMatched': 0, 'newItems': 0, 'changedItems': 0, 'errors': run.errors_json}
if owns_session:
session.close()
return result
history_retention = {'removed': 0, 'retainedRuns': None}
if settings.discovery_history_runs > 0:
history_retention = prune_discovery_history(session, settings.discovery_history_runs)
previous = _latest_by_container(session)
existing_mappings = session.exec(select(AppContainerMapping)).all()
mappings_by_container_id = {mapping.container_id: mapping for mapping in existing_mappings}
mappings_by_name_and_app: dict[tuple[str, str], list[AppContainerMapping]] = {}
for mapping in existing_mappings:
key = (mapping.container_name.strip().casefold(), mapping.app_id)
mappings_by_name_and_app.setdefault(key, []).append(mapping)
current_ids: set[str] = set()
apps_matched = 0
new_items = 0
changed_items = 0
missing_items = 0
reconciled_mappings = 0
duplicate_mappings_removed = 0
ignored_signatures = _ignore_signatures(session)
for raw in containers_result.get('containers', []):
if not isinstance(raw, dict):
continue
normalized = _normalize_container(raw)
current_ids.add(normalized['container_id'])
classification = classify_container(normalized)
matched_app = session.get(App, classification['suggested_app_id'])
mapped = mappings_by_container_id.get(normalized['container_id'])
mapping_candidates = mappings_by_name_and_app.get(
(normalized['name'].strip().casefold(), matched_app.id),
[],
) if matched_app else []
if not mapped and mapping_candidates:
mapped = max(mapping_candidates, key=lambda item: (item.last_seen_at, item.id))
mappings_by_container_id.pop(mapped.container_id, None)
mapped.container_id = normalized['container_id']
reconciled_mappings += 1
if mapped:
for duplicate in mapping_candidates:
if duplicate.id == mapped.id:
continue
mappings_by_container_id.pop(duplicate.container_id, None)
session.delete(duplicate)
duplicate_mappings_removed += 1
mapped.container_name = normalized['name']
mapped.last_seen_run_id = run.id
mapped.last_seen_at = utc_now()
session.add(mapped)
mappings_by_container_id[normalized['container_id']] = mapped
changed = _has_changed(previous.get(normalized['container_id']), normalized)
if matched_app:
apps_matched += 1
if normalized['container_id'] not in previous:
new_items += 1
if changed:
changed_items += 1
import_status = classification['import_status']
signature = _ignore_signature(normalized['name'], normalized['image'])
is_ignored = _is_ephemeral_container(normalized['name'], normalized['image']) or (
signature['name'], signature['imageRepository']
) in ignored_signatures
if is_ignored:
import_status = 'ignored'
elif changed:
import_status = 'changed'
elif mapped:
import_status = 'managed'
item = DiscoveredContainer(
id=_id('dc'),
discovery_run_id=run.id,
container_id=normalized['container_id'],
name=normalized['name'],
image=normalized['image'],
state=normalized['state'],
status=normalized['status'],
ports_json=normalized['ports'],
mounts_json=normalized['mounts'],
networks_json=normalized['networks'],
labels_json=normalized['labels'],
suggested_app_type=classification['app_type'],
suggested_category=classification['category'],
suggested_criticality=classification['criticality'],
suggested_connector=classification['connector'],
suggested_policy_profile=classification['policy_profile'],
import_status=import_status,
matched_app_id=matched_app.id if matched_app else None,
raw_metadata_json={**normalized['raw_metadata'], 'classification': classification},
)
session.add(item)
for container_id, old in previous.items():
if container_id not in current_ids and old.import_status not in {'ignored', 'missing'}:
missing = DiscoveredContainer(
id=_id('dc'),
discovery_run_id=run.id,
container_id=old.container_id,
name=old.name,
image=old.image,
state='missing',
status='missing',
ports_json=old.ports_json,
mounts_json=old.mounts_json,
networks_json=old.networks_json,
labels_json=old.labels_json,
suggested_app_type=old.suggested_app_type,
suggested_category=old.suggested_category,
suggested_criticality=old.suggested_criticality,
suggested_connector=old.suggested_connector,
suggested_policy_profile=old.suggested_policy_profile,
import_status='missing',
matched_app_id=old.matched_app_id,
raw_metadata_json=old.raw_metadata_json,
)
session.add(missing)
missing_items += 1
run.status = 'success'
run.finished_at = utc_now()
run.containers_found = len(current_ids)
run.apps_matched = apps_matched
run.new_items = new_items
run.changed_items = changed_items
session.add(run)
session.commit()
invalidate_discovery_cache()
sync_app_connector_statuses(session)
if new_items or changed_items or missing_items or reconciled_mappings or duplicate_mappings_removed:
safety_center_service.invalidate_cache()
if reconciled_mappings or duplicate_mappings_removed:
log_event(
session,
source='inventory',
identity='system',
action_id='inventory.mapping.reconcile',
target=run.id,
target_type='discovery_run',
risk='low',
policy_decision='allowed',
reason='Reconciled AppOps metadata for same-name container recreations; no external system was mutated.',
result='success',
after_state={
'reconciledMappings': reconciled_mappings,
'duplicateMappingsRemoved': duplicate_mappings_removed,
},
redacted_metadata={'runId': run.id, 'metadataOnly': True},
)
result = {
'success': True,
'runId': run.id,
'status': 'success',
'containersFound': run.containers_found,
'appsMatched': run.apps_matched,
'newItems': run.new_items,
'changedItems': run.changed_items,
'reconciledMappings': reconciled_mappings,
'duplicateMappingsRemoved': duplicate_mappings_removed,
'historyRowsRemoved': history_retention['removed'],
'historyRunsRetained': history_retention['retainedRuns'],
'errors': [],
}
if owns_session:
session.close()
return result
def import_discovered_container(session: Session, discovered_container_id: str) -> dict[str, Any]:
from .app_registry import category_label
item = session.get(DiscoveredContainer, discovered_container_id)
if not item:
return {'success': False, 'message': 'discovered_container_not_found'}
raw = item.raw_metadata_json or {}
classification = raw.get('classification') if isinstance(raw.get('classification'), dict) else classify_container({'name': item.name, 'image': item.image, 'ports': item.ports_json})
app_id = str(classification.get('suggested_app_id') or item.suggested_app_type or item.name).replace('-', '_')
protected = bool(classification.get('protected', False))
connector = str(classification.get('connector') or item.suggested_connector or 'none')
connector_status = connector_runtime_status(session, connector)
app = session.get(App, app_id)
if not app:
app = App(
id=app_id,
name=str(classification.get('display_name') or item.name),
category=category_label(item.suggested_category),
status='healthy' if item.state == 'running' else 'warning',
criticality=item.suggested_criticality,
connector_status=connector_status,
container_status=item.state if item.state else 'running',
voice_enabled=False,
protected=protected,
discovery_status='managed',
discovery_source='portainer',
needs_review=True,
)
else:
app.category = category_label(item.suggested_category)
app.criticality = item.suggested_criticality
app.protected = app.protected or protected
app.voice_enabled = False
app.discovery_status = 'managed'
app.discovery_source = 'portainer'
app.needs_review = True
session.add(app)
mapping = session.exec(select(AppContainerMapping).where(AppContainerMapping.container_id == item.container_id)).first()
if not mapping:
mapping = AppContainerMapping(id=_id('map'), app_id=app_id, container_id=item.container_id, container_name=item.name, role='primary', source='discovery', is_primary=True, last_seen_run_id=item.discovery_run_id)
else:
mapping.app_id = app_id
mapping.container_name = item.name
mapping.last_seen_run_id = item.discovery_run_id
mapping.last_seen_at = utc_now()
session.add(mapping)
item.import_status = 'managed'
item.matched_app_id = app_id
session.add(item)
session.commit()
session.refresh(app)
session.refresh(mapping)
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
return {
'success': True,
'message': 'imported',
'app': _serialize_app(app),
'mapping': serialize_mapping(mapping),
}
def update_discovered_review_metadata(session: Session, discovered_container_id: str, updates: dict[str, Any]) -> dict[str, Any]:
item = session.get(DiscoveredContainer, discovered_container_id)
if not item:
return {'success': False, 'message': 'discovered_container_not_found'}
if item.import_status in {'managed', 'ignored', 'missing'}:
return {'success': False, 'message': 'discovered_container_not_editable'}
raw = dict(item.raw_metadata_json) if isinstance(item.raw_metadata_json, dict) else {}
classification = raw.get('classification') if isinstance(raw.get('classification'), dict) else {}
classification = dict(classification)
def text_value(key: str, limit: int = 80) -> str | None:
value = updates.get(key)
if not isinstance(value, str):
return None
value = value.strip()
return value[:limit] if value else None
display_name = text_value('displayName')
if display_name:
classification['display_name'] = display_name
category = text_value('category', 40)
if category:
item.suggested_category = re.sub(r'[^a-zA-Z0-9_\- ]', '', category).strip().lower().replace(' ', '_') or item.suggested_category
criticality = text_value('criticality', 20)
if criticality in {'low', 'medium', 'high', 'critical'}:
item.suggested_criticality = criticality
classification['criticality'] = criticality
connector = text_value('connectorType', 40)
if connector:
normalized_connector = re.sub(r'[^a-zA-Z0-9_\-]', '', connector).lower()
item.suggested_connector = normalized_connector or 'none'
classification['connector'] = item.suggested_connector
policy_profile = text_value('policyProfile', 40)
if policy_profile:
item.suggested_policy_profile = re.sub(r'[^a-zA-Z0-9_\-]', '', policy_profile).lower() or item.suggested_policy_profile
raw['classification'] = classification
item.raw_metadata_json = raw
session.add(item)
session.commit()
session.refresh(item)
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
log_event(
session,
source='manual',
identity='operator',
action_id='inventory.review.update_metadata',
target=item.name,
target_type='inventory',
risk='low',
policy_decision='allowed',
reason='Operator updated AppOps discovery metadata only; no container or external service was mutated.',
result='success',
redacted_metadata={'discoveredContainerId': item.id, 'fields': sorted(updates.keys())},
)
return {'success': True, 'message': 'review_metadata_updated', 'discovered': serialize_discovered(item)}
def map_discovered_to_existing_app(session: Session, discovered_container_id: str, app_id: str) -> dict[str, Any]:
item = session.get(DiscoveredContainer, discovered_container_id)
if not item:
return {'success': False, 'message': 'discovered_container_not_found'}
app = session.get(App, app_id)
if not app:
return {'success': False, 'message': 'app_not_found'}
if item.import_status in {'ignored', 'missing'}:
return {'success': False, 'message': 'discovered_container_not_mappable'}
mapping = session.exec(select(AppContainerMapping).where(AppContainerMapping.container_id == item.container_id)).first()
if not mapping:
mapping = AppContainerMapping(
id=_id('map'),
app_id=app.id,
container_id=item.container_id,
container_name=item.name,
role='primary',
source='manual_review',
is_primary=True,
last_seen_run_id=item.discovery_run_id,
)
else:
mapping.app_id = app.id
mapping.container_name = item.name
mapping.source = 'manual_review'
mapping.last_seen_run_id = item.discovery_run_id
mapping.last_seen_at = utc_now()
session.add(mapping)
app.container_status = item.state if item.state else app.container_status
app.discovery_status = 'managed'
app.discovery_source = 'portainer'
app.voice_enabled = False
app.needs_review = True
app.connector_status = connector_runtime_status(session, _connector_type(item))
session.add(app)
item.import_status = 'managed'
item.matched_app_id = app.id
session.add(item)
session.commit()
session.refresh(app)
session.refresh(mapping)
session.refresh(item)
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
log_event(
session,
source='manual',
identity='operator',
action_id='inventory.review.map_existing',
target=app.id,
target_type='app',
risk='low',
policy_decision='allowed',
reason='Operator mapped a discovered container to an existing AppOps app; no container or external service was mutated.',
result='success',
redacted_metadata={'discoveredContainerId': item.id, 'appId': app.id, 'mappingId': mapping.id},
)
return {
'success': True,
'message': 'mapped_existing_app',
'app': _serialize_app(app),
'mapping': serialize_mapping(mapping),
'discovered': serialize_discovered(item),
}
def ignore_discovered_container(session: Session, discovered_container_id: str) -> dict[str, Any]:
item = session.get(DiscoveredContainer, discovered_container_id)
if not item:
return {'success': False, 'message': 'discovered_container_not_found'}
item.import_status = 'ignored'
_store_ignore_signature(session, item)
session.add(item)
session.commit()
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
return {'success': True, 'message': 'ignored', 'id': item.id}
def complete_app_review(session: Session, *, app_ids: list[str] | None = None) -> dict[str, Any]:
query = select(App).where(App.discovery_status == 'managed')
if app_ids:
query = query.where(App.id.in_(app_ids))
apps = session.exec(query).all()
reviewed = 0
for app in apps:
if app.needs_review:
app.needs_review = False
session.add(app)
reviewed += 1
latest = _latest_by_container(session)
mapped_container_ids = {mapping.container_id for mapping in session.exec(select(AppContainerMapping)).all()}
drift_reviewed = 0
for item in latest.values():
if item.import_status == 'changed' and item.container_id in mapped_container_ids:
item.import_status = 'managed'
session.add(item)
drift_reviewed += 1
session.commit()
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
log_event(
session,
source='manual',
identity='setup',
action_id='inventory.review.complete',
target='inventory',
target_type='inventory',
risk='low',
policy_decision='allowed',
reason='Operator completed discovery import review; no containers were mutated and voice remains disabled.',
result='success',
redacted_metadata={'reviewed': reviewed, 'driftReviewed': drift_reviewed, 'scope': 'selected' if app_ids else 'all_managed'},
)
return {
'success': True,
'status': 'success',
'message': 'review_completed',
'reviewed': reviewed,
'driftReviewed': drift_reviewed,
'scope': 'selected' if app_ids else 'all_managed',
}
def prune_historical_discovery_rows(session: Session) -> dict[str, Any]:
latest_ids = {item.id for item in _latest_by_container(session).values()}
all_rows = session.exec(select(DiscoveredContainer)).all()
stale_ids = [row.id for row in all_rows if row.id not in latest_ids]
if stale_ids:
session.exec(delete(DiscoveredContainer).where(DiscoveredContainer.id.in_(stale_ids)))
session.commit()
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
retention = settings.discovery_history_runs if settings.discovery_history_runs > 0 else 48
history = prune_discovery_history(session, retention)
return {
'success': True,
'status': 'success',
'removed': len(stale_ids) + int(history['removed']),
'runsRemoved': int(history['runsRemoved']),
'kept': len(latest_ids),
'retainedRuns': int(history['retainedRuns']),
}
def prune_discovery_history(session: Session, keep_runs: int) -> dict[str, Any]:
"""Bound detailed discovery history and its summary ledger."""
retention = max(1, int(keep_runs))
retained_run_ids = list(session.exec(
select(DiscoveryRun.id)
.where(DiscoveryRun.status == 'success')
.order_by(DiscoveryRun.started_at.desc())
.limit(retention)
).all())
removed = 0
if retained_run_ids:
result = session.exec(
delete(DiscoveredContainer).where(DiscoveredContainer.discovery_run_id.notin_(retained_run_ids))
)
removed = max(0, int(getattr(result, 'rowcount', 0) or 0))
summary_run_ids = list(session.exec(
select(DiscoveryRun.id)
.order_by(DiscoveryRun.started_at.desc())
.limit(retention)
).all())
referenced_run_ids = select(DiscoveredContainer.discovery_run_id).distinct()
run_result = session.exec(
delete(DiscoveryRun).where(
DiscoveryRun.id.notin_(summary_run_ids),
DiscoveryRun.id.notin_(referenced_run_ids),
)
)
runs_removed = max(0, int(getattr(run_result, 'rowcount', 0) or 0))
if removed or runs_removed:
session.commit()
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
return {
'success': True,
'status': 'success',
'removed': removed,
'runsRemoved': runs_removed,
'retainedRuns': len(retained_run_ids),
}
def sync_app_connector_statuses(session: Session) -> dict[str, Any]:
from .setup_service import apply_stored_connector_settings
apply_stored_connector_settings(session)
latest_by_container = _latest_by_container(session)
mappings = session.exec(select(AppContainerMapping)).all()
connector_by_app: dict[str, str] = {}
containers_by_app: dict[str, list[DiscoveredContainer]] = {}
for mapping in mappings:
discovered = latest_by_container.get(mapping.container_id)
if discovered:
connector_by_app[mapping.app_id] = _connector_type(discovered)
containers_by_app.setdefault(mapping.app_id, []).append(discovered)
updated = 0
for app in session.exec(select(App)).all():
connector = connector_by_app.get(app.id)
if (connector is None or connector == 'none') and app.id in _CONNECTOR_REQUIRED_SECRETS:
connector = app.id
if connector is None:
connector = 'none'
next_status = connector_runtime_status(None, connector)
if app.connector_status != next_status:
app.connector_status = next_status
updated += 1
mapped_containers = containers_by_app.get(app.id, [])
active_containers = [
item for item in mapped_containers if item.import_status != 'ignored'
]
if active_containers:
app_name_keys = {
_container_name_key(app.id.replace('_', '-')),
_container_name_key(app.name),
}
canonical_containers = [
item
for item in active_containers
if _container_name_key(item.name) in app_name_keys
]
health_containers = canonical_containers or active_containers
unhealthy_states = {
'dead',
'error',
'exited',
'missing',
'stopped',
'unhealthy',
'warning',
}
unhealthy = [
item
for item in health_containers
if str(item.state or '').lower() in unhealthy_states
or str(item.status or '').lower() in unhealthy_states
]
next_health = 'warning' if unhealthy else 'healthy'
next_container_status = (
str(unhealthy[0].state or unhealthy[0].status or 'unknown')
if unhealthy
else 'running'
)
elif mapped_containers:
# Ignored one-shot migration and rollback containers are complete,
# not unhealthy long-running services.
next_health = 'healthy'
next_container_status = 'ignored'
else:
next_health = app.status
next_container_status = app.container_status
if app.status != next_health:
app.status = next_health
updated += 1
if app.container_status != next_container_status:
app.container_status = next_container_status
updated += 1
session.add(app)
if updated:
session.commit()
safety_center_service.invalidate_cache()
return {'success': True, 'status': 'success', 'updated': updated}
def serialize_mapping(mapping: AppContainerMapping) -> dict[str, Any]:
return {
'id': mapping.id,
'appId': mapping.app_id,
'containerId': mapping.container_id,
'containerName': mapping.container_name,
'role': mapping.role,
'source': mapping.source,
'isPrimary': mapping.is_primary,
'lastSeenRunId': mapping.last_seen_run_id,
'lastSeenAt': mapping.last_seen_at.isoformat() if mapping.last_seen_at else None,
}
def _serialize_app(app: App) -> dict[str, Any]:
return {
'id': app.id,
'name': app.name,
'category': app.category,
'status': app.status,
'criticality': app.criticality,
'connectorStatus': app.connector_status,
'containerStatus': app.container_status,
'voiceEnabled': app.voice_enabled,
'lastAction': app.last_action,
'protected': app.protected,
'discoveryStatus': app.discovery_status,
'discoverySource': app.discovery_source,
'needsReview': app.needs_review,
}
def list_runs(session: Session, *, limit: int = 100, offset: int = 0) -> list[dict[str, Any]]:
bounded_limit = max(1, min(int(limit), 500))
bounded_offset = max(0, int(offset))
query = (
select(DiscoveryRun)
.order_by(DiscoveryRun.started_at.desc())
.offset(bounded_offset)
.limit(bounded_limit)
)
return [_serialize_run(run) for run in session.exec(query).all()]
def get_run_detail(session: Session, run_id: str) -> dict[str, Any] | None:
run = session.get(DiscoveryRun, run_id)
if not run:
return None
containers = session.exec(select(DiscoveredContainer).where(DiscoveredContainer.discovery_run_id == run_id)).all()
return {**_serialize_run(run), 'containers': [serialize_discovered(item) for item in containers]}
def list_discovered(session: Session) -> list[dict[str, Any]]:
return [serialize_discovered(item) for item in _latest_discovered_items(session)]
def _latest_discovered_items(session: Session) -> list[DiscoveredContainer]:
run_order = _discovery_run_order(session)
latest_by_name: dict[str, DiscoveredContainer] = {}
for item in _latest_by_container(session).values():
if _is_appops_deploy_artifact_name(item.name):
continue
name_key = _container_name_key(item.name) or item.container_id
current = latest_by_name.get(name_key)
item_rank = (
item.import_status != 'missing',
run_order.get(item.discovery_run_id, datetime.min),
item.id,
)
current_rank = (
current.import_status != 'missing',
run_order.get(current.discovery_run_id, datetime.min),
current.id,
) if current else None
if current_rank is None or item_rank > current_rank:
latest_by_name[name_key] = item
return sorted(
latest_by_name.values(),
key=lambda item: item.name.lower(),
)
def serialize_connector_candidate(item: DiscoveredContainer, connector_id: str) -> dict[str, Any]:
discovered = serialize_discovered(item)
return {
'connectorId': connector_id,
'discoveredContainerId': item.id,
'containerId': item.container_id,
'containerName': item.name,
'displayName': discovered['displayName'],
'appType': item.suggested_app_type,
'lifecycleState': item.import_status,
'suggestedAppId': discovered['suggestedAppId'],
'connectorStatus': _connector_status(item),
'missingSecrets': connector_missing_secrets(connector_id),
'requiredSecrets': connector_required_secrets(connector_id),
'confidence': discovered['confidence'],
'reasons': discovered['reasons'],
'ports': item.ports_json,
}
def list_connector_candidates(session: Session, connector_id: str) -> list[dict[str, Any]]:
return [
serialize_connector_candidate(item, connector_id)
for item in _latest_discovered_items(session)
if _connector_type(item) == connector_id and item.import_status not in {'ignored', 'missing'}
]
def _find_latest_discovered_for_container(session: Session, container_id: str) -> DiscoveredContainer | None:
return _latest_by_container(session).get(container_id)
def serialize_connector_binding(
session: Session,
mapping: AppContainerMapping,
connector_id: str,
*,
app: App | None = None,
discovered: DiscoveredContainer | None = None,
) -> dict[str, Any]:
app = app if app is not None else session.get(App, mapping.app_id)
discovered = discovered if discovered is not None else _find_latest_discovered_for_container(session, mapping.container_id)
return {
'connectorId': connector_id,
'appId': mapping.app_id,
'appName': app.name if app else mapping.app_id,
'containerId': mapping.container_id,
'containerName': mapping.container_name,
'mappingId': mapping.id,
'connectorStatus': app.connector_status if app else 'disconnected',
'lifecycleState': discovered.import_status if discovered else 'managed',
'missingSecrets': connector_missing_secrets(connector_id),
'requiredSecrets': connector_required_secrets(connector_id),
'needsReview': bool(app.needs_review) if app else True,
}
def list_connector_bindings(session: Session, connector_id: str) -> list[dict[str, Any]]:
bindings: list[dict[str, Any]] = []
latest_by_container = _latest_by_container(session)
mappings = session.exec(select(AppContainerMapping).order_by(AppContainerMapping.container_name)).all()
apps = {app.id: app for app in session.exec(select(App)).all()}
for mapping in mappings:
discovered = latest_by_container.get(mapping.container_id)
if not discovered or _connector_type(discovered) != connector_id:
continue
bindings.append(serialize_connector_binding(session, mapping, connector_id, app=apps.get(mapping.app_id), discovered=discovered))
return bindings
def bind_connector_to_discovered(session: Session, connector_id: str, discovered_container_id: str) -> dict[str, Any]:
item = session.get(DiscoveredContainer, discovered_container_id)
if not item:
return {'success': False, 'message': 'discovered_container_not_found'}
if _connector_type(item) != connector_id:
return {'success': False, 'message': 'connector_candidate_mismatch'}
if item.import_status in {'ignored', 'missing'}:
return {'success': False, 'message': 'connector_candidate_not_bindable'}
result = import_discovered_container(session, discovered_container_id)
if not result.get('success'):
return result
app_id = str(result['app']['id'])
app = session.get(App, app_id)
mapping = session.exec(select(AppContainerMapping).where(AppContainerMapping.container_id == item.container_id)).first()
if not app or not mapping:
return {'success': False, 'message': 'connector_binding_failed'}
app.connector_status = _connector_status(item)
app.voice_enabled = False
app.needs_review = True
session.add(app)
session.commit()
session.refresh(app)
session.refresh(mapping)
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
return {
'success': True,
'message': 'connector_bound',
'app': _serialize_app(app),
'mapping': serialize_mapping(mapping),
'binding': serialize_connector_binding(session, mapping, connector_id),
}
def unbind_connector_from_app(session: Session, connector_id: str, app_id: str) -> dict[str, Any]:
app = session.get(App, app_id)
if not app:
return {'success': False, 'message': 'app_not_found'}
mappings = session.exec(select(AppContainerMapping).where(AppContainerMapping.app_id == app_id)).all()
has_connector_mapping = False
for mapping in mappings:
discovered = _find_latest_discovered_for_container(session, mapping.container_id)
if discovered and _connector_type(discovered) == connector_id:
has_connector_mapping = True
break
if not has_connector_mapping:
return {'success': False, 'message': 'connector_binding_not_found'}
app.connector_status = 'disconnected'
app.voice_enabled = False
app.needs_review = True
session.add(app)
session.commit()
session.refresh(app)
invalidate_discovery_cache()
safety_center_service.invalidate_cache()
return {'success': True, 'message': 'connector_unbound', 'app': _serialize_app(app)}
def list_inventory_containers(session: Session, include_ignored: bool = True) -> list[dict[str, Any]]:
items = _latest_discovered_items(session)
if not include_ignored:
items = [item for item in items if item.import_status != 'ignored']
return [serialize_discovered(item) for item in items]
def list_inventory_apps(session: Session, include_unmanaged: bool = True, include_ignored: bool = False) -> list[dict[str, Any]]:
from .app_registry import list_active_apps
apps = [
{
'id': app.id,
'display_name': app.name,
'name': app.name,
'app_type': app.id,
'category': str(app.category).lower(),
'criticality': app.criticality,
'lifecycle_state': app.discovery_status or 'managed',
'container_names': [],
'primary_container': None,
'connector_type': 'none',
'connector_status': app.connector_status,
'connector_required_secrets': [],
'is_protected': app.protected,
'policy_profile': 'managed',
'health_status': app.status,
'source': app.discovery_source or 'manual',
'actions_available': list(_GENERIC_ACTIONS),
'setup_checklist': [],
'discovery_confidence': 1,
'discovery_reasons': ['Managed app registry entry.'],
}
for app in list_active_apps(session)
]
if not include_unmanaged:
return apps
managed_container_ids = {mapping.container_id for mapping in session.exec(select(AppContainerMapping)).all()}
for item in _latest_discovered_items(session):
if item.container_id in managed_container_ids or item.import_status == 'managed':
continue
if item.import_status == 'ignored' and not include_ignored:
continue
apps.append(serialize_inventory_app(item))
return apps
def list_unmapped(session: Session, include_ignored: bool = False) -> list[dict[str, Any]]:
return [
serialize_discovered(item)
for item in _latest_discovered_items(session)
if item.import_status == 'unmapped' or (include_ignored and item.import_status == 'ignored')
]
def list_unknown(session: Session, include_ignored: bool = False) -> list[dict[str, Any]]:
return [
serialize_discovered(item)
for item in _latest_discovered_items(session)
if item.suggested_app_type == 'unknown' and (include_ignored or item.import_status != 'ignored')
]
def list_suggestions(session: Session) -> list[dict[str, Any]]:
return [item for item in list_discovered(session) if item['importStatus'] not in {'ignored', 'missing'}]
def list_mappings(session: Session) -> list[dict[str, Any]]:
return [serialize_mapping(mapping) for mapping in session.exec(select(AppContainerMapping)).all()]
def _stable_sequence(values: Any) -> list[str]:
if not isinstance(values, list):
return []
return sorted(str(value) for value in values if value is not None)
def _container_name_key(value: str | None) -> str:
return (value or '').strip().lower()
def _is_appops_deploy_artifact_name(value: str | None) -> bool:
from .app_registry import is_appops_deployment_container_name
return is_appops_deployment_container_name(value)
def _setting_json(session: Session, key: str, fallback: Any) -> Any:
row = session.get(AppSetting, key)
if not row or row.value_json is None:
return fallback
return row.value_json
def _set_setting_json(session: Session, key: str, value: Any) -> None:
row = session.get(AppSetting, key)
if not row:
row = AppSetting(key=key, value_json=value)
else:
row.value_json = value
row.updated_at = utc_now()
session.add(row)
def _drift_id(item: dict[str, Any]) -> str:
material = json.dumps(
{
'type': item.get('type'),
'containerId': item.get('containerId'),
'before': item.get('before'),
'after': item.get('after'),
},
sort_keys=True,
default=str,
)
return f"drift-{hashlib.sha256(material.encode('utf-8')).hexdigest()[:16]}"
def _drift_item(
*,
type: str,
severity: str,
container_id: str,
container_name: str,
before: Any,
after: Any,
suggested_action: str,
) -> dict[str, Any]:
item = {
'type': type,
'severity': severity,
'containerId': container_id,
'containerName': container_name,
'before': before,
'after': after,
'suggestedAction': suggested_action,
'status': 'open',
}
item['driftId'] = _drift_id(item)
return item
def _baseline_container(item: DiscoveredContainer) -> dict[str, Any]:
return {
'containerId': item.container_id,
'name': item.name,
'image': item.image,
'status': item.status,
'ports': _stable_sequence(item.ports_json),
'mounts': _json_clone(item.mounts_json, []),
'networks': _json_clone(item.networks_json, []),
'criticality': item.suggested_criticality,
}
def update_inventory_baseline(session: Session) -> dict[str, Any]:
current = [
item
for item in _latest_discovered_items(session)
if item.import_status not in {'ignored', 'missing'}
]
if not current:
return {
'success': False,
'status': 'failed',
'message': 'Geen actuele inventory beschikbaar. Draai eerst discovery.',
}
captured_at = utc_now().isoformat()
containers = {
item.container_id: _baseline_container(item)
for item in current
}
previous = _setting_json(session, _DRIFT_BASELINE_SETTING, {})
snapshot = {
'version': 1,
'capturedAt': captured_at,
'containerCount': len(containers),
'containers': containers,
}
_set_setting_json(session, _DRIFT_BASELINE_SETTING, snapshot)
_set_setting_json(session, _DRIFT_ACK_SETTING, {'ids': []})
session.commit()
audit = log_event(
session,
source='manual',
identity='operator',
action_id='inventory.baseline.update',
target='inventory',
target_type='inventory_baseline',
risk='low',
policy_decision='allowed',
reason='Operator stored the current AppOps inventory ledger as drift baseline.',
result='success',
before_state={
'capturedAt': previous.get('capturedAt') if isinstance(previous, dict) else None,
'containerCount': previous.get('containerCount', 0) if isinstance(previous, dict) else 0,
},
after_state={'capturedAt': captured_at, 'containerCount': len(containers)},
redacted_metadata={'containerCount': len(containers), 'version': 1},
)
return {
'success': True,
'status': 'success',
'message': f'Baseline bijgewerkt voor {len(containers)} containers.',
'auditId': audit.id,
'data': {'capturedAt': captured_at, 'containerCount': len(containers)},
}
def _calculate_baseline_drift(session: Session, baseline: dict[str, Any]) -> list[dict[str, Any]]:
raw_baseline = baseline.get('containers')
if not isinstance(raw_baseline, dict):
return []
latest = {
item.container_id: item
for item in _latest_discovered_items(session)
}
active_by_name = {
_container_name_key(item.name): item
for item in latest.values()
if item.import_status != 'missing'
}
matched_current_ids: set[str] = set()
drift: list[dict[str, Any]] = []
for baseline_id, raw in raw_baseline.items():
if not isinstance(raw, dict):
continue
current = latest.get(str(baseline_id))
if current is None or current.import_status == 'missing':
recreated = active_by_name.get(_container_name_key(str(raw.get('name') or '')))
if recreated is not None:
current = recreated
if current is None or current.import_status == 'missing':
criticality = str(raw.get('criticality') or 'medium')
drift.append(_drift_item(
type='missing_container',
severity='high' if criticality in {'high', 'critical'} else 'medium',
container_id=str(baseline_id),
container_name=str(raw.get('name') or baseline_id),
before=raw.get('status') or 'baseline',
after='missing',
suggested_action='investigate',
))
continue
matched_current_ids.add(current.container_id)
comparisons = [
('image_changed', 'medium', raw.get('image'), current.image, 'review_baseline'),
('port_changed', 'medium', _stable_sequence(raw.get('ports')), _stable_sequence(current.ports_json), 'review_exposure'),
('mount_changed', 'high', raw.get('mounts') or [], current.mounts_json, 'review_storage'),
('network_changed', 'medium', raw.get('networks') or [], current.networks_json, 'review_network'),
(
'status_changed',
'high' if current.suggested_criticality in {'high', 'critical'} and current.status != 'running' else 'medium',
raw.get('status'),
current.status,
'review_health',
),
]
for drift_type, severity, before, after, suggested_action in comparisons:
if before != after:
drift.append(_drift_item(
type=drift_type,
severity=severity,
container_id=current.container_id,
container_name=current.name,
before=before,
after=after,
suggested_action=suggested_action,
))
for current in latest.values():
if current.container_id in matched_current_ids or current.import_status in {'ignored', 'missing'}:
continue
drift.append(_drift_item(
type='new_container',
severity='low',
container_id=current.container_id,
container_name=current.name,
before=None,
after=current.image,
suggested_action='review_import',
))
return drift
def calculate_drift(session: Session) -> list[dict[str, Any]]:
acknowledged = set(_setting_json(session, _DRIFT_ACK_SETTING, {}).get('ids', []))
baseline = _setting_json(session, _DRIFT_BASELINE_SETTING, {})
if isinstance(baseline, dict) and isinstance(baseline.get('containers'), dict):
return [item for item in _calculate_baseline_drift(session, baseline) if item['driftId'] not in acknowledged]
grouped = _recent_by_container(session)
latest_by_container: dict[str, DiscoveredContainer] = {}
for container_id, items in grouped.items():
latest_by_container[container_id] = items[-1]
active_latest_names = {
_container_name_key(item.name)
for item in latest_by_container.values()
if item.import_status != 'missing' and not _is_appops_deploy_artifact_name(item.name)
}
missing_latest_names = {
_container_name_key(item.name)
for item in latest_by_container.values()
if item.import_status == 'missing' and not _is_appops_deploy_artifact_name(item.name)
}
recreated_names = active_latest_names & missing_latest_names
drift: list[dict[str, Any]] = []
for container_id, items in grouped.items():
if _is_appops_deploy_artifact_name(items[-1].name):
continue
if len(items) == 1:
item = items[-1]
if item.import_status in {'discovered', 'unmapped'} and _container_name_key(item.name) not in recreated_names:
drift.append(_drift_item(type='new_container', severity='low', container_id=container_id, container_name=item.name, before=None, after=item.image, suggested_action='review_import'))
continue
previous, current = items[-2], items[-1]
if current.import_status == 'missing':
if _container_name_key(current.name) in recreated_names:
continue
drift.append(_drift_item(type='missing_container', severity='high' if previous.suggested_criticality in {'high', 'critical'} else 'medium', container_id=container_id, container_name=current.name, before=previous.status, after='missing', suggested_action='investigate'))
continue
if previous.image != current.image:
drift.append(_drift_item(type='image_changed', severity='medium', container_id=container_id, container_name=current.name, before=previous.image, after=current.image, suggested_action='review_baseline'))
if _stable_sequence(previous.ports_json) != _stable_sequence(current.ports_json):
drift.append(_drift_item(type='port_changed', severity='medium', container_id=container_id, container_name=current.name, before=previous.ports_json, after=current.ports_json, suggested_action='review_exposure'))
if previous.mounts_json != current.mounts_json:
drift.append(_drift_item(type='mount_changed', severity='high', container_id=container_id, container_name=current.name, before=previous.mounts_json, after=current.mounts_json, suggested_action='review_storage'))
if previous.networks_json != current.networks_json:
drift.append(_drift_item(type='network_changed', severity='medium', container_id=container_id, container_name=current.name, before=previous.networks_json, after=current.networks_json, suggested_action='review_network'))
if previous.status != current.status:
severity = 'high' if current.suggested_criticality in {'high', 'critical'} and current.status != 'running' else 'medium'
drift.append(_drift_item(type='status_changed', severity=severity, container_id=container_id, container_name=current.name, before=previous.status, after=current.status, suggested_action='review_health'))
known_latest = {items[-1].container_id for items in grouped.values()}
for mapping in session.exec(select(AppContainerMapping)).all():
if mapping.container_id not in known_latest:
drift.append(_drift_item(type='missing_container', severity='medium', container_id=mapping.container_id, container_name=mapping.container_name, before='mapped', after='missing', suggested_action='investigate'))
return [item for item in drift if item['driftId'] not in acknowledged]
def acknowledge_drift(session: Session, drift_id: str) -> dict[str, Any]:
current = _setting_json(session, _DRIFT_ACK_SETTING, {})
ids = set(current.get('ids', [])) if isinstance(current, dict) else set()
ids.add(drift_id)
_set_setting_json(session, _DRIFT_ACK_SETTING, {'ids': sorted(ids)})
session.commit()
log_event(
session,
source='manual',
identity='operator',
action_id='inventory.drift.acknowledge',
target=drift_id,
target_type='inventory_drift',
risk='low',
policy_decision='allowed',
reason='Operator acknowledged inventory drift in AppOps metadata only.',
result='success',
redacted_metadata={'driftId': drift_id},
)
return {'success': True, 'message': 'drift_acknowledged', 'status': 'success', 'driftId': drift_id}
def _drift_incidents(session: Session) -> list[dict[str, Any]]:
data = _setting_json(session, _DRIFT_INCIDENTS_SETTING, {'items': []})
items = data.get('items', []) if isinstance(data, dict) else []
return [item for item in items if isinstance(item, dict)]
def list_drift_incidents(session: Session) -> list[dict[str, Any]]:
return [item for item in _drift_incidents(session) if item.get('status') != 'resolved']
def create_drift_incident(session: Session, drift_id: str) -> dict[str, Any]:
drift = next((item for item in calculate_drift(session) if item['driftId'] == drift_id), None)
if not drift:
return {'success': False, 'message': 'drift_not_found_or_acknowledged', 'status': 'failed'}
incidents = _drift_incidents(session)
existing = next((item for item in incidents if item.get('driftId') == drift_id and item.get('status') != 'resolved'), None)
if existing:
return {'success': True, 'message': 'drift_incident_exists', 'status': 'success', 'incident': existing}
incident = {
'id': f"incident-{drift_id}",
'driftId': drift_id,
'severity': drift['severity'],
'status': 'open',
'title': f"Inventory drift: {str(drift['type']).replace('_', ' ')}",
'detectedTime': utc_now().isoformat(),
'evidence': {
'source': 'Inventory Drift',
'check': drift['type'],
'observed': f"{drift['containerName']}: {drift['after']}",
'expected': str(drift['before']),
'timestamp': utc_now().isoformat(),
},
'suggestedActions': [drift['suggestedAction'], 'inventory.review'],
}
incidents.append(incident)
_set_setting_json(session, _DRIFT_INCIDENTS_SETTING, {'items': incidents})
session.commit()
log_event(
session,
source='manual',
identity='operator',
action_id='inventory.drift.create_incident',
target=drift_id,
target_type='inventory_drift',
risk='low',
policy_decision='allowed',
reason='Operator created an AppOps incident from inventory drift metadata.',
result='success',
redacted_metadata={'driftId': drift_id, 'incidentId': incident['id']},
)
return {'success': True, 'message': 'drift_incident_created', 'status': 'success', 'incident': incident}
def update_drift_incident_status(session: Session, incident_id: str, status: str) -> dict[str, Any] | None:
if status not in {'open', 'acknowledged', 'muted', 'resolved'}:
status = 'acknowledged'
incidents = _drift_incidents(session)
updated = False
for incident in incidents:
if incident.get('id') == incident_id:
incident['status'] = status
updated = True
break
if not updated:
return None
_set_setting_json(session, _DRIFT_INCIDENTS_SETTING, {'items': incidents})
session.commit()
return {'success': True, 'message': f'Incident {incident_id} status updated', 'result': 'success', 'status': 'success'}