Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
Public Source Publisher committed
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1 @@
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,614 @@
|
||||
from __future__ import annotations
|
||||
|
||||
ACTIONS = [
|
||||
'container.status',
|
||||
'container.logs',
|
||||
'container.clear_logs',
|
||||
'container.restart',
|
||||
'container.stop',
|
||||
'runner.fleet.upgrade_v2',
|
||||
'controller.artifact.promote',
|
||||
'gateway.clear_cache',
|
||||
'stack.status',
|
||||
'stack.redeploy',
|
||||
'deployment.preflight',
|
||||
'deployment.targets.sync',
|
||||
'deployment.runnerless.provision',
|
||||
'deployment.target.deploy_workmesh',
|
||||
'deployment.target.promote_workmesh',
|
||||
'deployment.target.register_portfolio',
|
||||
'deployment.target.register_privatemuse',
|
||||
'deployment.target.reconcile_media_codec_dashboard',
|
||||
'deployment.webhook.repair_privatemuse',
|
||||
'privatemuse.comfyui.install_artifact',
|
||||
'deployment.apply',
|
||||
'deployment.rollback',
|
||||
'volume.delete',
|
||||
'unraid.health_summary',
|
||||
'unraid.vms',
|
||||
'unraid.vm.start',
|
||||
'unraid.vm.resume',
|
||||
'unraid.vm.pause',
|
||||
'unraid.vm.stop',
|
||||
'unraid.vm.reboot',
|
||||
'unraid.vm.force_stop',
|
||||
'unraid.vm.reset',
|
||||
'unraid.mover.start',
|
||||
'setup.status',
|
||||
'safety.summary',
|
||||
'media_pipeline.check',
|
||||
'media_pipeline.diagnose',
|
||||
'notifications.summary',
|
||||
'notifications.evaluate_safety',
|
||||
'sonarr.search_series',
|
||||
'sonarr.add_series',
|
||||
'sonarr.health',
|
||||
'sonarr.queue_status',
|
||||
'sonarr.wanted_missing',
|
||||
'sonarr.delete_series',
|
||||
'sonarr.mass_edit',
|
||||
'radarr.search_movie',
|
||||
'radarr.add_movie',
|
||||
'radarr.health',
|
||||
'radarr.queue_status',
|
||||
'radarr.wanted_missing',
|
||||
'radarr.delete_movie',
|
||||
'radarr.delete_files',
|
||||
'radarr.mass_edit',
|
||||
'jellyfin.scan_library',
|
||||
'widefrog.health',
|
||||
'widefrog.queue_status',
|
||||
'widefrog.failed_jobs',
|
||||
'widefrog.activity',
|
||||
'widefrog.retry_failed',
|
||||
'widefrog.scan_library',
|
||||
'widefrog.download_status',
|
||||
'prowlarr.health',
|
||||
'prowlarr.indexer_status',
|
||||
'prowlarr.application_status',
|
||||
'prowlarr.indexer_health',
|
||||
'prowlarr.search_test',
|
||||
'prowlarr.sync_status',
|
||||
'prowlarr.delete_indexer',
|
||||
'prowlarr.delete_application',
|
||||
'prowlarr.update_indexer',
|
||||
'prowlarr.update_application',
|
||||
'prowlarr.grab_release',
|
||||
'prowlarr.push_release',
|
||||
'prowlarr.mass_edit',
|
||||
'prowlarr.test_all_indexers',
|
||||
'deluge.health',
|
||||
'deluge.session_status',
|
||||
'deluge.torrent_status',
|
||||
'deluge.active_downloads',
|
||||
'deluge.stalled_downloads',
|
||||
'deluge.paused_downloads',
|
||||
'deluge.speed_status',
|
||||
'deluge.pause_torrent',
|
||||
'deluge.resume_torrent',
|
||||
'deluge.pause_all',
|
||||
'deluge.resume_all',
|
||||
'deluge.remove_torrent',
|
||||
'deluge.remove_torrent_data',
|
||||
'deluge.delete_data',
|
||||
'deluge.force_recheck',
|
||||
'deluge.move_storage',
|
||||
'deluge.add_torrent',
|
||||
'deluge.add_magnet',
|
||||
'deluge.mass_edit',
|
||||
'deluge.set_options',
|
||||
'deluge.unknown_mutation',
|
||||
'plex.health',
|
||||
'plex.server_status',
|
||||
'plex.library_status',
|
||||
'plex.active_sessions',
|
||||
'plex.recent_media',
|
||||
'plex.scan_library',
|
||||
'plex.scan_all_libraries',
|
||||
'plex.stream_summary',
|
||||
'tautulli.health',
|
||||
'tautulli.status',
|
||||
'tautulli.activity',
|
||||
'tautulli.history',
|
||||
'tautulli.users',
|
||||
'tautulli.libraries',
|
||||
'tautulli.recently_added',
|
||||
'tautulli.watch_stats',
|
||||
'tautulli.stream_summary',
|
||||
'tautulli.plex_info',
|
||||
'npm.health',
|
||||
'npm.proxy_hosts',
|
||||
'npm.certificates',
|
||||
'npm.expiring_certificates',
|
||||
'npm.access_lists',
|
||||
'npm.exposure_summary',
|
||||
'npm.host_reachability',
|
||||
'npm.security_warnings',
|
||||
'npm.create_proxy_host',
|
||||
'npm.update_proxy_host',
|
||||
'npm.delete_proxy_host',
|
||||
'npm.enable_proxy_host',
|
||||
'npm.disable_proxy_host',
|
||||
'npm.renew_certificate',
|
||||
'npm.delete_certificate',
|
||||
'npm.modify_access_list',
|
||||
'npm.create_user',
|
||||
'npm.update_user',
|
||||
'npm.delete_user',
|
||||
'npm.mass_edit',
|
||||
'npm.unknown_mutation',
|
||||
'authentik.health',
|
||||
'authentik.system_status',
|
||||
'authentik.applications',
|
||||
'authentik.providers',
|
||||
'authentik.proxy_providers',
|
||||
'authentik.oauth2_providers',
|
||||
'authentik.outposts',
|
||||
'authentik.outpost_health',
|
||||
'authentik.flows_summary',
|
||||
'authentik.policies_summary',
|
||||
'authentik.events',
|
||||
'authentik.auth_coverage',
|
||||
'authentik.security_warnings',
|
||||
'authentik.create_application',
|
||||
'authentik.update_application',
|
||||
'authentik.delete_application',
|
||||
'authentik.create_provider',
|
||||
'authentik.update_provider',
|
||||
'authentik.delete_provider',
|
||||
'authentik.create_outpost',
|
||||
'authentik.update_outpost',
|
||||
'authentik.delete_outpost',
|
||||
'authentik.create_policy',
|
||||
'authentik.update_policy',
|
||||
'authentik.delete_policy',
|
||||
'authentik.create_user',
|
||||
'authentik.update_user',
|
||||
'authentik.delete_user',
|
||||
'authentik.create_group',
|
||||
'authentik.update_group',
|
||||
'authentik.delete_group',
|
||||
'authentik.rotate_token',
|
||||
'authentik.force_sync',
|
||||
'authentik.mass_edit',
|
||||
'authentik.unknown_mutation',
|
||||
'tautulli.terminate_session',
|
||||
'tautulli.delete_newsletter',
|
||||
'tautulli.delete_notification_log',
|
||||
'tautulli.delete_notifier',
|
||||
'tautulli.delete_recently_added',
|
||||
'tautulli.download_config',
|
||||
'tautulli.download_database',
|
||||
'tautulli.download_export',
|
||||
'tautulli.download_log',
|
||||
'tautulli.download_plex_log',
|
||||
'tautulli.notify',
|
||||
'tautulli.user_management',
|
||||
'tautulli.mass_edit',
|
||||
'tautulli.unknown_mutation',
|
||||
'plex.delete_media',
|
||||
'plex.delete_library',
|
||||
'plex.empty_trash',
|
||||
'plex.clean_bundles',
|
||||
'plex.optimize_database',
|
||||
'plex.kill_session',
|
||||
'plex.user_management',
|
||||
'plex.force_refresh_all_metadata',
|
||||
'plex.mass_edit',
|
||||
'plex.unknown_mutation',
|
||||
'widefrog.delete_job',
|
||||
'widefrog.delete_media',
|
||||
'widefrog.clear_library',
|
||||
'widefrog.clear_queue',
|
||||
'widefrog.reset_database',
|
||||
'widefrog.delete_downloads',
|
||||
'widefrog.prune',
|
||||
'playbook.run',
|
||||
]
|
||||
|
||||
ACTION_RISK = {
|
||||
'container.status': 'low',
|
||||
'container.logs': 'low',
|
||||
'container.clear_logs': 'high',
|
||||
'container.restart': 'medium',
|
||||
'container.stop': 'high',
|
||||
'runner.fleet.upgrade_v2': 'high',
|
||||
'controller.artifact.promote': 'high',
|
||||
'gateway.clear_cache': 'high',
|
||||
'stack.status': 'low',
|
||||
'stack.redeploy': 'high',
|
||||
'deployment.preflight': 'low',
|
||||
'deployment.targets.sync': 'high',
|
||||
'deployment.runnerless.provision': 'high',
|
||||
'deployment.target.deploy_workmesh': 'high',
|
||||
'deployment.target.promote_workmesh': 'high',
|
||||
'deployment.target.register_portfolio': 'high',
|
||||
'deployment.target.register_privatemuse': 'high',
|
||||
'deployment.target.reconcile_media_codec_dashboard': 'high',
|
||||
'deployment.webhook.repair_privatemuse': 'high',
|
||||
'privatemuse.comfyui.install_artifact': 'high',
|
||||
'deployment.apply': 'high',
|
||||
'deployment.rollback': 'high',
|
||||
'volume.delete': 'critical',
|
||||
'unraid.health_summary': 'low',
|
||||
'unraid.vms': 'low',
|
||||
'unraid.vm.start': 'medium',
|
||||
'unraid.vm.resume': 'medium',
|
||||
'unraid.vm.pause': 'high',
|
||||
'unraid.vm.stop': 'critical',
|
||||
'unraid.vm.reboot': 'critical',
|
||||
'unraid.vm.force_stop': 'critical',
|
||||
'unraid.vm.reset': 'critical',
|
||||
'unraid.mover.start': 'high',
|
||||
'setup.status': 'low',
|
||||
'safety.summary': 'low',
|
||||
'media_pipeline.check': 'low',
|
||||
'media_pipeline.diagnose': 'low',
|
||||
'notifications.summary': 'low',
|
||||
'notifications.evaluate_safety': 'medium',
|
||||
'sonarr.search_series': 'low',
|
||||
'sonarr.add_series': 'low',
|
||||
'sonarr.health': 'low',
|
||||
'sonarr.queue_status': 'low',
|
||||
'sonarr.wanted_missing': 'low',
|
||||
'sonarr.delete_series': 'critical',
|
||||
'sonarr.mass_edit': 'critical',
|
||||
'radarr.search_movie': 'low',
|
||||
'radarr.add_movie': 'low',
|
||||
'radarr.health': 'low',
|
||||
'radarr.queue_status': 'low',
|
||||
'radarr.wanted_missing': 'low',
|
||||
'radarr.delete_movie': 'critical',
|
||||
'radarr.delete_files': 'critical',
|
||||
'radarr.mass_edit': 'critical',
|
||||
'jellyfin.scan_library': 'medium',
|
||||
'widefrog.health': 'low',
|
||||
'widefrog.queue_status': 'low',
|
||||
'widefrog.failed_jobs': 'low',
|
||||
'widefrog.activity': 'low',
|
||||
'widefrog.retry_failed': 'medium',
|
||||
'widefrog.scan_library': 'medium',
|
||||
'widefrog.download_status': 'low',
|
||||
'prowlarr.health': 'low',
|
||||
'prowlarr.indexer_status': 'low',
|
||||
'prowlarr.application_status': 'low',
|
||||
'prowlarr.indexer_health': 'low',
|
||||
'prowlarr.search_test': 'medium',
|
||||
'prowlarr.sync_status': 'low',
|
||||
'prowlarr.delete_indexer': 'critical',
|
||||
'prowlarr.delete_application': 'critical',
|
||||
'prowlarr.update_indexer': 'high',
|
||||
'prowlarr.update_application': 'high',
|
||||
'prowlarr.grab_release': 'critical',
|
||||
'prowlarr.push_release': 'critical',
|
||||
'prowlarr.mass_edit': 'critical',
|
||||
'prowlarr.test_all_indexers': 'medium',
|
||||
'deluge.health': 'low',
|
||||
'deluge.session_status': 'low',
|
||||
'deluge.torrent_status': 'low',
|
||||
'deluge.active_downloads': 'low',
|
||||
'deluge.stalled_downloads': 'low',
|
||||
'deluge.paused_downloads': 'low',
|
||||
'deluge.speed_status': 'low',
|
||||
'deluge.pause_torrent': 'medium',
|
||||
'deluge.resume_torrent': 'medium',
|
||||
'deluge.pause_all': 'high',
|
||||
'deluge.resume_all': 'high',
|
||||
'deluge.remove_torrent': 'critical',
|
||||
'deluge.remove_torrent_data': 'critical',
|
||||
'deluge.delete_data': 'critical',
|
||||
'deluge.force_recheck': 'high',
|
||||
'deluge.move_storage': 'high',
|
||||
'deluge.add_torrent': 'high',
|
||||
'deluge.add_magnet': 'high',
|
||||
'deluge.mass_edit': 'critical',
|
||||
'deluge.set_options': 'critical',
|
||||
'deluge.unknown_mutation': 'critical',
|
||||
'plex.health': 'low',
|
||||
'plex.server_status': 'low',
|
||||
'plex.library_status': 'low',
|
||||
'plex.active_sessions': 'low',
|
||||
'plex.recent_media': 'low',
|
||||
'plex.scan_library': 'medium',
|
||||
'plex.scan_all_libraries': 'high',
|
||||
'plex.stream_summary': 'low',
|
||||
'tautulli.health': 'low',
|
||||
'tautulli.status': 'low',
|
||||
'tautulli.activity': 'low',
|
||||
'tautulli.history': 'medium',
|
||||
'tautulli.users': 'medium',
|
||||
'tautulli.libraries': 'low',
|
||||
'tautulli.recently_added': 'low',
|
||||
'tautulli.watch_stats': 'medium',
|
||||
'tautulli.stream_summary': 'low',
|
||||
'tautulli.plex_info': 'low',
|
||||
'npm.health': 'low',
|
||||
'npm.proxy_hosts': 'medium',
|
||||
'npm.certificates': 'low',
|
||||
'npm.expiring_certificates': 'low',
|
||||
'npm.access_lists': 'medium',
|
||||
'npm.exposure_summary': 'medium',
|
||||
'npm.host_reachability': 'low',
|
||||
'npm.security_warnings': 'low',
|
||||
'npm.create_proxy_host': 'critical',
|
||||
'npm.update_proxy_host': 'critical',
|
||||
'npm.delete_proxy_host': 'critical',
|
||||
'npm.enable_proxy_host': 'high',
|
||||
'npm.disable_proxy_host': 'high',
|
||||
'npm.renew_certificate': 'high',
|
||||
'npm.delete_certificate': 'critical',
|
||||
'npm.modify_access_list': 'critical',
|
||||
'npm.create_user': 'critical',
|
||||
'npm.update_user': 'critical',
|
||||
'npm.delete_user': 'critical',
|
||||
'npm.mass_edit': 'critical',
|
||||
'npm.unknown_mutation': 'critical',
|
||||
'authentik.health': 'low',
|
||||
'authentik.system_status': 'low',
|
||||
'authentik.applications': 'medium',
|
||||
'authentik.providers': 'medium',
|
||||
'authentik.proxy_providers': 'medium',
|
||||
'authentik.oauth2_providers': 'medium',
|
||||
'authentik.outposts': 'medium',
|
||||
'authentik.outpost_health': 'low',
|
||||
'authentik.flows_summary': 'medium',
|
||||
'authentik.policies_summary': 'medium',
|
||||
'authentik.events': 'medium',
|
||||
'authentik.auth_coverage': 'medium',
|
||||
'authentik.security_warnings': 'low',
|
||||
'authentik.create_application': 'critical',
|
||||
'authentik.update_application': 'critical',
|
||||
'authentik.delete_application': 'critical',
|
||||
'authentik.create_provider': 'critical',
|
||||
'authentik.update_provider': 'critical',
|
||||
'authentik.delete_provider': 'critical',
|
||||
'authentik.create_outpost': 'critical',
|
||||
'authentik.update_outpost': 'critical',
|
||||
'authentik.delete_outpost': 'critical',
|
||||
'authentik.create_policy': 'critical',
|
||||
'authentik.update_policy': 'critical',
|
||||
'authentik.delete_policy': 'critical',
|
||||
'authentik.create_user': 'critical',
|
||||
'authentik.update_user': 'critical',
|
||||
'authentik.delete_user': 'critical',
|
||||
'authentik.create_group': 'critical',
|
||||
'authentik.update_group': 'critical',
|
||||
'authentik.delete_group': 'critical',
|
||||
'authentik.rotate_token': 'critical',
|
||||
'authentik.force_sync': 'high',
|
||||
'authentik.mass_edit': 'critical',
|
||||
'authentik.unknown_mutation': 'critical',
|
||||
'tautulli.terminate_session': 'high',
|
||||
'tautulli.delete_newsletter': 'critical',
|
||||
'tautulli.delete_notification_log': 'critical',
|
||||
'tautulli.delete_notifier': 'critical',
|
||||
'tautulli.delete_recently_added': 'critical',
|
||||
'tautulli.download_config': 'critical',
|
||||
'tautulli.download_database': 'critical',
|
||||
'tautulli.download_export': 'critical',
|
||||
'tautulli.download_log': 'critical',
|
||||
'tautulli.download_plex_log': 'critical',
|
||||
'tautulli.notify': 'high',
|
||||
'tautulli.user_management': 'critical',
|
||||
'tautulli.mass_edit': 'critical',
|
||||
'tautulli.unknown_mutation': 'critical',
|
||||
'plex.delete_media': 'critical',
|
||||
'plex.delete_library': 'critical',
|
||||
'plex.empty_trash': 'critical',
|
||||
'plex.clean_bundles': 'critical',
|
||||
'plex.optimize_database': 'critical',
|
||||
'plex.kill_session': 'high',
|
||||
'plex.user_management': 'critical',
|
||||
'plex.force_refresh_all_metadata': 'critical',
|
||||
'plex.mass_edit': 'critical',
|
||||
'plex.unknown_mutation': 'critical',
|
||||
'widefrog.delete_job': 'critical',
|
||||
'widefrog.delete_media': 'critical',
|
||||
'widefrog.clear_library': 'critical',
|
||||
'widefrog.clear_queue': 'critical',
|
||||
'widefrog.reset_database': 'critical',
|
||||
'widefrog.delete_downloads': 'critical',
|
||||
'widefrog.prune': 'critical',
|
||||
'playbook.run': 'medium',
|
||||
}
|
||||
|
||||
READ_ONLY_VERBS = {
|
||||
'activity',
|
||||
'active_downloads',
|
||||
'active_sessions',
|
||||
'application_status',
|
||||
'applications',
|
||||
'auth_coverage',
|
||||
'certificates',
|
||||
'check',
|
||||
'download_status',
|
||||
'diagnose',
|
||||
'events',
|
||||
'expiring_certificates',
|
||||
'exposure_summary',
|
||||
'failed_jobs',
|
||||
'flows_summary',
|
||||
'health',
|
||||
'health_summary',
|
||||
'history',
|
||||
'host_reachability',
|
||||
'indexer_health',
|
||||
'indexer_status',
|
||||
'libraries',
|
||||
'library_status',
|
||||
'logs',
|
||||
'oauth2_providers',
|
||||
'outpost_health',
|
||||
'outposts',
|
||||
'paused_downloads',
|
||||
'plex_info',
|
||||
'preflight',
|
||||
'policies_summary',
|
||||
'providers',
|
||||
'proxy_hosts',
|
||||
'proxy_providers',
|
||||
'queue_status',
|
||||
'recent_media',
|
||||
'recently_added',
|
||||
'search_movie',
|
||||
'search_series',
|
||||
'security_warnings',
|
||||
'server_status',
|
||||
'session_status',
|
||||
'speed_status',
|
||||
'stalled_downloads',
|
||||
'status',
|
||||
'stream_summary',
|
||||
'summary',
|
||||
'sync_status',
|
||||
'system_status',
|
||||
'torrent_status',
|
||||
'users',
|
||||
'wanted_missing',
|
||||
'watch_stats',
|
||||
'vms',
|
||||
}
|
||||
|
||||
DESTRUCTIVE_VERBS = {
|
||||
'clear_cache',
|
||||
'clear_library',
|
||||
'clear_logs',
|
||||
'clear_queue',
|
||||
'clean_bundles',
|
||||
'delete_application',
|
||||
'delete_certificate',
|
||||
'delete_data',
|
||||
'delete_downloads',
|
||||
'delete_files',
|
||||
'delete_group',
|
||||
'delete_indexer',
|
||||
'delete_job',
|
||||
'delete_library',
|
||||
'delete_media',
|
||||
'delete_movie',
|
||||
'delete_newsletter',
|
||||
'delete_notification_log',
|
||||
'delete_notifier',
|
||||
'delete_outpost',
|
||||
'delete_policy',
|
||||
'delete_provider',
|
||||
'delete_proxy_host',
|
||||
'delete_recently_added',
|
||||
'delete_series',
|
||||
'delete_user',
|
||||
'download_config',
|
||||
'download_database',
|
||||
'download_export',
|
||||
'download_log',
|
||||
'download_plex_log',
|
||||
'empty_trash',
|
||||
'force_refresh_all_metadata',
|
||||
'grab_release',
|
||||
'mass_edit',
|
||||
'modify_access_list',
|
||||
'prune',
|
||||
'push_release',
|
||||
'remove_torrent',
|
||||
'remove_torrent_data',
|
||||
'reset_database',
|
||||
'rotate_token',
|
||||
'set_options',
|
||||
'stop',
|
||||
'unknown_mutation',
|
||||
}
|
||||
|
||||
|
||||
def action_category(action: str) -> str:
|
||||
return action.split('.', 1)[0] if '.' in action else 'general'
|
||||
|
||||
|
||||
def action_default_target(action: str) -> str:
|
||||
if action == 'controller.artifact.promote':
|
||||
return 'tower/deployment-controller'
|
||||
if action == 'runner.fleet.upgrade_v2':
|
||||
return 'tower/runner-fleet-v2'
|
||||
if action == 'deployment.targets.sync':
|
||||
return 'jens/*'
|
||||
if action == 'deployment.target.deploy_workmesh':
|
||||
return 'jens/workmesh'
|
||||
if action == 'deployment.target.promote_workmesh':
|
||||
return 'jens/workmesh'
|
||||
if action == 'deployment.target.register_portfolio':
|
||||
return 'jens/portfolio'
|
||||
if action in {
|
||||
'deployment.target.register_privatemuse',
|
||||
'deployment.webhook.repair_privatemuse',
|
||||
}:
|
||||
return 'jens/privatemuse'
|
||||
if action == 'deployment.target.reconcile_media_codec_dashboard':
|
||||
return 'jens/media-codec-dashboard'
|
||||
category = action_category(action)
|
||||
if category == 'container':
|
||||
return 'container'
|
||||
if category == 'volume':
|
||||
return 'volume'
|
||||
if category == 'media_pipeline':
|
||||
return 'media-pipeline'
|
||||
return category
|
||||
|
||||
|
||||
def action_label(action: str) -> str:
|
||||
return action.replace('_', ' ').replace('.', ' / ').title()
|
||||
|
||||
|
||||
def action_capability(action: str) -> str:
|
||||
verb = action.split('.', 1)[1] if '.' in action else action
|
||||
if action in {'unraid.vm.stop', 'unraid.vm.reboot', 'unraid.vm.force_stop', 'unraid.vm.reset'}:
|
||||
return 'destructive'
|
||||
if verb in DESTRUCTIVE_VERBS or ACTION_RISK.get(action) == 'critical':
|
||||
return 'destructive'
|
||||
if verb in READ_ONLY_VERBS:
|
||||
return 'read_only'
|
||||
return 'operator'
|
||||
|
||||
|
||||
def action_blast_radius(action: str, capability: str) -> list[str]:
|
||||
category = action_category(action)
|
||||
if capability == 'read_only':
|
||||
return [f'Read-only visibility for {category}; no state changes are performed.']
|
||||
if category == 'volume':
|
||||
return ['Storage volume data and dependent apps could be affected if execution were ever enabled.']
|
||||
if category == 'container':
|
||||
return ['Target container availability and dependent routes may be affected.']
|
||||
if category == 'stack':
|
||||
return ['Multiple containers in the stack may restart or change state.']
|
||||
if category == 'gateway':
|
||||
return ['AppOps runtime visibility, cached connector state or operator context may be affected if execution were ever enabled.']
|
||||
if category in {'authentik', 'npm'}:
|
||||
return ['Authentication, exposure, routing or access-control behavior may be affected.']
|
||||
if category in {'sonarr', 'radarr', 'prowlarr', 'deluge', 'plex', 'tautulli', 'widefrog'}:
|
||||
return ['Media pipeline state, queues or user-facing media availability may be affected.']
|
||||
if category == 'unraid':
|
||||
return ['Unraid host or array operations may be affected.']
|
||||
return [f'{category} operational state may be affected.']
|
||||
|
||||
|
||||
def action_safeguards(action: str, decision: str, capability: str) -> list[str]:
|
||||
safeguards = ['Registry detail is read-only and cannot execute this action.']
|
||||
if capability == 'destructive':
|
||||
safeguards.append('Destructive or critical actions stay blocked unless an explicit future policy path is designed.')
|
||||
if decision == 'blocked':
|
||||
safeguards.append('AppOps Gateway V1 exposes this only as a blocked boundary; no executor is registered.')
|
||||
safeguards.append('Current policy decision is blocked.')
|
||||
elif decision == 'approval_required':
|
||||
safeguards.append('Current policy decision requires operator approval before execution.')
|
||||
else:
|
||||
safeguards.append('Allowed policy preview does not bypass preflight, audit or runtime checks.')
|
||||
if 'token' in action or 'secret' in action:
|
||||
safeguards.append('Secret-like values are never returned by the registry.')
|
||||
return safeguards
|
||||
|
||||
|
||||
def action_operator_guidance(action: str, decision: str, capability: str) -> str:
|
||||
if decision == 'blocked':
|
||||
return 'AppOps Gateway V1 blocks this action; treat it as governance visibility only and use external/manual tooling until a separate safe execution design exists.'
|
||||
if decision == 'approval_required':
|
||||
return 'Use approved operational flows only after reviewing impact and maintenance context.'
|
||||
if capability == 'read_only':
|
||||
return 'Safe to inspect for operational awareness.'
|
||||
return 'Use the existing preflight and audited action flow before any live operation.'
|
||||
@@ -0,0 +1,551 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting, Connector
|
||||
from ..schemas.ai_remediation import (
|
||||
RemediationApprovalProposal,
|
||||
RemediationApprovalRequirement,
|
||||
RemediationDryRunResult,
|
||||
RemediationDryRunStepResult,
|
||||
RemediationEvidence,
|
||||
RemediationPlan,
|
||||
RemediationPlanCollection,
|
||||
RemediationStep,
|
||||
RemediationVerificationCheck,
|
||||
)
|
||||
from . import ai_security_reviewer
|
||||
from .audit_logger import log_event
|
||||
from .policy_engine import evaluate_policy
|
||||
from .secret_store import list_secret_status, secret_storage_status
|
||||
|
||||
|
||||
SOURCE_KEY = 'ai_remediation'
|
||||
MUTATION_TOKENS = ('delete', 'remove', 'prune', 'format', 'renew', 'create', 'update', 'rotate', 'redeploy', 'restart', 'resume', 'pause', 'retry')
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _redact(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
lower = value.lower()
|
||||
if any(token in lower for token in ('token_value', 'secret_value', 'api_key=', 'password=', 'bearer ')):
|
||||
return '[REDACTED]'
|
||||
return redact_message(value)
|
||||
if isinstance(value, list):
|
||||
return [_redact(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _redact(item) for key, item in value.items() if 'secret' not in str(key).lower() and 'token' not in str(key).lower()}
|
||||
return value
|
||||
|
||||
|
||||
def _setting_key(kind: str, plan_id: str) -> str:
|
||||
return f'{SOURCE_KEY}:{kind}:{plan_id}'
|
||||
|
||||
|
||||
def _get_status(session: Session, plan_id: str) -> str:
|
||||
row = session.get(AppSetting, _setting_key('plan', plan_id))
|
||||
if not row:
|
||||
return 'draft'
|
||||
return str(row.value_json.get('status') or 'draft')
|
||||
|
||||
|
||||
def _set_status(session: Session, plan_id: str, status: str) -> dict[str, Any]:
|
||||
row = session.get(AppSetting, _setting_key('plan', plan_id))
|
||||
if not row:
|
||||
row = AppSetting(key=_setting_key('plan', plan_id), is_secret=False)
|
||||
row.value_json = {'status': status, 'updatedAt': _now()}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
_audit(session, f'ai.remediation.plan.{status}', plan_id, {'status': status})
|
||||
plan = get_plan(session, plan_id)
|
||||
return {'success': True, 'id': plan_id, 'status': status, 'plan': plan.model_dump() if plan else None}
|
||||
|
||||
|
||||
def _audit(session: Session, action_id: str, target: str, metadata: dict[str, Any] | None = None) -> None:
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='ai_remediation_planner',
|
||||
action_id=action_id,
|
||||
target=target,
|
||||
target_type='ai_remediation',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='AI Remediation Planner V1 non-executing workflow.',
|
||||
result='success',
|
||||
redacted_metadata=_redact(metadata or {}),
|
||||
)
|
||||
|
||||
|
||||
def _evidence(source: str, label: str, value: Any = None) -> RemediationEvidence:
|
||||
redacted = _redact(value)
|
||||
return RemediationEvidence(source=source, label=label, value=redacted, redacted=redacted != value)
|
||||
|
||||
|
||||
def _check(label: str, endpoint: str, expected: str, source: str) -> RemediationVerificationCheck:
|
||||
return RemediationVerificationCheck(label=label, endpoint=endpoint, expectedResult=expected, source=source, readOnly=True)
|
||||
|
||||
|
||||
def _resource_sample(resources: list[str], limit: int = 5) -> str:
|
||||
sample = resources[:limit]
|
||||
if not sample:
|
||||
return 'unknown resource'
|
||||
suffix = f' (+{len(resources) - limit} more)' if len(resources) > limit else ''
|
||||
return ', '.join(sample) + suffix
|
||||
|
||||
|
||||
def _connector_setup_href_for_secret(session: Session, secret_ref: str) -> str:
|
||||
connector = session.exec(select(Connector).where(Connector.secret_ref == secret_ref)).first()
|
||||
if connector:
|
||||
return f'/setup?step=connectors&connector={connector.id}'
|
||||
return '/secrets'
|
||||
|
||||
|
||||
def _approval(session: Session, action_id: str, target: str, source: str = 'manual') -> RemediationApprovalRequirement:
|
||||
policy = evaluate_policy(session, action=action_id, target=target, source=source)
|
||||
return RemediationApprovalRequirement(
|
||||
actionId=action_id,
|
||||
target=target,
|
||||
reason=str(policy.get('reason') or 'Policy evaluation required.'),
|
||||
risk=str(policy.get('risk') or 'medium'),
|
||||
policyDecision=str(policy.get('decision') or 'blocked'),
|
||||
)
|
||||
|
||||
|
||||
def _step(
|
||||
session: Session,
|
||||
step_id: str,
|
||||
label: str,
|
||||
description: str,
|
||||
step_type: str,
|
||||
*,
|
||||
href: str | None = None,
|
||||
action_id: str | None = None,
|
||||
target: str | None = None,
|
||||
source: str = 'manual',
|
||||
evidence: list[RemediationEvidence] | None = None,
|
||||
) -> RemediationStep:
|
||||
policy = {'decision': 'allowed', 'risk': 'low'}
|
||||
if action_id and target:
|
||||
policy = evaluate_policy(session, action=action_id, target=target, source=source)
|
||||
return RemediationStep(
|
||||
id=step_id,
|
||||
label=label,
|
||||
description=description,
|
||||
type=step_type, # type: ignore[arg-type]
|
||||
actionId=action_id,
|
||||
target=target,
|
||||
href=href,
|
||||
policyDecision=str(policy.get('decision') or 'allowed'),
|
||||
risk=str(policy.get('risk') or 'low'),
|
||||
requiresApproval=policy.get('decision') == 'approval_required',
|
||||
canExecute=False,
|
||||
evidence=evidence or [],
|
||||
)
|
||||
|
||||
|
||||
def _blocked(session: Session, step_id: str, label: str, action_id: str, target: str, description: str) -> RemediationStep:
|
||||
return _step(session, step_id, label, description, 'blocked_action', action_id=action_id, target=target)
|
||||
|
||||
|
||||
def _base_plan(
|
||||
session: Session,
|
||||
*,
|
||||
plan_id: str,
|
||||
title: str,
|
||||
description: str,
|
||||
source_type: str,
|
||||
source_id: str,
|
||||
severity: str,
|
||||
confidence: str,
|
||||
affected: list[str],
|
||||
prerequisites: list[str],
|
||||
steps: list[RemediationStep],
|
||||
verification: list[RemediationVerificationCheck],
|
||||
blocked: list[RemediationStep],
|
||||
approvals: list[RemediationApprovalRequirement],
|
||||
risk: str,
|
||||
impact: str,
|
||||
rollback: str,
|
||||
) -> RemediationPlan:
|
||||
now = _now()
|
||||
return RemediationPlan(
|
||||
id=plan_id,
|
||||
title=str(_redact(title)),
|
||||
description=str(_redact(description)),
|
||||
sourceType=source_type, # type: ignore[arg-type]
|
||||
sourceId=source_id,
|
||||
severity=severity, # type: ignore[arg-type]
|
||||
confidence=confidence, # type: ignore[arg-type]
|
||||
status=_get_status(session, plan_id), # type: ignore[arg-type]
|
||||
affectedResources=[str(_redact(item)) for item in affected],
|
||||
prerequisites=[str(_redact(item)) for item in prerequisites],
|
||||
steps=steps,
|
||||
verificationChecks=verification,
|
||||
blockedActions=blocked,
|
||||
approvalRequirements=approvals,
|
||||
riskAssessment=str(_redact(risk)),
|
||||
estimatedImpact=str(_redact(impact)),
|
||||
rollbackNotes=str(_redact(rollback)),
|
||||
auditPreview=f'Plan {plan_id} can be reviewed or dry-run only; no remediation execution is available in V1.',
|
||||
createdAt=now,
|
||||
updatedAt=now,
|
||||
executable=False,
|
||||
)
|
||||
|
||||
|
||||
def _generic_finding(finding_id: str) -> dict[str, Any]:
|
||||
title_map = {
|
||||
'ai-sec-auth-public-host-without-authentik': ('Public host without Authentik mapping', 'high', 'auth_coverage', ['public_host']),
|
||||
'ai-sec-npm-certificate-expiring-soon': ('NPM certificate expiring soon', 'medium', 'exposure', ['certificate']),
|
||||
'ai-sec-npm-host-without-certificate': ('NPM public host without certificate', 'high', 'exposure', ['public_host']),
|
||||
'ai-sec-npm-host-without-ssl-force': ('NPM host does not force SSL', 'medium', 'exposure', ['public_host']),
|
||||
'ai-sec-auth-outpost-unavailable': ('Expected Authentik outpost unavailable', 'medium', 'auth_coverage', ['authentik']),
|
||||
'ai-sec-policy-unraid-mover-voice-allowed': ('Unraid mover voice action is allowed', 'high', 'policies', ['unraid.mover.start']),
|
||||
'ai-sec-media-pipeline-issue': ('Media pipeline needs review', 'medium', 'media', ['media_pipeline']),
|
||||
}
|
||||
title, severity, category, affected = title_map.get(finding_id, ('Security finding needs remediation plan', 'medium', 'unknown', ['unknown']))
|
||||
return {
|
||||
'id': finding_id,
|
||||
'title': title,
|
||||
'description': title,
|
||||
'severity': severity,
|
||||
'confidence': 'medium',
|
||||
'category': category,
|
||||
'affectedResources': affected,
|
||||
'evidence': [],
|
||||
}
|
||||
|
||||
|
||||
def _plan_for_finding(session: Session, finding: dict[str, Any]) -> RemediationPlan:
|
||||
finding_id = str(finding.get('id') or 'unknown')
|
||||
title = str(_redact(finding.get('title') or finding_id))
|
||||
severity = str(finding.get('severity') or 'medium')
|
||||
confidence = str(finding.get('confidence') or 'medium')
|
||||
affected = [str(item) for item in finding.get('affectedResources', [])] or ['unknown']
|
||||
evidence = [_evidence(str(item.get('source', 'security')), str(item.get('label', 'evidence')), item.get('value', item.get('observed'))) for item in finding.get('evidence', []) if isinstance(item, dict)]
|
||||
|
||||
if 'missing-secret' in finding_id or finding.get('category') == 'secrets':
|
||||
secret_ref = affected[0]
|
||||
setup_href = _connector_setup_href_for_secret(session, secret_ref)
|
||||
steps = [
|
||||
_step(session, 'open-secrets', 'Open connector setup', 'Open the write-only connector configuration page.', 'open_page', href=setup_href, evidence=evidence),
|
||||
_step(session, 'configure-secret', 'Configure required secret', 'Enter the connector secret in setup. The planner never reads or stores the value.', 'configure_secret', href=setup_href),
|
||||
_step(session, 'test-connector', 'Test connector', 'Run the existing connector test after the secret is configured.', 'run_readonly_check', href=setup_href),
|
||||
_step(session, 'rerun-review', 'Re-run security review', 'Confirm the finding is resolved after connector status updates.', 'run_readonly_check', href='/ai/security-reviewer'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title=f'Remediate missing secret: {secret_ref}', description='Configure the missing connector secret through the write-only vault and validate with read-only checks.', source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Operator has the secret value outside AppOps.', 'Secret storage key is configured.'], steps=steps, verification=[_check('Connector status connected', '/api/connectors/{id}/test', 'connected or configured', 'connectors'), _check('Security finding resolved', '/api/ai/security/findings?scope=secrets', 'finding absent or acknowledged', 'ai_security')], blocked=[], approvals=[], risk='Low operational risk; primary risk is entering the wrong secret.', impact='Connector read operations become available after configuration.', rollback='Replace the secret with the previous value in the vault if validation fails.')
|
||||
|
||||
if 'auth-public-host-without-authentik' in finding_id or 'authentik-missing-public-hosts' in finding_id:
|
||||
host_count = len(affected)
|
||||
host_label = 'public host' if host_count == 1 else 'public hosts'
|
||||
host_sample = _resource_sample(affected)
|
||||
host_evidence = evidence + [_evidence('authentik', 'affectedHosts', affected)]
|
||||
blocked = [
|
||||
_blocked(session, 'blocked-npm-update', 'Automatic NPM modification blocked', 'npm.update_proxy_host', 'nginx_proxy_manager', 'NPM host changes are manual in V1.'),
|
||||
_blocked(session, 'blocked-auth-create', 'Automatic Authentik application/provider creation blocked', 'authentik.create_application', 'authentik', 'Identity-provider changes are manual in V1.'),
|
||||
]
|
||||
approvals = [_approval(session, 'authentik.create_application', 'authentik'), _approval(session, 'npm.update_proxy_host', 'nginx_proxy_manager')]
|
||||
steps = [
|
||||
_step(session, 'open-network', 'Open Exposure Center', f'Review the affected public host list host-by-host: {host_sample}.', 'open_page', href='/network', evidence=host_evidence),
|
||||
_step(session, 'review-npm-host', 'Review NPM proxy host', 'Confirm host-by-host whether each route should stay public and which access list or external auth applies.', 'run_readonly_check', action_id='npm.proxy_hosts', target='nginx_proxy_manager', href='/network'),
|
||||
_step(session, 'open-authentik', 'Open Authentik applications', 'Review existing Authentik applications/providers/outposts.', 'run_readonly_check', action_id='authentik.auth_coverage', target='authentik', href='/apps/authentik'),
|
||||
_step(session, 'manual-auth-change', 'Manually create or verify Authentik coverage', 'Make required Authentik/NPM changes outside this planner for every affected host that should be identity-protected.', 'manual_external_change', href='/apps/authentik'),
|
||||
_step(session, 'rerun-coverage', 'Re-run Auth Coverage', 'Confirm every affected host is protected or intentionally reviewed.', 'run_readonly_check', action_id='authentik.auth_coverage', target='authentik', href='/network'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title=f'Add identity coverage for {host_count} {host_label}', description=f'{title}: {host_sample}', source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['NPM and Authentik connectors are configured.', 'Operator can administer Authentik/NPM directly.', 'Each affected public host is reviewed host-by-host before external changes.'], steps=steps, verification=[_check('Auth coverage protected', '/api/apps/authentik/auth-coverage', 'each affected host is protected, likely_protected, or explicitly reviewed', 'authentik')], blocked=blocked, approvals=approvals, risk='High if left unresolved because public routes may bypass identity controls.', impact='Manual external changes may affect public access to one or more hosts.', rollback='Revert external Authentik/NPM changes in their native UIs if access breaks.')
|
||||
|
||||
if 'certificate-expiring' in finding_id or 'without-certificate' in finding_id:
|
||||
blocked = [_blocked(session, 'blocked-cert-renew', 'Automatic certificate renew blocked', 'npm.renew_certificate', 'nginx_proxy_manager', 'Certificate renewal remains manual in NPM for V1.')]
|
||||
steps = [
|
||||
_step(session, 'open-npm', 'Open NPM AppDetail certificate panel', 'Inspect certificate metadata and affected domains.', 'open_page', href='/apps/nginx_proxy_manager', evidence=evidence),
|
||||
_step(session, 'review-cert', 'Review certificate expiry', 'Validate expiry date, provider and covered domains.', 'run_readonly_check', action_id='npm.certificates', target='nginx_proxy_manager', href='/network'),
|
||||
_step(session, 'manual-renew', 'Manually renew in NPM if needed', 'Perform renewal in NPM directly; this planner will not call renew.', 'manual_external_change', href='/apps/nginx_proxy_manager'),
|
||||
_step(session, 'rerun-cert-check', 'Re-run NPM cert check', 'Refresh certificate warning data.', 'run_readonly_check', action_id='npm.expiring_certificates', target='nginx_proxy_manager', href='/network'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Review expiring NPM certificate', description=title, source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['NPM connector can read certificate metadata.', 'Operator has NPM admin access.'], steps=steps, verification=[_check('Certificate no longer expiring', '/api/apps/nginx-proxy-manager/certificates/expiring?days=30', 'affected certificate absent', 'npm')], blocked=blocked, approvals=[_approval(session, 'npm.renew_certificate', 'nginx_proxy_manager')], risk='Medium; access can fail when TLS expires.', impact='Manual renewal may briefly affect the public host.', rollback='Restore previous certificate state in NPM if renewal introduces errors.')
|
||||
|
||||
if 'without-ssl-force' in finding_id or 'ssl force' in title.lower() or 'force ssl' in title.lower():
|
||||
steps = [
|
||||
_step(session, 'open-network', 'Open Exposure Center', 'Inspect the affected NPM public host and its SSL policy.', 'open_page', href='/network', evidence=evidence),
|
||||
_step(session, 'review-proxy-host', 'Review proxy host SSL settings', 'Confirm Force SSL is disabled for the affected host and whether that is intentional.', 'run_readonly_check', action_id='npm.proxy_hosts', target='nginx_proxy_manager', href='/network'),
|
||||
_step(session, 'manual-ssl-change', 'Manually enable Force SSL in NPM if appropriate', 'Change Force SSL in NPM directly after checking the app still works over HTTPS.', 'manual_external_change', href='/apps/nginx_proxy_manager'),
|
||||
_step(session, 'rerun-exposure', 'Re-run exposure review', 'Confirm the SSL-force finding is resolved or explicitly acknowledged.', 'run_readonly_check', action_id='npm.proxy_hosts', target='nginx_proxy_manager', href='/network'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Review NPM SSL enforcement', description=f'Force SSL is not enabled for: {_resource_sample(affected)}.', source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['NPM connector can read proxy-host metadata.', 'Operator has NPM admin access.', 'HTTPS certificate is valid for the affected host.'], steps=steps, verification=[_check('Proxy host forces SSL', '/api/apps/nginx-proxy-manager/proxy-hosts', 'affected host has sslForced=true or finding acknowledged', 'npm')], blocked=[_blocked(session, 'blocked-npm-update', 'Automatic NPM proxy-host update blocked', 'npm.update_proxy_host', 'nginx_proxy_manager', 'Planner V1 does not mutate NPM proxy hosts.')], approvals=[_approval(session, 'npm.update_proxy_host', 'nginx_proxy_manager')], risk='Medium; users may reach the host over plain HTTP if Force SSL is disabled.', impact='Manual Force SSL changes can affect clients using HTTP-only URLs.', rollback='Disable Force SSL again in NPM if the host breaks, then review certificate and upstream settings.')
|
||||
|
||||
if 'outpost-unavailable' in finding_id or 'outpost' in title.lower():
|
||||
steps = [
|
||||
_step(session, 'open-authentik', 'Open Authentik AppDetail', 'Inspect Authentik connector health and outpost metadata.', 'open_page', href='/apps/authentik', evidence=evidence),
|
||||
_step(session, 'review-outpost-health', 'Review outpost health', 'Read the Authentik outpost health summary and provider count.', 'run_readonly_check', action_id='authentik.outpost_health', target='authentik', href='/apps/authentik'),
|
||||
_step(session, 'manual-outpost-review', 'Verify outpost in Authentik', 'Use Authentik directly to check the embedded outpost, providers and integration health.', 'manual_external_change', href='/apps/authentik'),
|
||||
_step(session, 'rerun-auth-review', 'Re-run Authentik security review', 'Confirm outpost health is healthy or intentionally documented.', 'run_readonly_check', action_id='authentik.outpost_health', target='authentik', href='/ai/security-reviewer'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Review Authentik outpost health', description='Authentik reports the expected outpost as unavailable or unknown.', source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Authentik connector is configured.', 'Operator can inspect Authentik outposts directly.'], steps=steps, verification=[_check('Outpost health is available', '/api/apps/authentik/outposts/health', 'available=true or explicitly reviewed', 'authentik')], blocked=[_blocked(session, 'blocked-auth-update', 'Automatic Authentik outpost mutation blocked', 'authentik.update_policy', 'authentik', 'Planner V1 does not mutate Authentik topology.')], approvals=[_approval(session, 'authentik.update_policy', 'authentik')], risk='Medium; unknown outpost health can make external auth coverage harder to trust.', impact='Manual Authentik changes can affect public authentication flows.', rollback='Revert outpost/provider changes in Authentik if application access is affected.')
|
||||
|
||||
if finding.get('category') == 'media':
|
||||
steps = [
|
||||
_step(session, 'open-media', 'Open Media Pipeline', 'Review connector status, stalled downloads and pipeline issues.', 'open_page', href='/media-pipeline', evidence=evidence),
|
||||
_step(session, 'run-diagnose', 'Run media pipeline diagnose', 'Use the existing read-only diagnose flow to collect current media pipeline state.', 'run_readonly_check', action_id='media_pipeline.diagnose', target='media-pipeline', href='/media-pipeline'),
|
||||
_step(session, 'review-connectors', 'Review missing media connectors', 'Configure only the connectors you intend to use; missing optional connectors should remain visible as partial coverage.', 'bind_connector', href='/setup?step=connectors'),
|
||||
_step(session, 'rerun-review', 'Re-run safety/security review', 'Confirm media finding is resolved, reduced to known partial coverage or acknowledged.', 'run_readonly_check', action_id='media_pipeline.diagnose', target='media-pipeline', href='/safety'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Review media pipeline issue', description=str(_redact(finding.get('description') or title)), source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Media Pipeline route is reachable.', 'Operator knows which media connectors should be active in this deployment.'], steps=steps, verification=[_check('Media pipeline summary reviewed', '/api/media-pipeline/summary', 'status healthy, warning accepted, or partial coverage documented', 'media_pipeline'), _check('Media pipeline issues reviewed', '/api/media-pipeline/issues', 'no unexpected critical issues', 'media_pipeline')], blocked=[_blocked(session, 'blocked-deluge-delete-data', 'No delete/remove with data', 'deluge.remove_torrent_data', 'deluge', 'Delete-with-data remains blocked.')], approvals=[_approval(session, 'media_pipeline.diagnose', 'media-pipeline')], risk='Medium; media connector gaps can hide stalled downloads or processing failures.', impact='Read-only review has no runtime impact; any connector setup is manual.', rollback='Remove or replace connector settings manually if a configured connector is wrong.')
|
||||
|
||||
if finding.get('category') == 'policies' or 'policy-' in finding_id or 'voice' in title.lower():
|
||||
action_id = affected[0] if affected else 'unraid.mover.start'
|
||||
steps = [
|
||||
_step(session, 'open-policies', 'Open Policies page', 'Inspect the risky policy rule.', 'open_page', href='/policies', evidence=evidence),
|
||||
_step(session, 'simulate-voice', 'Simulate voice policy decision', 'Use policy simulation for the affected action and target.', 'review_policy', action_id=action_id, target='unraid', source='voice', href='/policies'),
|
||||
_step(session, 'draft-policy', 'Create policy draft suggestion', 'Draft a safer blocked or approval-required setting for operator review.', 'documentation', href='/policies'),
|
||||
_step(session, 'manual-apply', 'Manual policy apply outside V1', 'Apply the policy change through an explicit future policy workflow, not this planner.', 'manual_external_change', href='/policies'),
|
||||
]
|
||||
blocked = [_blocked(session, 'blocked-policy-update', 'Automatic policy apply blocked', 'authentik.update_policy', 'policy', 'Planner V1 does not mutate policies.')]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Reduce risky voice policy', description=title, source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Operator understands desired policy posture.', 'Policy changes are reviewed manually.'], steps=steps, verification=[_check('Policy no longer voice-allowed', '/api/policies/evaluate', 'blocked or approval_required for voice', 'policies')], blocked=blocked, approvals=[_approval(session, action_id, 'unraid', 'voice')], risk='High when voice can trigger state-changing operations.', impact='Safer policy can require more manual approvals.', rollback='Restore previous policy value from audit/config backup if needed.')
|
||||
|
||||
if finding.get('category') == 'inventory' or 'critical-app-unprotected' in finding_id or 'needs review' in title.lower():
|
||||
app_id = affected[0]
|
||||
steps = [
|
||||
_step(session, 'open-inventory', 'Open Inventory/AppDetail', 'Review app import, category and criticality.', 'open_page', href='/inventory', evidence=evidence),
|
||||
_step(session, 'mark-protected', 'Mark protected manually or review import', 'Use existing inventory workflows to classify the app.', 'manual_external_change', href=f'/apps/{app_id}'),
|
||||
_step(session, 'review-policy-profile', 'Review policy profile', 'Confirm policy profile matches app criticality.', 'review_policy', href='/policies'),
|
||||
_step(session, 'rerun-review', 'Re-run safety/security review', 'Confirm protection finding is gone.', 'run_readonly_check', href='/ai/security-reviewer'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Review critical app protection', description=title, source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Inventory source is current.', 'Operator has enough context to classify the app.'], steps=steps, verification=[_check('App managed/protected', '/api/inventory/apps', 'protected=true or reviewed', 'inventory')], blocked=[_blocked(session, 'blocked-policy-mutation', 'Automatic policy mutation blocked', 'stack.redeploy', app_id, 'Planner does not mutate app policy profile.')], approvals=[], risk='High for critical apps without explicit protection.', impact='Manual classification affects future policy decisions.', rollback='Change classification back manually if misclassified.')
|
||||
|
||||
if finding.get('category') == 'audit' or 'blocked-attempts' in finding_id:
|
||||
steps = [
|
||||
_step(session, 'open-audit', 'Open Audit Log', 'Inspect repeated blocked/failed attempts.', 'open_page', href='/audit', evidence=evidence),
|
||||
_step(session, 'review-automation', 'Review automation or voice source', 'Identify whether attempts are expected or noisy.', 'documentation', href='/audit'),
|
||||
_step(session, 'review-policy', 'Review related policy', 'Confirm blocked behavior is intentional.', 'review_policy', href='/policies'),
|
||||
]
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title='Triage repeated blocked attempts', description=title, source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Recent audit events are available.'], steps=steps, verification=[_check('Blocked attempts decrease', '/api/audit', 'no repeated attempts in next window', 'audit')], blocked=[], approvals=[], risk='Medium; repeated denials can indicate noisy automation or probing.', impact='May require changing external automation or voice phrasing.', rollback='Re-enable previous automation only after policy review.')
|
||||
|
||||
return _base_plan(session, plan_id=f'rem-{finding_id}', title=f'Review finding: {title}', description=str(_redact(finding.get('description') or title)), source_type='security_finding', source_id=finding_id, severity=severity, confidence=confidence, affected=affected, prerequisites=['Review source finding evidence.'], steps=[_step(session, 'open-reviewer', 'Open Security Reviewer', 'Inspect finding evidence and links.', 'open_page', href='/ai/security-reviewer', evidence=evidence)], verification=[_check('Finding reviewed', '/api/ai/security/findings', 'status acknowledged or resolved', 'ai_security')], blocked=[], approvals=[], risk='Depends on source finding severity.', impact='No direct impact; V1 is plan-only.', rollback='Dismiss plan if not actionable.')
|
||||
|
||||
|
||||
def _plan_for_media_issue(session: Session, issue: dict[str, Any]) -> RemediationPlan:
|
||||
issue_id = str(issue.get('id') or issue.get('title') or 'media-issue').lower().replace(' ', '-')
|
||||
title = str(_redact(issue.get('title') or 'Media pipeline issue'))
|
||||
affected = [str(item) for item in issue.get('affectedConnectors', issue.get('affected', []))] or ['media_pipeline']
|
||||
evidence = [_evidence('media_pipeline', 'issue', issue.get('evidence', []))]
|
||||
if 'widefrog' in title.lower() or 'widefrog' in affected:
|
||||
steps = [
|
||||
_step(session, 'open-widefrog', 'Open Widefrog AppDetail', 'Review failed jobs and queue state.', 'open_page', href='/apps/widefrog', evidence=evidence),
|
||||
_step(session, 'review-failed-jobs', 'Review failed jobs', 'Inspect failure reason before retry.', 'run_readonly_check', action_id='widefrog.failed_jobs', target='widefrog', href='/apps/widefrog'),
|
||||
_step(session, 'retry-if-approved', 'Plan retry_failed through policy flow', 'Retry can only run through existing policy/approval flow outside this planner.', 'request_approval', action_id='widefrog.retry_failed', target='widefrog', href='/apps/widefrog'),
|
||||
_step(session, 'verify-queue', 'Verify queue/activity', 'Confirm failed count decreases after manual execution.', 'run_readonly_check', action_id='widefrog.queue_status', target='widefrog', href='/apps/widefrog'),
|
||||
]
|
||||
blocked = [_blocked(session, 'blocked-delete-jobs', 'Delete jobs/media blocked', 'widefrog.delete_job', 'widefrog', 'Delete operations are forbidden.')]
|
||||
return _base_plan(session, plan_id='rem-media-widefrog-failed', title='Plan Widefrog failed jobs recovery', description=title, source_type='media_issue', source_id=issue_id, severity='medium', confidence='medium', affected=affected, prerequisites=['Widefrog connector configured.', 'Operator confirms retry is safe.'], steps=steps, verification=[_check('Failed jobs decrease', '/api/apps/widefrog/failed', 'failed count lower', 'widefrog')], blocked=blocked, approvals=[_approval(session, 'widefrog.retry_failed', 'widefrog')], risk='Medium; retry can repeat failed processing.', impact='May increase load on media processing.', rollback='Stop external retry workflow if failures repeat.')
|
||||
steps = [
|
||||
_step(session, 'open-media', 'Open Media Pipeline', 'Review pipeline issue context.', 'open_page', href='/media-pipeline', evidence=evidence),
|
||||
_step(session, 'open-deluge', 'Open Deluge detail', 'Review stalled torrents without deleting data.', 'run_readonly_check', action_id='deluge.stalled_downloads', target='deluge', href='/apps/deluge'),
|
||||
_step(session, 'resume-specific', 'Plan resume specific torrent if policy allows', 'Resume must go through existing policy/approval flow outside this planner.', 'request_approval', action_id='deluge.resume_torrent', target='deluge', href='/apps/deluge'),
|
||||
_step(session, 'verify-stalled', 'Verify stalled count decreases', 'Re-run download checks after manual action.', 'run_readonly_check', action_id='deluge.stalled_downloads', target='deluge', href='/media-pipeline'),
|
||||
]
|
||||
blocked = [_blocked(session, 'blocked-remove-data', 'No delete/remove with data', 'deluge.remove_torrent_data', 'deluge', 'Delete-with-data remains blocked.')]
|
||||
return _base_plan(session, plan_id='rem-media-deluge-stalled', title='Plan Deluge stalled downloads recovery', description=title, source_type='media_issue', source_id=issue_id, severity='medium', confidence='medium', affected=affected, prerequisites=['Deluge connector configured.', 'Operator identifies the specific stalled torrent.'], steps=steps, verification=[_check('Stalled downloads decrease', '/api/apps/deluge/stalled', 'stalled count lower', 'deluge')], blocked=blocked, approvals=[_approval(session, 'deluge.resume_torrent', 'deluge')], risk='Medium; resuming downloads can affect bandwidth and storage.', impact='Download queue may resume network/storage activity.', rollback='Pause the specific torrent through normal policy flow if needed.')
|
||||
|
||||
|
||||
def _inventory_plans(session: Session) -> list[RemediationPlan]:
|
||||
from .app_registry import list_active_apps
|
||||
|
||||
plans: list[RemediationPlan] = []
|
||||
for app in list_active_apps(session):
|
||||
if app.needs_review:
|
||||
finding = {'id': f'inventory-needs-review-{app.id}', 'title': f'Discovered app needs review: {app.name}', 'severity': app.criticality if app.criticality in {'low', 'medium', 'high', 'critical'} else 'medium', 'confidence': 'medium', 'category': 'inventory', 'affectedResources': [app.id], 'evidence': []}
|
||||
plans.append(_plan_for_finding(session, finding))
|
||||
break
|
||||
return plans
|
||||
|
||||
|
||||
def _connector_plans(session: Session) -> list[RemediationPlan]:
|
||||
plans: list[RemediationPlan] = []
|
||||
for connector in session.exec(select(Connector)).all():
|
||||
if connector.status == 'missing_config':
|
||||
finding = {'id': f'connector-missing-config-{connector.id}', 'title': f'Connector missing config: {connector.name}', 'severity': 'medium', 'confidence': 'high', 'category': 'connectors', 'affectedResources': [connector.id], 'evidence': [{'source': 'connectors', 'label': 'status', 'value': connector.status}]}
|
||||
plans.append(_plan_for_finding(session, finding))
|
||||
break
|
||||
return plans
|
||||
|
||||
|
||||
def _secret_storage_plan(session: Session) -> list[RemediationPlan]:
|
||||
status = secret_storage_status()
|
||||
if status.get('status') != 'warning' and status.get('productionReady') is not False:
|
||||
return []
|
||||
plan = _base_plan(
|
||||
session,
|
||||
plan_id='rem-secret-storage-key-warning',
|
||||
title='Configure production secret storage key',
|
||||
description='Configure APPOPS_SECRET_KEY or APPOPS_SECRET_KEY_FILE and back it up with the database.',
|
||||
source_type='secret',
|
||||
source_id='secret-storage-key-warning',
|
||||
severity='medium',
|
||||
confidence='high',
|
||||
affected=['secret_storage'],
|
||||
prerequisites=['Operator can set deployment environment variables.', 'Backup process is available.'],
|
||||
steps=[
|
||||
_step(session, 'open-settings', 'Open settings/secrets status', 'Review key source and production readiness.', 'open_page', href='/secrets'),
|
||||
_step(session, 'read-docs', 'Open Secret Storage docs', 'Follow production key setup guidance.', 'documentation', href='/docs/SECRET_STORAGE.md'),
|
||||
_step(session, 'configure-key', 'Configure APPOPS_SECRET_KEY', 'Set the key in deployment environment outside this planner.', 'manual_external_change'),
|
||||
_step(session, 'backup-db-key', 'Back up database with key', 'Back up /data/appops.db and the key material together.', 'documentation'),
|
||||
_step(session, 'runtime-health', 'Re-run runtime health', 'Confirm production readiness.', 'run_readonly_check', href='/settings'),
|
||||
],
|
||||
verification=[_check('Runtime health production ready', '/api/runtime/info', 'secretStorage.productionReady=true', 'runtime')],
|
||||
blocked=[_blocked(session, 'blocked-key-rotation', 'Automated key rotation blocked', 'secrets.show', 'secret_storage', 'Key automation is not implemented in planner V1.')],
|
||||
approvals=[],
|
||||
risk='Medium; missing production key affects encrypted secret durability.',
|
||||
impact='Deployment environment change and backup process update.',
|
||||
rollback='Restore previous environment and database backup if startup fails.',
|
||||
)
|
||||
return [plan]
|
||||
|
||||
|
||||
def generate_plans(session: Session, audit_run: bool = False, include_media_issues: bool | None = None) -> RemediationPlanCollection:
|
||||
if include_media_issues is None:
|
||||
include_media_issues = audit_run
|
||||
|
||||
review = ai_security_reviewer.generate_review(session)
|
||||
findings = [finding.model_dump() for finding in review.findings]
|
||||
recommendations = [recommendation.model_dump() for recommendation in review.recommendations]
|
||||
finding_ids = {str(finding.get('id')) for finding in findings}
|
||||
plans = [_plan_for_finding(session, finding) for finding in findings if finding.get('status') != 'dismissed']
|
||||
plans.extend(
|
||||
_plan_for_recommendation(session, recommendation)
|
||||
for recommendation in recommendations
|
||||
if recommendation.get('status') != 'dismissed' and str(recommendation.get('id') or '').removeprefix('rec-') not in finding_ids
|
||||
)
|
||||
if include_media_issues:
|
||||
try:
|
||||
from . import media_pipeline_service
|
||||
|
||||
issues = media_pipeline_service.issues(session).get('issues', [])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
issues = [{'id': 'media-unavailable', 'title': redact_message(str(exc)), 'affectedConnectors': ['media_pipeline']}]
|
||||
plans.extend(_plan_for_media_issue(session, issue) for issue in issues if isinstance(issue, dict))
|
||||
plans.extend(_inventory_plans(session))
|
||||
plans.extend(_connector_plans(session))
|
||||
plans.extend(_secret_storage_plan(session))
|
||||
deduped = list({plan.id: plan for plan in plans}.values())
|
||||
if audit_run:
|
||||
_audit(session, 'ai.remediation.generate', 'all', {'plans': len(deduped)})
|
||||
return RemediationPlanCollection(status='ok', generatedAt=_now(), total=len(deduped), plans=deduped)
|
||||
|
||||
|
||||
def _plan_for_recommendation(session: Session, recommendation: dict[str, Any]) -> RemediationPlan:
|
||||
source_id = str(recommendation.get('id') or 'recommendation')
|
||||
finding_id = source_id.removeprefix('rec-')
|
||||
finding = {
|
||||
'id': finding_id,
|
||||
'title': recommendation.get('title') or source_id,
|
||||
'description': recommendation.get('description') or recommendation.get('suggestedAction') or source_id,
|
||||
'severity': recommendation.get('severity') or 'medium',
|
||||
'confidence': recommendation.get('confidence') or 'medium',
|
||||
'category': 'unknown',
|
||||
'affectedResources': recommendation.get('affectedResources', []),
|
||||
'evidence': recommendation.get('evidence', []),
|
||||
}
|
||||
plan = _plan_for_finding(session, finding)
|
||||
return plan.model_copy(update={'id': f'rem-{source_id}', 'sourceType': 'security_recommendation', 'sourceId': source_id, 'status': _get_status(session, f'rem-{source_id}')})
|
||||
|
||||
|
||||
def get_plans(session: Session) -> RemediationPlanCollection:
|
||||
return generate_plans(session)
|
||||
|
||||
|
||||
def get_plan(session: Session, plan_id: str) -> RemediationPlan | None:
|
||||
plan = next((item for item in generate_plans(session).plans if item.id == plan_id), None)
|
||||
if plan:
|
||||
return plan
|
||||
if plan_id.startswith('rem-ai-sec-'):
|
||||
return _plan_for_finding(session, _generic_finding(plan_id.removeprefix('rem-')))
|
||||
return None
|
||||
|
||||
|
||||
def generate_from_finding(session: Session, finding_id: str) -> RemediationPlan:
|
||||
finding = ai_security_reviewer.get_finding(session, finding_id) or _generic_finding(finding_id)
|
||||
plan = _plan_for_finding(session, finding)
|
||||
_audit(session, 'ai.remediation.plan.generated', plan.id, {'sourceType': 'security_finding', 'sourceId': finding_id})
|
||||
return plan
|
||||
|
||||
|
||||
def generate_from_recommendation(session: Session, recommendation_id: str) -> RemediationPlan:
|
||||
recommendation = next((item for item in ai_security_reviewer.get_recommendations(session) if item['id'] == recommendation_id), {'id': recommendation_id, 'title': recommendation_id, 'severity': 'medium', 'confidence': 'medium', 'affectedResources': []})
|
||||
plan = _plan_for_recommendation(session, recommendation)
|
||||
_audit(session, 'ai.remediation.plan.generated', plan.id, {'sourceType': 'security_recommendation', 'sourceId': recommendation_id})
|
||||
return plan
|
||||
|
||||
|
||||
def review_plan(session: Session, plan_id: str) -> dict[str, Any]:
|
||||
return _set_status(session, plan_id, 'reviewed')
|
||||
|
||||
|
||||
def dismiss_plan(session: Session, plan_id: str) -> dict[str, Any]:
|
||||
return _set_status(session, plan_id, 'dismissed')
|
||||
|
||||
|
||||
def dry_run(session: Session, plan_id: str) -> RemediationDryRunResult | None:
|
||||
plan = get_plan(session, plan_id)
|
||||
if not plan:
|
||||
return None
|
||||
step_results: list[RemediationDryRunStepResult] = []
|
||||
approvals: list[RemediationApprovalRequirement] = []
|
||||
blocked: list[RemediationApprovalRequirement] = []
|
||||
reasons: list[str] = ['Planner V1 dry-run only; no steps were executed.']
|
||||
for step in plan.steps + plan.blockedActions:
|
||||
decision = step.policyDecision
|
||||
risk = step.risk
|
||||
reason = 'No executable action attached.'
|
||||
if step.actionId and step.target:
|
||||
policy = evaluate_policy(session, action=step.actionId, target=step.target, source='manual')
|
||||
decision = str(policy.get('decision') or 'blocked')
|
||||
risk = str(policy.get('risk') or step.risk)
|
||||
reason = str(policy.get('reason') or 'Policy evaluated.')
|
||||
status = 'blocked' if decision == 'blocked' or step.type == 'blocked_action' else ('warning' if decision == 'approval_required' else 'pass')
|
||||
result = RemediationDryRunStepResult(stepId=step.id, label=step.label, actionId=step.actionId, target=step.target, policyDecision=decision, risk=risk, status=status, reason=reason, canExecute=False)
|
||||
step_results.append(result)
|
||||
if step.actionId and step.target and decision == 'approval_required':
|
||||
approvals.append(RemediationApprovalRequirement(actionId=step.actionId, target=step.target, reason=reason, risk=risk, policyDecision=decision))
|
||||
if step.actionId and step.target and status == 'blocked':
|
||||
blocked.append(RemediationApprovalRequirement(actionId=step.actionId, target=step.target, reason=reason, risk=risk, policyDecision=decision))
|
||||
missing_secret_names = [item.get('secretRef') for item in list_secret_status(session) if not item.get('hasSecret')]
|
||||
unresolved = [f'Missing secret: {name}' for name in missing_secret_names if name]
|
||||
if blocked:
|
||||
status = 'blocked'
|
||||
elif approvals or unresolved:
|
||||
status = 'warning'
|
||||
else:
|
||||
status = 'pass'
|
||||
result = RemediationDryRunResult(planId=plan.id, status=status, executed=False, reasons=reasons, prerequisites=plan.prerequisites, unresolvedBlockers=unresolved, blockedActions=blocked, approvalRequirements=approvals, verificationChecks=plan.verificationChecks, stepResults=step_results)
|
||||
_audit(session, 'ai.remediation.dry_run', plan_id, {'status': status, 'blocked': len(blocked), 'approvals': len(approvals)})
|
||||
return result
|
||||
|
||||
|
||||
def create_approval_proposal(session: Session, plan_id: str) -> RemediationApprovalProposal | None:
|
||||
plan = get_plan(session, plan_id)
|
||||
if not plan:
|
||||
return None
|
||||
proposals = plan.approvalRequirements or [
|
||||
RemediationApprovalRequirement(actionId=step.actionId, target=step.target, reason='Manual approval may be required by policy.', risk=step.risk, policyDecision=step.policyDecision)
|
||||
for step in plan.steps
|
||||
if step.actionId and step.target and step.policyDecision == 'approval_required'
|
||||
]
|
||||
row = session.get(AppSetting, _setting_key('proposal', plan_id))
|
||||
if not row:
|
||||
row = AppSetting(key=_setting_key('proposal', plan_id), is_secret=False)
|
||||
row.value_json = {'createdAt': _now(), 'proposals': [proposal.model_dump() for proposal in proposals], 'executed': False, 'approvalId': None}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
_audit(session, 'ai.remediation.approval_proposal', plan_id, {'proposals': len(proposals), 'executed': False})
|
||||
return RemediationApprovalProposal(planId=plan_id, created=True, executed=False, approvalId=None, proposals=proposals, message='Approval proposal created as planner draft only. No approval or action was executed.')
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _text(raw: dict[str, Any]) -> str:
|
||||
labels = raw.get('labels') if isinstance(raw.get('labels'), dict) else {}
|
||||
label_text = ' '.join(f'{key} {value}' for key, value in labels.items())
|
||||
ports = ' '.join(str(port) for port in raw.get('ports', []) if port is not None)
|
||||
text = f"{raw.get('name', '')} {raw.get('image', '')} {label_text} {ports}".lower()
|
||||
# Preserve the source while also matching humanized names such as "Gpu Hot"
|
||||
# against the same conservative patterns used for container IDs and images.
|
||||
return ' '.join((text, re.sub(r'[\s_]+', '-', text), re.sub(r'[\s-]+', '_', text)))
|
||||
|
||||
|
||||
def _match(text: str, *patterns: str) -> bool:
|
||||
return any(re.search(pattern, text) for pattern in patterns)
|
||||
|
||||
|
||||
def _base(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
name = str(raw.get('name') or raw.get('container_id') or 'unknown').strip('/') or 'unknown'
|
||||
safe_id = re.sub(r'[^a-z0-9_]+', '-', name.lower()).strip('-') or 'unknown'
|
||||
return {
|
||||
'suggested_app_id': safe_id.replace('-', '_'),
|
||||
'display_name': name.replace('-', ' ').replace('_', ' ').title(),
|
||||
'app_type': 'unknown',
|
||||
'category': 'unknown',
|
||||
'criticality': 'low',
|
||||
'connector': 'none',
|
||||
'policy_profile': 'generic_unknown',
|
||||
'protected': False,
|
||||
'healthcheck_suggestion': None,
|
||||
'voice_suggestion': 'disabled',
|
||||
'confidence': 0.2,
|
||||
'reasons': ['No known pattern matched.'],
|
||||
'import_status': 'unmapped',
|
||||
}
|
||||
|
||||
|
||||
def classify_container(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
text = _text(raw)
|
||||
result = _base(raw)
|
||||
|
||||
def apply(
|
||||
*,
|
||||
app_type: str,
|
||||
category: str,
|
||||
criticality: str,
|
||||
connector: str = 'none',
|
||||
policy_profile: str | None = None,
|
||||
subcategory: str | None = None,
|
||||
protected: bool = False,
|
||||
privacy_sensitive: bool = False,
|
||||
confidence: float = 0.85,
|
||||
reason: str,
|
||||
default_port: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
result.update(
|
||||
{
|
||||
'suggested_app_id': app_type,
|
||||
'display_name': app_type.replace('_', ' ').title(),
|
||||
'app_type': app_type,
|
||||
'category': category,
|
||||
'subcategory': subcategory,
|
||||
'criticality': criticality,
|
||||
'connector': connector,
|
||||
'policy_profile': policy_profile or category,
|
||||
'protected': protected,
|
||||
'privacy_sensitive': privacy_sensitive,
|
||||
'healthcheck_suggestion': {'port': default_port} if default_port else {'type': 'http'},
|
||||
'voice_suggestion': 'blocked' if protected else 'disabled',
|
||||
'confidence': confidence,
|
||||
'reasons': [reason],
|
||||
'import_status': 'discovered',
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
if _match(text, r'\b(binhex-)?sonarr\b', r'arch-sonarr'):
|
||||
return apply(app_type='sonarr', category='media', criticality='medium', connector='sonarr', policy_profile='media_app', reason='Matched Sonarr name/image pattern.')
|
||||
if _match(text, r'\b(binhex-)?radarr\b', r'arch-radarr'):
|
||||
return apply(app_type='radarr', category='media', criticality='medium', connector='radarr', policy_profile='media_app', reason='Matched Radarr name/image pattern.')
|
||||
if _match(text, r'\bprowlarr\b'):
|
||||
return apply(app_type='prowlarr', category='media', criticality='medium', connector='prowlarr', policy_profile='media_indexer', reason='Matched Prowlarr pattern.', default_port=9696)
|
||||
if _match(text, r'\b(binhex-)?deluge(vpn)?\b', r'arch-delugevpn', r'linuxserver/deluge'):
|
||||
return apply(app_type='deluge', category='downloads', subcategory='torrent_client', criticality='medium', connector='deluge', policy_profile='download_client', reason='Matched Deluge download client pattern.', default_port=8112)
|
||||
if _match(text, r'\b(binhex-)?plex(pass)?\b', r'arch-plexpass', r'linuxserver/plex', r'plexinc/pms-docker'):
|
||||
return apply(app_type='plex', category='media', subcategory='media_server', criticality='medium', connector='plex', policy_profile='media_server', reason='Matched Plex media server pattern.', default_port=32400)
|
||||
if _match(text, r'\bjellyfin\b'):
|
||||
return apply(app_type='jellyfin', category='media', criticality='medium', connector='none', policy_profile='media_app', reason='Matched Jellyfin media server pattern; no AppOps Jellyfin connector is registered.')
|
||||
if _match(text, r'\b(binhex-)?tautulli\b', r'linuxserver/tautulli', r'tautulli/tautulli'):
|
||||
return apply(app_type='tautulli', category='media', subcategory='plex_analytics', criticality='medium', connector='tautulli', policy_profile='media_analytics', privacy_sensitive=True, reason='Matched Tautulli Plex analytics pattern.', default_port=8181)
|
||||
if _match(text, r'\b(qbittorrent|deluge|sabnzbd|nzbget)\b'):
|
||||
app_type = next((name for name in ['qbittorrent', 'deluge', 'sabnzbd', 'nzbget'] if name in text), 'downloads')
|
||||
return apply(app_type=app_type, category='downloads', criticality='medium', policy_profile='download_client', reason='Matched download client pattern.')
|
||||
if _match(text, r'\bauthentik\b', r'\bauthentik-(server|worker)\b', r'ghcr\.io/goauthentik/(server|proxy)', r'beryju/authentik', r'\bgoauthentik\b'):
|
||||
return apply(
|
||||
app_type='authentik',
|
||||
category='auth',
|
||||
subcategory='identity_provider',
|
||||
criticality='critical',
|
||||
connector='authentik',
|
||||
protected=True,
|
||||
policy_profile='auth_provider',
|
||||
reason='Matched Authentik identity provider pattern.',
|
||||
default_port=9000,
|
||||
)
|
||||
if _match(text, r'\b(postgres|postgresql|mariadb|mysql)\b'):
|
||||
app_type = next((name for name in ['postgres', 'postgresql', 'mariadb', 'mysql'] if name in text), 'database')
|
||||
return apply(app_type=app_type, category='database', criticality='critical', protected=True, policy_profile='database', reason='Matched database pattern.')
|
||||
if _match(text, r'\bredis\b'):
|
||||
return apply(app_type='redis', category='database', criticality='high', protected=True, policy_profile='database', reason='Matched Redis pattern.')
|
||||
if _match(text, r'\b(nginxproxymanager|nginx-proxy-manager|jc21/nginx-proxy-manager|npm)\b'):
|
||||
return apply(
|
||||
app_type='nginx_proxy_manager',
|
||||
category='network',
|
||||
subcategory='reverse_proxy',
|
||||
criticality='critical',
|
||||
connector='nginx_proxy_manager',
|
||||
protected=True,
|
||||
policy_profile='reverse_proxy',
|
||||
reason='Matched Nginx Proxy Manager reverse proxy pattern.',
|
||||
default_port=81,
|
||||
)
|
||||
if _match(text, r'\bportainer\b'):
|
||||
return apply(app_type='portainer', category='infrastructure', criticality='critical', protected=True, policy_profile='protected_service', reason='Matched Portainer pattern.')
|
||||
if _match(text, r'\b(homeassistant|home-assistant|home_assistant)\b'):
|
||||
return apply(app_type='home_assistant', category='home_automation', criticality='high', protected=True, policy_profile='protected_service', reason='Matched Home Assistant pattern.')
|
||||
if _match(text, r'\bgitea\b'):
|
||||
return apply(app_type='gitea', category='dev', criticality='medium', policy_profile='dev_tool', reason='Matched Gitea pattern.')
|
||||
if _match(text, r'\bwidefrog\b'):
|
||||
return apply(app_type='widefrog', category='media', criticality='medium', connector='widefrog', policy_profile='media_custom_app', reason='Matched Widefrog pattern.', default_port=1207)
|
||||
if _match(text, r'\b(adguard|adguardhome|adguard-home)\b'):
|
||||
return apply(app_type='adguard_home', category='network', criticality='high', protected=True, policy_profile='network_service', reason='Matched AdGuard network service pattern.')
|
||||
if _match(text, r'\b(vaultwarden|bitwarden)\b'):
|
||||
return apply(app_type='vaultwarden', category='auth', criticality='critical', protected=True, privacy_sensitive=True, policy_profile='auth_provider', reason='Matched credential vault pattern.')
|
||||
if _match(text, r'\b(adminer|phpmyadmin)\b'):
|
||||
app_type = 'adminer' if 'adminer' in text else 'phpmyadmin'
|
||||
return apply(app_type=app_type, category='database', criticality='high', protected=True, policy_profile='database_admin', reason='Matched database administration pattern.')
|
||||
if _match(text, r'\b(audiobookshelf|audiodeck|bazarr|seerr|komga|lidarr|mylar3?|immich|peertube|romm|tdarr|grimmory|listenarr|readmeabook)\b'):
|
||||
app_type = next((name for name in ['audiobookshelf', 'audiodeck', 'bazarr', 'seerr', 'komga', 'lidarr', 'mylar3', 'immich', 'peertube', 'romm', 'tdarr', 'grimmory', 'listenarr', 'readmeabook'] if name.replace('_', '-') in text or name in text), 'media_app')
|
||||
protected = app_type == 'immich'
|
||||
return apply(app_type=app_type, category='media', criticality='high' if protected else 'medium', protected=protected, privacy_sensitive=protected, policy_profile='media_app', reason='Matched media library or companion app pattern.')
|
||||
if _match(text, r'\b(paperless-ngx|paperless_ngx|stirlingpdf|stirling-pdf|leafwiki|infrabinder-local|infrabinder_local)\b'):
|
||||
app_type = next((name for name in ['paperless_ngx', 'stirlingpdf', 'leafwiki', 'infrabinder_local'] if name.replace('_', '-') in text or name in text), 'document_service')
|
||||
protected = app_type in {'paperless_ngx', 'infrabinder_local'}
|
||||
return apply(app_type=app_type, category='productivity', subcategory='documents', criticality='high' if protected else 'medium', protected=protected, privacy_sensitive=True, policy_profile='document_service', reason='Matched document or knowledge-management service pattern.')
|
||||
if _match(text, r'\b(jdownloader2?|metube|slskd|unpackerr)\b'):
|
||||
app_type = next((name for name in ['jdownloader2', 'metube', 'slskd', 'unpackerr'] if name in text), 'download_tool')
|
||||
return apply(app_type=app_type, category='downloads', criticality='medium', policy_profile='download_client', reason='Matched download workflow pattern.')
|
||||
if _match(text, r'\b(ollama|anythingllm|litellm|chromadb|hivekeep|odysseus-aio|odysseus_aio)\b'):
|
||||
app_type = next((name for name in ['ollama', 'anythingllm', 'litellm', 'chromadb', 'hivekeep', 'odysseus_aio'] if name.replace('_', '-') in text or name in text), 'ai_service')
|
||||
category = 'database' if app_type == 'chromadb' else 'infrastructure'
|
||||
protected = app_type in {'chromadb', 'hivekeep', 'odysseus_aio'}
|
||||
return apply(app_type=app_type, category=category, criticality='high' if protected else 'medium', protected=protected, privacy_sensitive=protected, policy_profile='database' if app_type == 'chromadb' else 'ai_service', reason='Matched local AI platform pattern.')
|
||||
if _match(text, r'\b(searxng|flaresolverr|postfix-relay|postfix_relay)\b'):
|
||||
app_type = next((name for name in ['searxng', 'flaresolverr', 'postfix_relay'] if name.replace('_', '-') in text or name in text), 'network_service')
|
||||
return apply(app_type=app_type, category='network', criticality='medium', protected=app_type == 'postfix_relay', policy_profile='network_service', reason='Matched network support service pattern.')
|
||||
if _match(text, r'\b(unraid-appops-gateway|unraid_appops_gateway|watchtower)\b'):
|
||||
app_type = 'unraid_appops_gateway' if 'appops' in text else 'watchtower'
|
||||
return apply(app_type=app_type, category='infrastructure', criticality='critical' if app_type == 'unraid_appops_gateway' else 'high', protected=True, policy_profile='protected_service', reason='Matched infrastructure control-plane service pattern.')
|
||||
if _match(text, r'\b(organizrv?2?|organizr|unraid-progressive-web-app|unraid_progressive_web_app|ntfy)\b'):
|
||||
app_type = next((name for name in ['organizrv2', 'unraid_progressive_web_app', 'ntfy'] if name.replace('_', '-') in text or name in text), 'infrastructure_service')
|
||||
return apply(app_type=app_type, category='infrastructure', criticality='medium', policy_profile='infrastructure_service', reason='Matched self-hosted infrastructure support service pattern.')
|
||||
if _match(text, r'\b(retroarch|rickroll)\b'):
|
||||
app_type = 'retroarch' if 'retroarch' in text else 'rickroll'
|
||||
return apply(app_type=app_type, category='media', criticality='low', policy_profile='media_app', reason='Matched media or entertainment service pattern.')
|
||||
if _match(text, r'\bdockdeck\b', r'dockdeck:local'):
|
||||
return apply(app_type='dockdeck', category='infrastructure', subcategory='service_portal', criticality='medium', privacy_sensitive=True, policy_profile='internal_service', reason='Matched the local DockDeck self-hosted start dashboard pattern.', default_port=1218)
|
||||
if _match(text, r'\blaunchpad\b', r'launchpad:local'):
|
||||
return apply(app_type='launchpad', category='infrastructure', subcategory='service_portal', criticality='medium', policy_profile='internal_service', reason='Matched the local Launchpad service portal pattern.')
|
||||
if _match(text, r'\bmedia-codec-dashboard\b', r'\bmedia_codec_dashboard\b'):
|
||||
return apply(app_type='media_codec_dashboard', category='media', subcategory='media_observability', criticality='medium', privacy_sensitive=True, policy_profile='media_app', reason='Matched the local media codec observability dashboard pattern.')
|
||||
if _match(text, r'\bgeointel\b'):
|
||||
return apply(app_type='geointel', category='ai', subcategory='geospatial_workbench', criticality='medium', protected=True, privacy_sensitive=True, policy_profile='ai_service', reason='Matched the local GeoIntel GeoAI workbench pattern.')
|
||||
if _match(text, r'\bopscenter-it-tools\b', r'\bopscenter_it_tools\b'):
|
||||
return apply(app_type='opscenter_it_tools', category='dev', subcategory='operations_tooling', criticality='high', protected=True, privacy_sensitive=True, policy_profile='dev_tool', reason='Matched the local OpsCenter diagnostics workbench pattern.')
|
||||
if _match(text, r'\bprojectbrain\b'):
|
||||
return apply(app_type='projectbrain', category='productivity', subcategory='project_operations', criticality='high', protected=True, privacy_sensitive=True, policy_profile='document_service', reason='Matched the local ProjectBrain project operating system pattern.')
|
||||
if _match(text, r'\bpsforge\b'):
|
||||
return apply(app_type='psforge', category='productivity', subcategory='project_planning', criticality='medium', protected=True, privacy_sensitive=True, policy_profile='document_service', reason='Matched the local PSForge planning and documentation workbench pattern.')
|
||||
if _match(text, r'\bquizmonitor\b', r'\bquiz-monitor\b'):
|
||||
return apply(app_type='quizmonitor', category='ai', subcategory='knowledge_assistant', criticality='high', protected=True, privacy_sensitive=True, policy_profile='ai_service', reason='Matched the local QuizMonitor AI knowledge platform pattern.')
|
||||
if _match(text, r'\bspoortracker\b'):
|
||||
return apply(app_type='spoortracker', category='monitoring', subcategory='transport_monitoring', criticality='low', policy_profile='monitoring', reason='Matched the local SpoorTracker live transport monitoring pattern.')
|
||||
if _match(text, r'\bcertscope\b'):
|
||||
return apply(app_type='certscope', category='monitoring', subcategory='domain_inspection', criticality='low', policy_profile='monitoring', reason='Matched the local CertScope domain posture inspector pattern.')
|
||||
if _match(text, r'\bps4-http\b', r'\bps4_http\b'):
|
||||
return apply(app_type='ps4_http', category='infrastructure', subcategory='http_service', criticality='low', policy_profile='infrastructure_service', reason='Matched the local PS4 HTTP support service pattern.')
|
||||
if _match(text, r'\b(zabbix|grafana|uptime-kuma|glances|netdata|netronome|unraid-stats|unraid_stats|system-monitor|system_monitor|gpu-hot|gpu_hot)\b'):
|
||||
app_type = next((name for name in ['zabbix', 'grafana', 'uptime-kuma', 'glances', 'netdata', 'netronome', 'unraid_stats', 'system_monitor', 'gpu_hot'] if name.replace('_', '-') in text or name in text), 'monitoring')
|
||||
return apply(app_type=app_type.replace('-', '_'), category='monitoring', criticality='medium', policy_profile='monitoring', reason='Matched monitoring pattern.')
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..db.models import App
|
||||
|
||||
|
||||
_APPOPS_ROLLBACK_ID = re.compile(
|
||||
r'^unraid_appops_gateway(?:_1216)?_pre_\d{8}t\d{6}z$',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def is_appops_deployment_container_name(value: str | None) -> bool:
|
||||
key = str(value or '').strip().lower().replace('_', '-')
|
||||
return key in {'unraid-appops-gateway', 'unraid-appops-gateway-1216'} or bool(
|
||||
re.fullmatch(r'unraid-appops-gateway(?:-1216)?-pre-\d{8}t\d{6}z', key)
|
||||
)
|
||||
|
||||
|
||||
def is_historical_app_record(app: App) -> bool:
|
||||
"""Keep deployment history out of active product views without deleting it."""
|
||||
return bool(_APPOPS_ROLLBACK_ID.fullmatch(str(app.id or '').strip()))
|
||||
|
||||
|
||||
def category_label(value: str | None) -> str:
|
||||
words = str(value or 'Unknown').strip().replace('_', ' ').replace('-', ' ').title()
|
||||
replacements = {'Ai': 'AI', 'Api': 'API', 'It': 'IT'}
|
||||
return ' '.join(replacements.get(word, word) for word in words.split()) or 'Unknown'
|
||||
|
||||
|
||||
def list_active_apps(session: Session, *, order_by: str = 'name') -> list[App]:
|
||||
apps = [app for app in session.exec(select(App)).all() if not is_historical_app_record(app)]
|
||||
|
||||
# A port-specific canonical record supersedes the old unqualified 1211 record.
|
||||
if any(app.id == 'unraid_appops_gateway_1216' for app in apps):
|
||||
apps = [app for app in apps if app.id != 'unraid_appops_gateway']
|
||||
|
||||
key = (lambda app: app.id.lower()) if order_by == 'id' else (lambda app: app.name.lower())
|
||||
return sorted(apps, key=key)
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..db.models import Approval
|
||||
from ..core.time import utc_now
|
||||
|
||||
|
||||
VALID_DECISIONS = {'approved', 'rejected'}
|
||||
|
||||
|
||||
class ApprovalStateError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def approval_fingerprint(action: str, target: str, source: str, risk: str, context: dict | None = None) -> str:
|
||||
canonical_context = json.dumps(context or {}, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
canonical = "\x1f".join([*(part.strip() for part in (action, target, source, risk)), canonical_context])
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def get_approval(session: Session, approval_id: str) -> Approval | None:
|
||||
approval = session.get(Approval, approval_id)
|
||||
if approval and not approval.action_fingerprint:
|
||||
approval.action_fingerprint = approval_fingerprint(approval.action, approval.target, approval.source, approval.risk, approval.request_context_json)
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(approval)
|
||||
if approval and _expire_if_needed(session, approval):
|
||||
session.refresh(approval)
|
||||
return approval
|
||||
|
||||
|
||||
def list_approvals(session: Session, *, include_dismissed: bool = False):
|
||||
statement = select(Approval).order_by(Approval.created_at.desc())
|
||||
if not include_dismissed:
|
||||
statement = statement.where(Approval.dismissed_at.is_(None))
|
||||
approvals = session.exec(statement).all()
|
||||
changed = any(_expire_if_needed(session, approval, commit=False) for approval in approvals)
|
||||
if changed:
|
||||
session.commit()
|
||||
for approval in approvals:
|
||||
session.refresh(approval)
|
||||
return approvals
|
||||
|
||||
|
||||
def dismiss_expired_approvals(session: Session) -> list[Approval]:
|
||||
"""Archive only terminal expired approvals while retaining their audit evidence."""
|
||||
approvals = list_approvals(session)
|
||||
dismissed_at = utc_now()
|
||||
dismissed = [approval for approval in approvals if approval.status == 'expired']
|
||||
for approval in dismissed:
|
||||
approval.dismissed_at = dismissed_at
|
||||
session.add(approval)
|
||||
if dismissed:
|
||||
session.commit()
|
||||
for approval in dismissed:
|
||||
session.refresh(approval)
|
||||
return dismissed
|
||||
|
||||
|
||||
def _expire_if_needed(session: Session, approval: Approval, *, commit: bool = True) -> bool:
|
||||
if approval.status != 'pending' or approval.expires_at is None or approval.expires_at > utc_now():
|
||||
return False
|
||||
approval.status = 'expired'
|
||||
approval.decided_at = utc_now()
|
||||
session.add(approval)
|
||||
if commit:
|
||||
session.commit()
|
||||
return True
|
||||
|
||||
|
||||
def create_approval(
|
||||
session: Session,
|
||||
*,
|
||||
action: str,
|
||||
target: str,
|
||||
source: str,
|
||||
risk: str,
|
||||
reason: str | None = None,
|
||||
requester: str = 'manual',
|
||||
expires_in_minutes: int = 30,
|
||||
context: dict | None = None,
|
||||
) -> Approval:
|
||||
approval = Approval(
|
||||
id=f'appr-{uuid.uuid4().hex[:12]}',
|
||||
action=action,
|
||||
target=target,
|
||||
source=source,
|
||||
risk=risk,
|
||||
status='pending',
|
||||
requester=requester,
|
||||
reason=reason,
|
||||
expires_at=(utc_now() + timedelta(minutes=expires_in_minutes)),
|
||||
request_context_json=context or {},
|
||||
action_fingerprint=approval_fingerprint(action, target, source, risk, context),
|
||||
)
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(approval)
|
||||
return approval
|
||||
|
||||
|
||||
def update_approval_status(session: Session, approval_id: str, status: str, *, actor: str = "operator") -> Approval | None:
|
||||
approval = get_approval(session, approval_id)
|
||||
if not approval:
|
||||
return None
|
||||
|
||||
if status not in VALID_DECISIONS:
|
||||
raise ApprovalStateError('Approval status must be approved or rejected.')
|
||||
if approval.status == 'expired':
|
||||
raise ApprovalStateError('Approval has expired and can no longer be decided.')
|
||||
if approval.status != 'pending':
|
||||
raise ApprovalStateError(f'Approval is already {approval.status}.')
|
||||
|
||||
approval.status = status
|
||||
approval.decided_at = utc_now()
|
||||
approval.approved_by = actor if status == 'approved' else None
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(approval)
|
||||
return approval
|
||||
|
||||
|
||||
def mark_approval_executed(
|
||||
session: Session,
|
||||
approval: Approval,
|
||||
*,
|
||||
job_id: str,
|
||||
audit_event_id: str,
|
||||
actor: str = "operator",
|
||||
) -> Approval:
|
||||
if approval.status != 'approved':
|
||||
raise ApprovalStateError('Only an approved request can be executed.')
|
||||
expected = approval_fingerprint(approval.action, approval.target, approval.source, approval.risk, approval.request_context_json)
|
||||
if not approval.action_fingerprint or approval.action_fingerprint != expected:
|
||||
raise ApprovalStateError('Approval action context changed after review.')
|
||||
approval.status = 'executed'
|
||||
approval.executed_at = utc_now()
|
||||
approval.job_id = job_id
|
||||
approval.audit_event_id = audit_event_id
|
||||
approval.executed_by = actor
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(approval)
|
||||
return approval
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..db.models import AuditEvent
|
||||
from ..core.time import utc_now
|
||||
from ..core.redaction import redact_message, redact_value
|
||||
|
||||
|
||||
def _redact_mapping(value: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
return redact_value(value) if value else None
|
||||
|
||||
|
||||
def log_event(
|
||||
session: Session,
|
||||
*,
|
||||
source: str,
|
||||
identity: str,
|
||||
action_id: str,
|
||||
target: str,
|
||||
risk: str,
|
||||
policy_decision: str,
|
||||
reason: str,
|
||||
result: str,
|
||||
target_type: str = 'app',
|
||||
execution_plan: dict[str, Any] | None = None,
|
||||
before_state: dict[str, Any] | None = None,
|
||||
after_state: dict[str, Any] | None = None,
|
||||
post_checks: dict[str, Any] | None = None,
|
||||
redacted_metadata: dict[str, Any] | None = None,
|
||||
) -> AuditEvent:
|
||||
event = AuditEvent(
|
||||
id=f'audit-{uuid.uuid4().hex[:12]}',
|
||||
timestamp=utc_now(),
|
||||
source=source,
|
||||
identity=identity,
|
||||
action_id=redact_message(action_id),
|
||||
target=redact_message(target),
|
||||
target_type=redact_message(target_type),
|
||||
risk=risk,
|
||||
policy_decision=policy_decision,
|
||||
reason=redact_message(reason),
|
||||
execution_plan=_redact_mapping(execution_plan) or {},
|
||||
result=result,
|
||||
before_state=_redact_mapping(before_state),
|
||||
after_state=_redact_mapping(after_state),
|
||||
post_checks=_redact_mapping(post_checks),
|
||||
redacted_metadata=_redact_mapping(redacted_metadata),
|
||||
)
|
||||
|
||||
session.add(event)
|
||||
session.commit()
|
||||
session.refresh(event)
|
||||
return event
|
||||
|
||||
|
||||
def list_events(session: Session, *, limit: int = 200, offset: int = 0):
|
||||
statement = (
|
||||
select(AuditEvent)
|
||||
.order_by(AuditEvent.timestamp.desc(), AuditEvent.id.desc())
|
||||
.offset(max(0, offset))
|
||||
.limit(max(1, min(limit, 500)))
|
||||
)
|
||||
return session.exec(statement).all()
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..connectors.authentik import AuthentikConnector
|
||||
from ..connectors.deluge import DelugeConnector
|
||||
from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from ..connectors.plex import PlexConnector
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..connectors.prowlarr import ProwlarrConnector
|
||||
from ..connectors.radarr import RadarrConnector
|
||||
from ..connectors.sonarr import SonarrConnector
|
||||
from ..connectors.tautulli import TautulliConnector
|
||||
from ..connectors.unraid import UnraidConnector
|
||||
from ..connectors.widefrog import WidefrogConnector
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import App, Connector, NotificationChannel
|
||||
from . import notification_service, safety_center_service
|
||||
from .setup_service import apply_stored_connector_settings
|
||||
|
||||
|
||||
_TESTERS: dict[str, Callable[[], Any]] = {
|
||||
'authentik': AuthentikConnector,
|
||||
'deluge': DelugeConnector,
|
||||
'nginx_proxy_manager': NginxProxyManagerConnector,
|
||||
'plex': PlexConnector,
|
||||
'portainer': PortainerConnector,
|
||||
'prowlarr': ProwlarrConnector,
|
||||
'radarr': RadarrConnector,
|
||||
'sonarr': SonarrConnector,
|
||||
'tautulli': TautulliConnector,
|
||||
'unraid': UnraidConnector,
|
||||
'widefrog': WidefrogConnector,
|
||||
}
|
||||
|
||||
|
||||
def _stored_status(status: str) -> str:
|
||||
return 'error' if status == 'unreachable' else status
|
||||
|
||||
|
||||
def test_and_persist_connector(session: Session, connector_id: str) -> dict[str, Any]:
|
||||
connector = session.get(Connector, connector_id)
|
||||
if not connector:
|
||||
return {'status': 'disconnected', 'reason': 'connector_not_found'}
|
||||
|
||||
apply_stored_connector_settings(session, {connector_id})
|
||||
if connector_id == 'home_assistant':
|
||||
channel = session.get(NotificationChannel, 'home_assistant')
|
||||
result = (
|
||||
notification_service.test_channel_connection(session, 'home_assistant')
|
||||
if channel
|
||||
else {'status': 'missing_config', 'reason': 'notification_channel_not_found'}
|
||||
)
|
||||
else:
|
||||
factory = _TESTERS.get(connector_id)
|
||||
result = factory().test_connection() if factory else {'status': 'error', 'reason': 'test_not_implemented'}
|
||||
|
||||
status = str(result.get('status') or 'disconnected')
|
||||
persisted_status = _stored_status(status)
|
||||
connector.status = persisted_status
|
||||
connector.last_test = utc_now().isoformat()
|
||||
session.add(connector)
|
||||
|
||||
app = session.get(App, connector_id)
|
||||
if app and app.connector_status != persisted_status:
|
||||
app.connector_status = persisted_status
|
||||
session.add(app)
|
||||
|
||||
session.commit()
|
||||
safety_center_service.invalidate_cache()
|
||||
return {**result, 'status': status}
|
||||
|
||||
|
||||
def refresh_connector_statuses(session: Session) -> dict[str, Any]:
|
||||
connector_ids = list(session.exec(select(Connector.id).order_by(Connector.id)).all())
|
||||
results: dict[str, str] = {}
|
||||
for connector_id in connector_ids:
|
||||
if connector_id not in _TESTERS and connector_id != 'home_assistant':
|
||||
continue
|
||||
result = test_and_persist_connector(session, connector_id)
|
||||
results[connector_id] = str(result.get('status') or 'disconnected')
|
||||
return {
|
||||
'success': True,
|
||||
'status': 'ok',
|
||||
'tested': len(results),
|
||||
'connected': sum(status == 'connected' for status in results.values()),
|
||||
'results': results,
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from ..db.models import JobRun
|
||||
from ..services.approval_service import create_approval
|
||||
from ..services.audit_logger import log_event
|
||||
from ..services.job_runner import complete_job, create_job, start_job
|
||||
from ..services.policy_engine import evaluate_policy
|
||||
|
||||
|
||||
def _widefrog_health_endpoint() -> tuple[str | None, str | None]:
|
||||
raw_url = str(settings.apis.get("WIDEFROG_URL") or "").strip()
|
||||
if not raw_url:
|
||||
return None, "missing_config"
|
||||
|
||||
candidate = (
|
||||
raw_url if raw_url.startswith(("http://", "https://")) else f"http://{raw_url}"
|
||||
)
|
||||
try:
|
||||
parsed = urlparse(candidate)
|
||||
except Exception:
|
||||
return None, "invalid_widefrog_url"
|
||||
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
return None, "invalid_widefrog_url"
|
||||
|
||||
port = parsed.port or 1207
|
||||
return f"{parsed.scheme}://{host}:{port}/", None
|
||||
|
||||
|
||||
def _widefrog_health_check() -> dict[str, Any]:
|
||||
health_url, reason = _widefrog_health_endpoint()
|
||||
if not health_url:
|
||||
return {"status": "error", "reason": reason or "missing_config"}
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=4.0, verify=settings.verify_tls) as client:
|
||||
response = client.get(health_url)
|
||||
except (httpx.RequestError, ValueError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if response.status_code >= 500:
|
||||
return {
|
||||
"status": "error",
|
||||
"reason": f"widefrog health check failed: HTTP {response.status_code}",
|
||||
}
|
||||
if response.status_code >= 400:
|
||||
return {
|
||||
"status": "warning",
|
||||
"reason": f"widefrog health returned HTTP {response.status_code}",
|
||||
}
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def _format_success_result(
|
||||
summary: str,
|
||||
*,
|
||||
steps: list[str] | None = None,
|
||||
before_state: dict[str, Any] | None = None,
|
||||
after_state: dict[str, Any] | None = None,
|
||||
post_checks: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"success": True,
|
||||
"decision": "allowed",
|
||||
"status": "success",
|
||||
"policy_decision": "allowed",
|
||||
"result": "success",
|
||||
"message": summary,
|
||||
"plan": steps or [],
|
||||
"steps": steps or [],
|
||||
"before_state": before_state or {},
|
||||
"after_state": after_state or {},
|
||||
"post_checks": post_checks or [],
|
||||
}
|
||||
|
||||
|
||||
def _format_blocked_result(audit_id: str, reason: str) -> dict[str, Any]:
|
||||
return {
|
||||
"success": False,
|
||||
"decision": "blocked",
|
||||
"status": "failed",
|
||||
"policy_decision": "blocked",
|
||||
"result": "blocked",
|
||||
"message": reason,
|
||||
"auditId": audit_id,
|
||||
}
|
||||
|
||||
|
||||
def _format_approval_result(
|
||||
audit_id: str, approval_id: str, reason: str
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"success": False,
|
||||
"decision": "approval_required",
|
||||
"status": "failed",
|
||||
"policy_decision": "approval_required",
|
||||
"result": "approval_required",
|
||||
"message": reason,
|
||||
"approvalId": approval_id,
|
||||
"auditId": audit_id,
|
||||
}
|
||||
|
||||
|
||||
def execute_container_restart(
|
||||
*,
|
||||
session,
|
||||
action: str,
|
||||
target: str,
|
||||
source: str = "manual",
|
||||
risk: str = "medium",
|
||||
steps: list[str] | None = None,
|
||||
approved: bool = False,
|
||||
policy_override: dict[str, Any] | None = None,
|
||||
):
|
||||
plan_steps = steps or [
|
||||
"Resolve container",
|
||||
"Restart container",
|
||||
"Post-check health",
|
||||
]
|
||||
if policy_override is not None:
|
||||
decision = {
|
||||
"decision": str(policy_override.get("decision") or "blocked"),
|
||||
"reason": str(policy_override.get("reason") or "Policy override."),
|
||||
"risk": str(policy_override.get("risk") or risk),
|
||||
}
|
||||
elif approved:
|
||||
decision = {
|
||||
"decision": "allowed",
|
||||
"reason": "Exact action context was approved by an AppOps operator.",
|
||||
"risk": risk,
|
||||
}
|
||||
else:
|
||||
decision = evaluate_policy(session, action=action, target=target, source=source)
|
||||
|
||||
if decision["decision"] == "blocked":
|
||||
audit = log_event(
|
||||
session,
|
||||
source="action",
|
||||
identity=source,
|
||||
action_id=action,
|
||||
target=target,
|
||||
risk=risk,
|
||||
policy_decision=decision["decision"],
|
||||
reason=decision["reason"],
|
||||
result="blocked",
|
||||
execution_plan={"plan": plan_steps, "policyReason": decision["reason"]},
|
||||
)
|
||||
return _format_blocked_result(audit.id, decision["reason"])
|
||||
|
||||
if decision["decision"] == "approval_required":
|
||||
approval = create_approval(
|
||||
session,
|
||||
action=action,
|
||||
target=target,
|
||||
source=source,
|
||||
risk=risk,
|
||||
reason=decision["reason"],
|
||||
requester=source,
|
||||
)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="action",
|
||||
identity=source,
|
||||
action_id=action,
|
||||
target=target,
|
||||
risk=risk,
|
||||
policy_decision=decision["decision"],
|
||||
reason=decision["reason"],
|
||||
result="approval_requested",
|
||||
execution_plan={"plan": plan_steps, "approvalId": approval.id},
|
||||
redacted_metadata={"approval_id": approval.id},
|
||||
)
|
||||
return _format_approval_result(
|
||||
audit.id, approval.id, "Action requires approval before execution."
|
||||
)
|
||||
|
||||
# Allowed restart flow
|
||||
job: JobRun = create_job(
|
||||
session,
|
||||
action_id=action,
|
||||
target=target,
|
||||
source=source,
|
||||
risk=risk,
|
||||
policy_decision=decision["decision"],
|
||||
steps=plan_steps,
|
||||
)
|
||||
start_job(session, job)
|
||||
|
||||
try:
|
||||
connector = PortainerConnector()
|
||||
|
||||
precheck = connector.get_container_status(target)
|
||||
if precheck.get("status") != "ok":
|
||||
# keep execution context in logs while still allowing best-effort continuation
|
||||
precheck = {
|
||||
"status": "warn",
|
||||
"container": {"name": target},
|
||||
"reason": precheck.get("reason"),
|
||||
}
|
||||
|
||||
restart_result = connector.restart_container(target)
|
||||
if restart_result.get("status") != "ok":
|
||||
raise RuntimeError(
|
||||
str(restart_result.get("reason", "container_restart_failed"))
|
||||
)
|
||||
|
||||
post_check = {"container": restart_result.get("container", target)}
|
||||
if target == "widefrog":
|
||||
health = _widefrog_health_check()
|
||||
post_checks = [
|
||||
f"widefrog health: {health.get('status')}",
|
||||
f"widefrog reason: {health.get('reason', '')}".strip(),
|
||||
]
|
||||
post_check["widefrog_health"] = health
|
||||
else:
|
||||
post_checks = []
|
||||
health = {}
|
||||
|
||||
completed = complete_job(
|
||||
session,
|
||||
job,
|
||||
status="succeeded",
|
||||
result_summary="Container restart executed through Portainer connector.",
|
||||
)
|
||||
|
||||
audit = log_event(
|
||||
session,
|
||||
source="action",
|
||||
identity=source,
|
||||
action_id=action,
|
||||
target=target,
|
||||
risk=risk,
|
||||
policy_decision=decision["decision"],
|
||||
reason=decision["reason"],
|
||||
result="success",
|
||||
execution_plan={
|
||||
"plan": plan_steps,
|
||||
"jobId": completed.id,
|
||||
"precheck": precheck,
|
||||
},
|
||||
before_state=precheck,
|
||||
after_state=post_check,
|
||||
post_checks=health if target == "widefrog" else None,
|
||||
)
|
||||
|
||||
result = _format_success_result(
|
||||
"Container restart executed.",
|
||||
steps=plan_steps,
|
||||
before_state=precheck,
|
||||
after_state=post_check,
|
||||
post_checks=post_checks,
|
||||
)
|
||||
result["jobId"] = completed.id
|
||||
result["auditId"] = audit.id
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
failed = complete_job(
|
||||
session,
|
||||
job,
|
||||
status="failed",
|
||||
result_summary=f"Container restart failed: {redact_message(str(exc))}",
|
||||
)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="action",
|
||||
identity=source,
|
||||
action_id=action,
|
||||
target=target,
|
||||
risk=risk,
|
||||
policy_decision=decision["decision"],
|
||||
reason="Container restart execution failed.",
|
||||
result="failed",
|
||||
execution_plan={"plan": plan_steps, "jobId": failed.id},
|
||||
after_state={"status": "failed", "reason": redact_message(str(exc))},
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"decision": "failed",
|
||||
"status": "failed",
|
||||
"policy_decision": decision["decision"],
|
||||
"result": "failed",
|
||||
"message": redact_message(str(exc)),
|
||||
"jobId": failed.id,
|
||||
"auditId": audit.id,
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
|
||||
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
|
||||
GITEA_OWNER = "Jens"
|
||||
GITEA_REPOSITORY = "Unraid-AppOps-Gateway"
|
||||
REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"
|
||||
HELPER_NAME = "itworx-controller-artifact-promoter"
|
||||
CONTROLLER_CONTAINER = "gitea-deploy-control"
|
||||
CONTROLLER_VERSION = "2.1.5"
|
||||
HELPER_LABEL = "itworx.controller.promoter"
|
||||
_HELPER_TIMEOUT_SECONDS = 240
|
||||
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$")
|
||||
|
||||
|
||||
class ControllerArtifactPromotionError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]:
|
||||
if not _REVISION.fullmatch(revision):
|
||||
raise ValueError("controller_promoter_revision_invalid")
|
||||
if not _IMAGE_ID.fullmatch(image_id):
|
||||
raise ValueError("controller_promoter_image_invalid")
|
||||
|
||||
command = f"""set -Eeuo pipefail
|
||||
revision='{revision}'
|
||||
version='{CONTROLLER_VERSION}'
|
||||
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
|
||||
work='/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{revision[:12]}'
|
||||
rm -rf -- "$work"
|
||||
install -d -m 0700 "$work"
|
||||
cleanup() {{ rm -rf -- "$work"; }}
|
||||
trap cleanup EXIT
|
||||
command -v git >/dev/null
|
||||
command -v docker >/dev/null
|
||||
docker compose version >/dev/null
|
||||
command -v python3 >/dev/null
|
||||
command -v tar >/dev/null
|
||||
command -v curl >/dev/null
|
||||
git --git-dir="$repo" cat-file -e "$revision^{{commit}}"
|
||||
tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")"
|
||||
git --git-dir="$repo" archive --format=tar "$revision" \
|
||||
ops/gitea-deploy-broker \
|
||||
scripts/prepare_controller_artifact.py \
|
||||
scripts/reconcile_controller_state.py \
|
||||
scripts/run_repopulse_adoption.py > "$work/controller.tar"
|
||||
archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)"
|
||||
git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh"
|
||||
chmod 0700 "$work/upgrade.sh"
|
||||
bash "$work/upgrade.sh" \
|
||||
--archive "$work/controller.tar" \
|
||||
--archive-sha256 "$archive_sha" \
|
||||
--source-repository '{SOURCE_REPOSITORY}' \
|
||||
--source-revision "$revision" \
|
||||
--source-tree "$tree" \
|
||||
--controller-version "$version" \
|
||||
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
|
||||
"""
|
||||
|
||||
return {
|
||||
"Image": image_id,
|
||||
"User": "0:0",
|
||||
"Cmd": ["/bin/bash", "-lc", command],
|
||||
"Labels": {
|
||||
HELPER_LABEL: "true",
|
||||
"itworx.source.repository": SOURCE_REPOSITORY,
|
||||
"itworx.source.revision": revision,
|
||||
"itworx.controller.version": CONTROLLER_VERSION,
|
||||
},
|
||||
"HostConfig": {
|
||||
"Binds": [
|
||||
"/mnt/cache:/mnt/cache",
|
||||
"/mnt/user:/mnt/user",
|
||||
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
|
||||
"/var/run/docker.sock:/var/run/docker.sock",
|
||||
],
|
||||
"NetworkMode": "host",
|
||||
"Privileged": False,
|
||||
"SecurityOpt": ["no-new-privileges:true"],
|
||||
"RestartPolicy": {"Name": "no"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ControllerArtifactPromoter:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
portainer: PortainerConnector | None = None,
|
||||
broker: DeploymentBrokerClient | None = None,
|
||||
) -> None:
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
self.broker = broker or DeploymentBrokerClient()
|
||||
|
||||
@staticmethod
|
||||
def _gitea_base_url() -> str:
|
||||
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_gitea_url_invalid")
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", ""))
|
||||
|
||||
@staticmethod
|
||||
def _gitea_token() -> str:
|
||||
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "").strip()
|
||||
if not token:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_gitea_token_missing")
|
||||
return token
|
||||
|
||||
@staticmethod
|
||||
def _status_key(item: dict[str, Any]) -> tuple[datetime, int] | None:
|
||||
raw = str(item.get("updated_at") or item.get("created_at") or "")
|
||||
if not _TIMESTAMP.fullmatch(raw):
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
identifier = item.get("id")
|
||||
return parsed, identifier if isinstance(identifier, int) and not isinstance(identifier, bool) else -1
|
||||
|
||||
def _validated_main_revision(self) -> str:
|
||||
branch = "main"
|
||||
base = self._gitea_base_url()
|
||||
headers = {
|
||||
"Authorization": f"token {self._gitea_token()}",
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "itworx-controller-artifact-promoter/1",
|
||||
}
|
||||
timeout = max(2.0, min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)))
|
||||
verify = bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls))
|
||||
owner = quote(GITEA_OWNER, safe="")
|
||||
repository = quote(GITEA_REPOSITORY, safe="")
|
||||
try:
|
||||
with httpx.Client(timeout=timeout, verify=verify, follow_redirects=False) as client:
|
||||
branch_response = client.get(
|
||||
f"{base}/api/v1/repos/{owner}/{repository}/branches/{branch}",
|
||||
headers=headers,
|
||||
)
|
||||
if branch_response.status_code != 200:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_main_unavailable")
|
||||
branch_payload = branch_response.json()
|
||||
commit = branch_payload.get("commit") if isinstance(branch_payload, dict) else None
|
||||
revision = str(commit.get("id") or "") if isinstance(commit, dict) else ""
|
||||
if (
|
||||
not isinstance(branch_payload, dict)
|
||||
or branch_payload.get("name") != branch
|
||||
or branch_payload.get("protected") is not True
|
||||
or not _REVISION.fullmatch(revision)
|
||||
):
|
||||
raise ControllerArtifactPromotionError("controller_promoter_main_not_protected")
|
||||
|
||||
status_response = client.get(
|
||||
f"{base}/api/v1/repos/{owner}/{repository}/statuses/{revision}",
|
||||
params={"limit": 100, "page": 1},
|
||||
headers=headers,
|
||||
)
|
||||
if status_response.status_code != 200:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_status_unavailable")
|
||||
statuses = status_response.json()
|
||||
except ControllerArtifactPromotionError:
|
||||
raise
|
||||
except (httpx.RequestError, ValueError) as exc:
|
||||
raise ControllerArtifactPromotionError(
|
||||
"controller_promoter_gitea_unavailable"
|
||||
) from exc
|
||||
|
||||
if not isinstance(statuses, list) or len(statuses) > 100:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_status_invalid")
|
||||
latest: tuple[tuple[datetime, int], str] | None = None
|
||||
for item in statuses:
|
||||
if not isinstance(item, dict) or item.get("context") != REQUIRED_PUSH_CONTEXT:
|
||||
continue
|
||||
state = str(item.get("status") or "")
|
||||
if state not in {"pending", "success", "failure", "error", "warning", "neutral", "cancelled"}:
|
||||
continue
|
||||
key = self._status_key(item)
|
||||
if key is not None and (latest is None or key > latest[0]):
|
||||
latest = (key, state)
|
||||
if latest is None or latest[1] != "success":
|
||||
raise ControllerArtifactPromotionError("controller_promoter_main_not_validated")
|
||||
return revision
|
||||
|
||||
def _portainer_endpoint(self) -> str:
|
||||
config = self.portainer._config()
|
||||
if config is None or not config.endpoint_id:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_portainer_unavailable")
|
||||
return config.endpoint_id
|
||||
|
||||
def _controller_image_id(self) -> str:
|
||||
listed = self.portainer.list_containers()
|
||||
containers = listed.get("containers") if isinstance(listed, dict) else None
|
||||
candidate = next(
|
||||
(
|
||||
item
|
||||
for item in containers or []
|
||||
if isinstance(item, dict) and item.get("name") == CONTROLLER_CONTAINER
|
||||
),
|
||||
None,
|
||||
)
|
||||
if candidate is None or candidate.get("state") != "running":
|
||||
raise ControllerArtifactPromotionError("controller_promoter_controller_unavailable")
|
||||
identifier = str(candidate.get("id") or "")
|
||||
endpoint = self._portainer_endpoint()
|
||||
try:
|
||||
response = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{quote(identifier, safe='')}/json",
|
||||
)
|
||||
payload = response.json()
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise ControllerArtifactPromotionError(
|
||||
"controller_promoter_controller_inspect_failed"
|
||||
) from exc
|
||||
image_id = str(payload.get("Image") or "") if isinstance(payload, dict) else ""
|
||||
if not _IMAGE_ID.fullmatch(image_id):
|
||||
raise ControllerArtifactPromotionError("controller_promoter_controller_image_unpinned")
|
||||
return image_id
|
||||
|
||||
def _existing_helper(self) -> dict[str, Any] | None:
|
||||
listed = self.portainer.list_containers()
|
||||
containers = listed.get("containers") if isinstance(listed, dict) else None
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in containers or []
|
||||
if isinstance(item, dict) and item.get("name") == HELPER_NAME
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def _remove_helper(self, identifier: str) -> None:
|
||||
endpoint = self._portainer_endpoint()
|
||||
self.portainer._request(
|
||||
"DELETE",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{quote(identifier, safe='')}",
|
||||
params={"v": "1", "force": "0"},
|
||||
)
|
||||
|
||||
def _run_helper(self, *, revision: str, image_id: str) -> dict[str, Any]:
|
||||
endpoint = self._portainer_endpoint()
|
||||
existing = self._existing_helper()
|
||||
if existing is not None:
|
||||
identifier = str(existing.get("id") or "")
|
||||
if existing.get("state") == "running":
|
||||
raise ControllerArtifactPromotionError("controller_promoter_already_running")
|
||||
if identifier:
|
||||
try:
|
||||
inspect = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{quote(identifier, safe='')}/json",
|
||||
).json()
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise ControllerArtifactPromotionError(
|
||||
"controller_promoter_helper_inspect_failed"
|
||||
) from exc
|
||||
labels = ((inspect.get("Config") or {}).get("Labels") or {}) if isinstance(inspect, dict) else {}
|
||||
if not isinstance(labels, dict) or labels.get(HELPER_LABEL) != "true":
|
||||
raise ControllerArtifactPromotionError("controller_promoter_name_conflict")
|
||||
self._remove_helper(identifier)
|
||||
|
||||
spec = helper_container_spec(revision=revision, image_id=image_id)
|
||||
try:
|
||||
created_response = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/create",
|
||||
params={"name": HELPER_NAME},
|
||||
json=spec,
|
||||
)
|
||||
created = created_response.json()
|
||||
helper_id = str(created.get("Id") or "") if isinstance(created, dict) else ""
|
||||
if not helper_id:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_create_failed")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{quote(helper_id, safe='')}/start",
|
||||
)
|
||||
except ControllerArtifactPromotionError:
|
||||
raise
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise ControllerArtifactPromotionError("controller_promoter_start_failed") from exc
|
||||
|
||||
deadline = time.monotonic() + _HELPER_TIMEOUT_SECONDS
|
||||
state: dict[str, Any] = {}
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
payload = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{quote(helper_id, safe='')}/json",
|
||||
).json()
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise ControllerArtifactPromotionError(
|
||||
"controller_promoter_helper_inspect_failed"
|
||||
) from exc
|
||||
state = payload.get("State") if isinstance(payload, dict) and isinstance(payload.get("State"), dict) else {}
|
||||
if state.get("Running") is False:
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"revision": revision,
|
||||
"reason": "controller_promoter_timeout",
|
||||
"helperId": helper_id,
|
||||
}
|
||||
|
||||
exit_code = state.get("ExitCode")
|
||||
logs = self.portainer.get_container_logs(helper_id, tail=200)
|
||||
log_lines = logs.get("logs") if isinstance(logs, dict) and isinstance(logs.get("logs"), list) else []
|
||||
tail = "\n".join(str(line) for line in log_lines[-40:])[-4000:]
|
||||
if exit_code != 0:
|
||||
try:
|
||||
self._remove_helper(helper_id)
|
||||
except RuntimeError:
|
||||
pass
|
||||
return {
|
||||
"status": "failed",
|
||||
"revision": revision,
|
||||
"reason": "controller_promoter_helper_failed",
|
||||
"exitCode": exit_code,
|
||||
"logTail": redact_message(tail),
|
||||
}
|
||||
|
||||
try:
|
||||
self._remove_helper(helper_id)
|
||||
except RuntimeError:
|
||||
pass
|
||||
return {"status": "promoted", "revision": revision, "logTail": redact_message(tail)}
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
revision = self._validated_main_revision()
|
||||
current = self.broker.controller_health()
|
||||
artifact = current.get("artifact") if isinstance(current, dict) else None
|
||||
if (
|
||||
isinstance(artifact, dict)
|
||||
and artifact.get("sourceRevision") == revision
|
||||
and artifact.get("controllerVersion") == CONTROLLER_VERSION
|
||||
):
|
||||
return {
|
||||
"status": "already_applied",
|
||||
"revision": revision,
|
||||
"controllerVersion": CONTROLLER_VERSION,
|
||||
}
|
||||
|
||||
result = self._run_helper(revision=revision, image_id=self._controller_image_id())
|
||||
if result.get("status") != "promoted":
|
||||
return result
|
||||
|
||||
verified = self.broker.controller_health()
|
||||
artifact = verified.get("artifact") if isinstance(verified, dict) else None
|
||||
if (
|
||||
not isinstance(artifact, dict)
|
||||
or artifact.get("sourceRevision") != revision
|
||||
or artifact.get("controllerVersion") != CONTROLLER_VERSION
|
||||
):
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"revision": revision,
|
||||
"reason": "controller_promoter_postcheck_failed",
|
||||
}
|
||||
return {
|
||||
"status": "promoted",
|
||||
"revision": revision,
|
||||
"controllerVersion": CONTROLLER_VERSION,
|
||||
"artifactSha256": artifact.get("artifactSha256"),
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Write-only import of the fixed controller enrollment bootstrap credential."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..db.session import engine
|
||||
from .repopulse_secret_import import (
|
||||
BOOTSTRAP_SECRET_REF,
|
||||
MAX_SECRET_BYTES,
|
||||
import_controller_bootstrap,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
raw = sys.stdin.buffer.read(MAX_SECRET_BYTES + 1)
|
||||
if len(raw) > MAX_SECRET_BYTES:
|
||||
print(json.dumps({"status": "blocked", "reason": "credential input too large"}))
|
||||
return 2
|
||||
try:
|
||||
token = raw.decode("utf-8")
|
||||
with Session(engine) as session:
|
||||
outcome = import_controller_bootstrap(session, token)
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
print(json.dumps({"status": "blocked", "reason": "credential import rejected"}))
|
||||
return 2
|
||||
print(json.dumps({"status": outcome, "secretRef": BOOTSTRAP_SECRET_REF}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,562 @@
|
||||
"""Client for the private, typed deployment broker surface.
|
||||
|
||||
The client only sends a canonical DeploymentPlanV1 to a configured controller.
|
||||
It neither exposes a command field nor calls a Docker socket/Portainer exec
|
||||
endpoint. The controller independently validates repository, revision and
|
||||
runtime topology before it can mutate production.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from ..schemas.deployment import DeploymentPlanV1
|
||||
|
||||
_REQUEST_ID = re.compile(r"^[0-9a-f]{64}$")
|
||||
_MAX_DELIVERY_STATUS_BYTES = 512 * 1024
|
||||
_MAX_RUNNER_FLEET_STATUS_BYTES = 512 * 1024
|
||||
_MAX_CONTROLLER_HEALTH_BYTES = 64 * 1024
|
||||
_WORKMESH_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
_WORKMESH_PREFLIGHT_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_WORKMESH_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
|
||||
|
||||
class DeploymentBrokerClient:
|
||||
"""Minimal HMAC client for the controller's versioned deployment API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str | None = None,
|
||||
verify_tls: bool | None = None,
|
||||
ca_file: str | None = None,
|
||||
):
|
||||
configured_url = base_url or settings.apis.get("DEPLOYMENT_BROKER_URL")
|
||||
self.base_url = str(configured_url or "").strip().rstrip("/")
|
||||
self.verify_tls = (
|
||||
bool(settings.apis.get("DEPLOYMENT_BROKER_VERIFY_TLS"))
|
||||
if verify_tls is None
|
||||
else verify_tls
|
||||
)
|
||||
configured_ca_file = (
|
||||
settings.apis.get("DEPLOYMENT_BROKER_CA_FILE")
|
||||
if ca_file is None
|
||||
else ca_file
|
||||
)
|
||||
self.ca_file = str(configured_ca_file or "").strip() or None
|
||||
self.timeout_seconds = max(
|
||||
2.0,
|
||||
min(30.0, float(settings.apis.get("DEPLOYMENT_BROKER_TIMEOUT_SECONDS") or 10)),
|
||||
)
|
||||
|
||||
def configuration_error(self) -> str | None:
|
||||
if not self.base_url:
|
||||
return "deployment_broker_url_missing"
|
||||
parsed = urlparse(self.base_url)
|
||||
loopback_http = False
|
||||
if parsed.scheme == "http":
|
||||
hostname = parsed.hostname or ""
|
||||
try:
|
||||
loopback_http = ipaddress.ip_address(hostname).is_loopback
|
||||
except ValueError:
|
||||
loopback_http = hostname.lower() == "localhost"
|
||||
if (
|
||||
not (parsed.scheme == "https" or loopback_http)
|
||||
or not parsed.netloc
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
return "deployment_broker_url_invalid"
|
||||
if parsed.scheme == "https":
|
||||
if not self.verify_tls:
|
||||
return "deployment_broker_tls_verification_required"
|
||||
if self.ca_file and not self._trusted_ca_file_is_safe(self.ca_file):
|
||||
return "deployment_broker_ca_file_invalid"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _trusted_ca_file_is_safe(value: str) -> bool:
|
||||
"""Require an AppOps-owned readable CA bundle that others cannot write."""
|
||||
path = Path(value)
|
||||
if not path.is_absolute():
|
||||
return False
|
||||
try:
|
||||
info = path.lstat()
|
||||
except OSError:
|
||||
return False
|
||||
if (
|
||||
path.is_symlink()
|
||||
or not stat.S_ISREG(info.st_mode)
|
||||
or not 1 <= info.st_size <= 1024 * 1024
|
||||
):
|
||||
return False
|
||||
if os.name != "nt":
|
||||
return (
|
||||
info.st_uid == os.geteuid()
|
||||
and not (stat.S_IMODE(info.st_mode) & 0o022)
|
||||
and os.access(path, os.R_OK)
|
||||
)
|
||||
return True
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return self.configuration_error() is None
|
||||
|
||||
def _verify_option(self) -> bool | str:
|
||||
parsed = urlparse(self.base_url)
|
||||
if parsed.scheme == "https" and self.ca_file:
|
||||
return self.ca_file
|
||||
return self.verify_tls
|
||||
|
||||
@staticmethod
|
||||
def _canonical_payload(payload: dict[str, Any]) -> bytes:
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
@staticmethod
|
||||
def _signature(
|
||||
secret: str,
|
||||
method: str,
|
||||
path: str,
|
||||
timestamp: str,
|
||||
nonce: str,
|
||||
body: bytes,
|
||||
) -> str:
|
||||
"""Bind the HMAC to one versioned HTTP request, not merely its body."""
|
||||
body_digest = hashlib.sha256(body).hexdigest()
|
||||
signed = (
|
||||
f"itworx-controller-request/v2\n{method.upper()}\n{path}\n"
|
||||
f"{timestamp}\n{nonce}\n{body_digest}"
|
||||
).encode("ascii")
|
||||
return hmac.new(secret.encode("utf-8"), signed, hashlib.sha256).hexdigest()
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
secret: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
error_response_fields: frozenset[str] = frozenset(),
|
||||
) -> dict[str, Any]:
|
||||
if not self.is_configured():
|
||||
return {"status": "missing_config", "reason": self.configuration_error()}
|
||||
if not secret or len(secret) < 32:
|
||||
return {"status": "missing_secret", "reason": "deployment_broker_secret_missing_or_weak"}
|
||||
body = self._canonical_payload(payload) if payload is not None else b""
|
||||
timestamp = str(int(time.time()))
|
||||
nonce = uuid.uuid4().hex
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"X-ITWorx-Deployment-Timestamp": timestamp,
|
||||
"X-ITWorx-Deployment-Nonce": nonce,
|
||||
"X-ITWorx-Deployment-Signature": self._signature(
|
||||
secret, method, path, timestamp, nonce, body
|
||||
),
|
||||
}
|
||||
if payload is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.timeout_seconds,
|
||||
verify=self._verify_option(),
|
||||
) as client:
|
||||
response = client.request(method, f"{self.base_url}{path}", content=body, headers=headers)
|
||||
except httpx.RequestError as exc:
|
||||
return {"status": "unreachable", "reason": redact_message(str(exc))}
|
||||
if response.status_code in {401, 403}:
|
||||
return {"status": "unauthorized", "reason": "deployment_broker_authentication_failed"}
|
||||
try:
|
||||
decoded = response.json()
|
||||
except ValueError:
|
||||
decoded = {}
|
||||
if not isinstance(decoded, dict):
|
||||
decoded = {}
|
||||
if response.status_code >= 400:
|
||||
rejected = {
|
||||
"status": str(decoded.get("status") or "rejected"),
|
||||
"reason": redact_message(str(decoded.get("reason") or f"deployment_broker_http_{response.status_code}")),
|
||||
}
|
||||
for field in error_response_fields:
|
||||
value = decoded.get(field)
|
||||
if isinstance(value, str) and len(value) <= 256:
|
||||
rejected[field] = value
|
||||
return rejected
|
||||
return {"status": str(decoded.get("status") or "ok"), **decoded}
|
||||
|
||||
def controller_health(self) -> dict[str, Any]:
|
||||
"""Read the bounded root-controller health and release identity."""
|
||||
if not self.is_configured():
|
||||
return {"status": "missing_config", "reason": self.configuration_error()}
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.timeout_seconds,
|
||||
verify=self._verify_option(),
|
||||
follow_redirects=False,
|
||||
) as client, client.stream(
|
||||
"GET",
|
||||
f"{self.base_url}/health",
|
||||
headers={"Accept": "application/json"},
|
||||
) as response:
|
||||
if response.status_code != 200:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"reason": f"controller_health_http_{response.status_code}",
|
||||
}
|
||||
body = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > _MAX_CONTROLLER_HEALTH_BYTES:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"reason": "controller_health_response_too_large",
|
||||
}
|
||||
except httpx.RequestError as exc:
|
||||
return {"status": "unreachable", "reason": redact_message(str(exc))}
|
||||
try:
|
||||
decoded = json.loads(bytes(body).decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {"status": "unavailable", "reason": "controller_health_invalid_response"}
|
||||
return (
|
||||
decoded
|
||||
if isinstance(decoded, dict)
|
||||
else {"status": "unavailable", "reason": "controller_health_invalid_response"}
|
||||
)
|
||||
|
||||
def preflight(self, plan: DeploymentPlanV1, *, secret: str) -> dict[str, Any]:
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/deployments/preflight",
|
||||
secret=secret,
|
||||
payload={"plan": plan.model_dump(mode="json")},
|
||||
)
|
||||
|
||||
def appops_self_upgrade_preflight(
|
||||
self, revision: str, reason: str, expected_current_revision: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Ask the private controller to validate one fixed protected-main target."""
|
||||
return self._request(
|
||||
"POST", "/v1/appops/self-upgrade/preflight",
|
||||
secret=settings.mcp_token or "",
|
||||
payload={"repository": "jens/unraid-appops-gateway", "revision": revision,
|
||||
"reason": reason, "expectedCurrentRevision": expected_current_revision},
|
||||
)
|
||||
|
||||
def appops_self_upgrade_apply(
|
||||
self, revision: str, reason: str, expected_current_revision: str | None,
|
||||
preflight_sha256: str, idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
return self._request(
|
||||
"POST", "/v1/appops/self-upgrade/apply",
|
||||
secret=settings.mcp_token or "",
|
||||
payload={"repository": "jens/unraid-appops-gateway", "revision": revision,
|
||||
"reason": reason, "expectedCurrentRevision": expected_current_revision,
|
||||
"preflightSha256": preflight_sha256, "idempotencyKey": idempotency_key},
|
||||
)
|
||||
|
||||
def appops_self_upgrade_status(self, operation_id: str) -> dict[str, Any]:
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", operation_id):
|
||||
return {"status": "invalid_operation_id"}
|
||||
return self._request(
|
||||
"GET", f"/v1/appops/self-upgrade/status/{operation_id}",
|
||||
secret=settings.mcp_token or "",
|
||||
)
|
||||
|
||||
def apply(self, plan: DeploymentPlanV1, *, secret: str, idempotency_key: str) -> dict[str, Any]:
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/deployments/apply",
|
||||
secret=secret,
|
||||
payload={"plan": plan.model_dump(mode="json"), "idempotencyKey": idempotency_key},
|
||||
)
|
||||
|
||||
def rollback(self, plan: DeploymentPlanV1, *, secret: str, idempotency_key: str) -> dict[str, Any]:
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/deployments/rollback",
|
||||
secret=secret,
|
||||
payload={"plan": plan.model_dump(mode="json"), "idempotencyKey": idempotency_key},
|
||||
)
|
||||
|
||||
def enrollment_preflight(self, enrollment: dict[str, Any], *, secret: str) -> dict[str, Any]:
|
||||
"""Validate a bounded target enrollment with the controller, read-only."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/enrollments/preflight",
|
||||
secret=secret,
|
||||
payload={"enrollment": enrollment},
|
||||
)
|
||||
|
||||
def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]:
|
||||
"""Read the bootstrap-protected enrollment controller capability."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/enrollments/capabilities",
|
||||
secret=secret,
|
||||
payload={},
|
||||
)
|
||||
|
||||
def repopulse_adoption_capabilities(self, *, secret: str) -> dict[str, Any]:
|
||||
"""Read fixed RepoPulse adoption readiness through bootstrap HMAC v2."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/repopulse/adoption/capabilities",
|
||||
secret=secret,
|
||||
payload={},
|
||||
)
|
||||
|
||||
def repopulse_adoption_reserve(
|
||||
self,
|
||||
*,
|
||||
secret: str,
|
||||
intent_id: str,
|
||||
idempotency_sha256: str,
|
||||
preflight_sha256: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Reserve the fixed controller intent through bootstrap HMAC v2."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/repopulse/adoption/reserve",
|
||||
secret=secret,
|
||||
payload={
|
||||
"intentId": intent_id,
|
||||
"idempotencySha256": idempotency_sha256,
|
||||
"preflightSha256": preflight_sha256,
|
||||
},
|
||||
)
|
||||
|
||||
def repopulse_adoption_status(self, *, secret: str) -> dict[str, Any]:
|
||||
"""Read sanitized fixed adoption state through bootstrap HMAC v2."""
|
||||
return self._request(
|
||||
"GET",
|
||||
"/v1/appops/repopulse/adoption/status",
|
||||
secret=secret,
|
||||
)
|
||||
|
||||
def repopulse_adoption_execute(
|
||||
self,
|
||||
*,
|
||||
secret: str,
|
||||
intent_id: str,
|
||||
idempotency_sha256: str,
|
||||
preflight_sha256: str,
|
||||
preflight_projection: dict[str, Any],
|
||||
target_credential: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Request the one fixed migration; the controller owns every effect."""
|
||||
if urlparse(self.base_url).scheme != "https" or self.configuration_error() is not None:
|
||||
return {
|
||||
"status": "missing_config",
|
||||
"reason": "repopulse_adoption_private_tls_required",
|
||||
}
|
||||
if (
|
||||
not isinstance(target_credential, str)
|
||||
or not 32 <= len(target_credential) <= 256
|
||||
or not target_credential.isascii()
|
||||
or not target_credential.isprintable()
|
||||
):
|
||||
return {"status": "invalid", "reason": "repopulse_target_credential_invalid"}
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/repopulse/adoption/execute",
|
||||
secret=secret,
|
||||
payload={
|
||||
"intentId": intent_id,
|
||||
"idempotencySha256": idempotency_sha256,
|
||||
"preflightSha256": preflight_sha256,
|
||||
"preflightProjection": preflight_projection,
|
||||
"targetCredential": target_credential,
|
||||
},
|
||||
)
|
||||
|
||||
def enroll_target(
|
||||
self,
|
||||
enrollment: dict[str, Any],
|
||||
*,
|
||||
secret: str,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Create exactly one controller-owned target through bootstrap HMAC."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/enrollments",
|
||||
secret=secret,
|
||||
payload={"enrollment": enrollment, "idempotencyKey": idempotency_key},
|
||||
)
|
||||
|
||||
def configure_enrollment_runtime(
|
||||
self,
|
||||
runtime: dict[str, Any],
|
||||
*,
|
||||
secret: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Write profile-owned runtime values using that target's own HMAC."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/enrollments/runtime",
|
||||
secret=secret,
|
||||
payload={"runtime": runtime},
|
||||
)
|
||||
|
||||
def unenroll_target(
|
||||
self,
|
||||
target: dict[str, Any],
|
||||
*,
|
||||
secret: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Compensate an untouched target through the bootstrap boundary only."""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/enrollments/rollback",
|
||||
secret=secret,
|
||||
payload={"target": target},
|
||||
)
|
||||
|
||||
def status(self, request_id: str, *, secret: str) -> dict[str, Any]:
|
||||
if not _REQUEST_ID.fullmatch(request_id):
|
||||
return {"status": "invalid", "reason": "invalid_broker_request_id"}
|
||||
return self._request(
|
||||
"GET",
|
||||
f"/v1/appops/deployments/{request_id}",
|
||||
secret=secret,
|
||||
)
|
||||
|
||||
def workmesh_promotion_preflight(self, revision: str, *, secret: str) -> dict[str, Any]:
|
||||
"""Preflight only the fixed WorkMesh product target at one full SHA."""
|
||||
if not _WORKMESH_REVISION.fullmatch(revision):
|
||||
return {"status": "invalid", "reason": "invalid_workmesh_revision"}
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/workmesh/promotions/preflight",
|
||||
secret=secret,
|
||||
payload={"revision": revision},
|
||||
)
|
||||
|
||||
def workmesh_promotion_apply(
|
||||
self,
|
||||
revision: str,
|
||||
*,
|
||||
secret: str,
|
||||
preflight_sha256: str,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Queue a promotion only when its approved preflight digest still matches."""
|
||||
if not _WORKMESH_REVISION.fullmatch(revision):
|
||||
return {"status": "invalid", "reason": "invalid_workmesh_revision"}
|
||||
if not _WORKMESH_PREFLIGHT_SHA256.fullmatch(preflight_sha256):
|
||||
return {"status": "invalid", "reason": "invalid_workmesh_preflight_digest"}
|
||||
if not _WORKMESH_IDEMPOTENCY.fullmatch(idempotency_key):
|
||||
return {"status": "invalid", "reason": "invalid_workmesh_idempotency_key"}
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/workmesh/promotions/apply",
|
||||
secret=secret,
|
||||
payload={
|
||||
"revision": revision,
|
||||
"preflightSha256": preflight_sha256,
|
||||
"idempotencyKey": idempotency_key,
|
||||
},
|
||||
error_response_fields=frozenset({"request", "repository", "revision", "operation"}),
|
||||
)
|
||||
|
||||
def workmesh_promotion_status(self, request_id: str, *, secret: str) -> dict[str, Any]:
|
||||
"""Read one HMAC-authenticated fixed-target promotion receipt projection."""
|
||||
return self.status(request_id, secret=secret)
|
||||
|
||||
def delivery_status(self) -> dict[str, Any]:
|
||||
"""Read bounded, secret-free runnerless delivery health from controller.
|
||||
|
||||
This endpoint deliberately sends no deployment credential: the
|
||||
controller publishes only a private-network, read-only projection of
|
||||
its durable webhook inbox. All mutations keep using the HMAC-bound
|
||||
typed request surface above.
|
||||
"""
|
||||
if not self.is_configured():
|
||||
return {"status": "missing_config", "reason": self.configuration_error()}
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.timeout_seconds,
|
||||
verify=self._verify_option(),
|
||||
follow_redirects=False,
|
||||
) as client, client.stream(
|
||||
"GET",
|
||||
f"{self.base_url}/v1/gitea/deliveries",
|
||||
headers={"Accept": "application/json"},
|
||||
) as response:
|
||||
if response.status_code != 200:
|
||||
return {"status": "unavailable", "reason": f"delivery_status_http_{response.status_code}"}
|
||||
body = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > _MAX_DELIVERY_STATUS_BYTES:
|
||||
return {"status": "unavailable", "reason": "delivery_status_response_too_large"}
|
||||
except httpx.RequestError as exc:
|
||||
return {"status": "unreachable", "reason": redact_message(str(exc))}
|
||||
try:
|
||||
decoded = json.loads(bytes(body).decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {"status": "unavailable", "reason": "delivery_status_invalid_response"}
|
||||
return decoded if isinstance(decoded, dict) else {"status": "unavailable", "reason": "delivery_status_invalid_response"}
|
||||
|
||||
def runner_fleet_upgrade_v2(self, *, reason: str) -> dict[str, Any]:
|
||||
"""Request the one fixed Tower runner-fleet v2 migration.
|
||||
|
||||
The root controller derives and verifies the exact protected-main
|
||||
revision itself. The caller cannot select a script path, runner name,
|
||||
revision or Docker target.
|
||||
"""
|
||||
return self._request(
|
||||
"POST",
|
||||
"/v1/appops/runner-fleet/upgrade-v2",
|
||||
secret=settings.mcp_token or "",
|
||||
payload={"reason": reason[:1000]},
|
||||
)
|
||||
|
||||
def runner_fleet_status(self) -> dict[str, Any]:
|
||||
"""Read the controller's private, bounded CI sandbox projection.
|
||||
|
||||
This is deliberately read-only and sends no credential. The controller
|
||||
derives the payload from fixed runner identities and its private DinD
|
||||
daemon; AppOps never receives a Docker socket or a cleanup selector.
|
||||
"""
|
||||
if not self.is_configured():
|
||||
return {"status": "missing_config", "reason": self.configuration_error()}
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.timeout_seconds,
|
||||
verify=self._verify_option(),
|
||||
follow_redirects=False,
|
||||
) as client, client.stream(
|
||||
"GET",
|
||||
f"{self.base_url}/v1/runner-fleet",
|
||||
headers={"Accept": "application/json"},
|
||||
) as response:
|
||||
if response.status_code != 200:
|
||||
return {"status": "unavailable", "reason": f"runner_fleet_status_http_{response.status_code}"}
|
||||
body = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > _MAX_RUNNER_FLEET_STATUS_BYTES:
|
||||
return {"status": "unavailable", "reason": "runner_fleet_status_response_too_large"}
|
||||
except httpx.RequestError as exc:
|
||||
return {"status": "unreachable", "reason": redact_message(str(exc))}
|
||||
try:
|
||||
decoded = json.loads(bytes(body).decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {"status": "unavailable", "reason": "runner_fleet_status_invalid_response"}
|
||||
return decoded if isinstance(decoded, dict) else {"status": "unavailable", "reason": "runner_fleet_status_invalid_response"}
|
||||
@@ -0,0 +1,613 @@
|
||||
"""Deployment Control Plane V1 orchestration.
|
||||
|
||||
The service is intentionally a bounded state machine. It validates a typed
|
||||
plan, records durable idempotent jobs, and talks only to dedicated API clients.
|
||||
There is no free-form command, Compose content, Docker socket or host path
|
||||
input anywhere on the public API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..connectors.authentik import AuthentikConnector
|
||||
from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..connectors.unraid import UnraidConnector
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import DeploymentJob, Policy
|
||||
from ..schemas.deployment import DeploymentPlanV1
|
||||
from .audit_logger import log_event
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
from .policy_engine import evaluate_policy
|
||||
from .secret_store import get_secret, has_secret, secret_storage_status
|
||||
|
||||
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
_TERMINAL = {"preflight_failed", "succeeded", "failed", "rolled_back", "rollback_failed"}
|
||||
_BROKER_EVIDENCE_FIELDS = {"request", "repository", "revision", "status"}
|
||||
_OIDC_EVIDENCE_FIELDS = {"success", "status", "applicationId", "providerId", "clientSecretCreated"}
|
||||
_PROXY_EVIDENCE_FIELDS = {"success", "status", "hostId", "hostname"}
|
||||
_AUTO_ROLLBACK_REPOSITORY = "jens/itworx_mcp_hub"
|
||||
_AUTO_ROLLBACK_APPLICATION_ID = "itworx-mcp-hub"
|
||||
|
||||
|
||||
class DeploymentControlError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _canonical_plan(plan: DeploymentPlanV1) -> tuple[dict[str, Any], str]:
|
||||
value = plan.model_dump(mode="json")
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
return value, digest
|
||||
|
||||
|
||||
def _idempotency_digest(value: str) -> str:
|
||||
if not _IDEMPOTENCY_KEY.fullmatch(value):
|
||||
raise DeploymentControlError("idempotency key must contain 8 to 128 safe characters")
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _check(name: str, status: Literal["passed", "blocked", "warning", "unknown"], detail: str) -> dict[str, str]:
|
||||
return {"name": name, "status": status, "detail": detail}
|
||||
|
||||
|
||||
class DeploymentControlPlane:
|
||||
"""Durable orchestrator for strict DeploymentPlanV1 requests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
broker: DeploymentBrokerClient | None = None,
|
||||
portainer: PortainerConnector | None = None,
|
||||
npm: NginxProxyManagerConnector | None = None,
|
||||
authentik: AuthentikConnector | None = None,
|
||||
unraid: UnraidConnector | None = None,
|
||||
):
|
||||
self.broker = broker or DeploymentBrokerClient()
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
self.npm = npm or NginxProxyManagerConnector()
|
||||
self.authentik = authentik or AuthentikConnector()
|
||||
self.unraid = unraid or UnraidConnector()
|
||||
|
||||
def preflight(
|
||||
self,
|
||||
session: Session,
|
||||
plan: DeploymentPlanV1,
|
||||
*,
|
||||
operation: Literal["preflight", "apply", "rollback"] = "apply",
|
||||
) -> dict[str, Any]:
|
||||
checks: list[dict[str, str]] = []
|
||||
policy_action = "deployment.rollback" if operation == "rollback" else "deployment.apply"
|
||||
policy = evaluate_policy(
|
||||
session,
|
||||
action=policy_action,
|
||||
target=plan.application_id,
|
||||
source="manual",
|
||||
)
|
||||
exact_policy = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == policy_action) & (Policy.target == plan.application_id)
|
||||
)
|
||||
).first()
|
||||
if policy.get("decision") == "allowed" and exact_policy is not None:
|
||||
checks.append(_check("deployment_policy", "passed", "Exact-target deployment policy is present."))
|
||||
else:
|
||||
checks.append(_check("deployment_policy", "blocked", "Deployment requires an explicit allowed exact-target policy."))
|
||||
if has_secret(session, plan.broker_secret_ref):
|
||||
checks.append(_check("broker_secret_reference", "passed", "Stored controller secret is available by reference."))
|
||||
else:
|
||||
checks.append(_check("broker_secret_reference", "blocked", "The referenced deployment-broker secret is not configured."))
|
||||
|
||||
containers = self.portainer.list_containers()
|
||||
if containers.get("status") != "ok":
|
||||
checks.append(_check("portainer_inventory", "blocked", "Portainer inventory is unavailable for collision preflight."))
|
||||
else:
|
||||
names = {str(item.get("name") or "").casefold() for item in containers.get("containers", [])}
|
||||
collisions = sorted(name for name in plan.topology.containers if name.casefold() in names)
|
||||
# Existing expected containers are not unsafe by themselves: an exact
|
||||
# controller receipt decides whether an apply is an upgrade or a
|
||||
# conflicting takeover.
|
||||
detail = "No managed container name is currently present." if not collisions else "Existing containers require controller ownership verification: " + ", ".join(collisions)
|
||||
checks.append(_check("container_ownership", "warning" if collisions else "passed", detail))
|
||||
|
||||
if plan.proxy:
|
||||
hosts = self.npm.proxy_hosts()
|
||||
if not hosts.get("success"):
|
||||
checks.append(_check("proxy_hostname", "blocked", "Nginx Proxy Manager inventory is unavailable."))
|
||||
else:
|
||||
matches = [
|
||||
item
|
||||
for item in hosts.get("items", [])
|
||||
if plan.proxy.hostname in {str(value).lower() for value in item.get("domainNames", [])}
|
||||
]
|
||||
if not matches:
|
||||
checks.append(_check("proxy_hostname", "passed", "Requested hostname is free in the NPM inventory."))
|
||||
elif all(
|
||||
str(item.get("forwardHost") or "") == plan.application_id
|
||||
and int(item.get("forwardPort") or 0) == plan.topology.internal_port
|
||||
for item in matches
|
||||
):
|
||||
checks.append(_check("proxy_hostname", "passed", "Existing hostname already targets the planned managed container."))
|
||||
else:
|
||||
checks.append(_check("proxy_hostname", "blocked", "Requested hostname is owned by a different NPM target."))
|
||||
|
||||
if plan.oidc:
|
||||
auth_status = self.authentik.test_connection()
|
||||
if auth_status.get("status") != "connected":
|
||||
checks.append(_check("oidc_provider", "blocked", "Authentik is unavailable for the requested OIDC plan."))
|
||||
else:
|
||||
apps = self.authentik.applications()
|
||||
if not apps.get("success"):
|
||||
checks.append(_check("oidc_provider", "blocked", "Authentik applications cannot be inspected."))
|
||||
else:
|
||||
found = [item for item in apps.get("items", []) if item.get("slug") == plan.oidc.application_slug]
|
||||
checks.append(_check("oidc_provider", "warning" if found else "passed", "Existing Authentik application will be ownership-checked." if found else "OIDC application can be provisioned from the bounded template."))
|
||||
storage = secret_storage_status()
|
||||
if not (storage.get("productionReady") or storage.get("status") == "ready"):
|
||||
checks.append(_check("oidc_secret_reference", "blocked", "Encrypted secret storage is not ready for the referenced OIDC client secret."))
|
||||
elif has_secret(session, plan.oidc.client_secret_ref):
|
||||
checks.append(_check("oidc_secret_reference", "passed", "The write-only OIDC client secret reference is available for the bounded provider setup."))
|
||||
else:
|
||||
checks.append(_check("oidc_secret_reference", "blocked", "The referenced OIDC client secret is not configured in encrypted secret storage."))
|
||||
|
||||
shares = self.unraid.get_share_summary()
|
||||
if shares.get("status") in {"ok", "connected"}:
|
||||
checks.append(_check("trusted_appdata_root", "passed", f"Appdata path is derived as {plan.appdata_path}."))
|
||||
else:
|
||||
checks.append(_check("trusted_appdata_root", "warning", f"Appdata is derived as {plan.appdata_path}; Unraid share inventory is currently unavailable."))
|
||||
|
||||
broker_secret = get_secret(session, plan.broker_secret_ref, touch=False)
|
||||
if broker_secret:
|
||||
result = self.broker.preflight(plan, secret=broker_secret)
|
||||
status = str(result.get("status") or "unknown")
|
||||
if status in {"ok", "preflight_ok", "already_deployed"}:
|
||||
checks.append(_check("deployment_controller", "passed", "Controller accepted immutable repository and topology preflight."))
|
||||
elif status in {"missing_config", "missing_secret"}:
|
||||
checks.append(_check("deployment_controller", "blocked", "Deployment controller is not configured for the typed API."))
|
||||
else:
|
||||
checks.append(_check("deployment_controller", "blocked", "Controller rejected typed deployment preflight."))
|
||||
else:
|
||||
checks.append(_check("deployment_controller", "blocked", "Controller authentication secret is unavailable."))
|
||||
|
||||
blockers = [item["name"] for item in checks if item["status"] == "blocked"]
|
||||
warnings = [item["name"] for item in checks if item["status"] == "warning"]
|
||||
return {
|
||||
"status": "blocked" if blockers else "passed",
|
||||
"blockers": blockers,
|
||||
"warnings": warnings,
|
||||
"checks": checks,
|
||||
"appdataPath": plan.appdata_path,
|
||||
"publicUrl": plan.public_url,
|
||||
}
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
plan: DeploymentPlanV1,
|
||||
idempotency_key: str,
|
||||
operation: Literal["preflight", "apply", "rollback"],
|
||||
) -> DeploymentJob:
|
||||
plan_json, plan_digest = _canonical_plan(plan)
|
||||
idempotency_digest = _idempotency_digest(f"{operation}:{idempotency_key}")
|
||||
existing = session.exec(
|
||||
select(DeploymentJob).where(DeploymentJob.idempotency_sha256 == idempotency_digest)
|
||||
).first()
|
||||
if existing:
|
||||
if existing.plan_sha256 != plan_digest:
|
||||
raise DeploymentControlError("idempotency key was already used for a different deployment plan")
|
||||
return existing
|
||||
job = DeploymentJob(
|
||||
id=f"deploy-{uuid.uuid4().hex[:16]}",
|
||||
application_id=plan.application_id,
|
||||
repository=plan.repository,
|
||||
revision=plan.revision,
|
||||
plan_sha256=plan_digest,
|
||||
idempotency_sha256=idempotency_digest,
|
||||
status="queued",
|
||||
stage="accepted",
|
||||
plan_json=plan_json,
|
||||
evidence_json={"operation": operation},
|
||||
)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def serialize_job(job: DeploymentJob, *, replay: bool = False) -> dict[str, Any]:
|
||||
return {
|
||||
"id": job.id,
|
||||
"applicationId": job.application_id,
|
||||
"repository": job.repository,
|
||||
"revision": job.revision,
|
||||
"status": job.status,
|
||||
"stage": job.stage,
|
||||
"preflight": job.preflight_json,
|
||||
"evidence": job.evidence_json,
|
||||
"result": job.result_summary,
|
||||
"brokerRequestId": job.broker_request_id,
|
||||
"createdAt": job.created_at.isoformat(),
|
||||
"updatedAt": job.updated_at.isoformat(),
|
||||
"completedAt": job.completed_at.isoformat() if job.completed_at else None,
|
||||
"idempotentReplay": replay,
|
||||
}
|
||||
|
||||
def _complete_external_integrations(
|
||||
self,
|
||||
session: Session,
|
||||
plan: DeploymentPlanV1,
|
||||
) -> tuple[bool, dict[str, Any], str]:
|
||||
"""Complete only plan-derived edge/OIDC integration after live proof.
|
||||
|
||||
Application environment secret delivery remains an explicit controller
|
||||
enrollment responsibility. This code resolves the same write-only
|
||||
reference only inside the backend to create/verify the bounded OIDC
|
||||
provider, and never writes it to an API result or audit record.
|
||||
"""
|
||||
evidence: dict[str, Any] = {}
|
||||
if plan.oidc:
|
||||
client_secret = get_secret(session, plan.oidc.client_secret_ref, touch=False)
|
||||
if not client_secret:
|
||||
return False, evidence, "Referenced OIDC client secret is unavailable after deployment."
|
||||
oidc_result = self.authentik.ensure_deployment_oidc_application(
|
||||
application_slug=plan.oidc.application_slug,
|
||||
launch_url=plan.public_url or "",
|
||||
redirect_uri=f"{plan.public_url}{plan.oidc.redirect_path}",
|
||||
client_secret=client_secret,
|
||||
administrator_group=plan.oidc.roles.administrator_group,
|
||||
support_agent_group=plan.oidc.roles.support_agent_group,
|
||||
read_only_group=plan.oidc.roles.read_only_group,
|
||||
)
|
||||
evidence["oidc"] = {
|
||||
key: oidc_result[key] for key in _OIDC_EVIDENCE_FIELDS if key in oidc_result
|
||||
}
|
||||
if not oidc_result.get("success") or oidc_result.get("status") not in {"created", "already_managed"}:
|
||||
return False, evidence, "Typed OIDC provider/application setup was not verified."
|
||||
if plan.proxy:
|
||||
proxy_result = self.npm.ensure_managed_proxy_host(
|
||||
hostname=plan.proxy.hostname,
|
||||
forward_host=plan.application_id,
|
||||
forward_port=plan.topology.internal_port,
|
||||
certificate_mode=plan.proxy.certificate_mode,
|
||||
force_ssl=plan.proxy.force_ssl,
|
||||
http2=plan.proxy.http2,
|
||||
upload_limit_mb=plan.proxy.upload_limit_mb,
|
||||
read_timeout_seconds=plan.proxy.read_timeout_seconds,
|
||||
)
|
||||
evidence["proxy"] = {
|
||||
key: proxy_result[key] for key in _PROXY_EVIDENCE_FIELDS if key in proxy_result
|
||||
}
|
||||
if not proxy_result.get("success") or proxy_result.get("status") not in {"created", "already_managed"}:
|
||||
return False, evidence, "Typed reverse-proxy setup was not verified."
|
||||
return True, evidence, "External integration is verified."
|
||||
|
||||
def _complete_apply_job(
|
||||
self,
|
||||
session: Session,
|
||||
job: DeploymentJob,
|
||||
plan: DeploymentPlanV1,
|
||||
) -> None:
|
||||
success, integration_evidence, message = self._complete_external_integrations(session, plan)
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"externalIntegration": integration_evidence,
|
||||
}
|
||||
if success:
|
||||
job.status = "succeeded"
|
||||
job.stage = "controller_and_edge_verified"
|
||||
job.result_summary = "Deployment controller and required edge integration returned verified evidence."
|
||||
job.completed_at = utc_now()
|
||||
return
|
||||
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"postApplyVerification": {"status": "failed", "detail": message},
|
||||
}
|
||||
if (
|
||||
plan.repository.casefold() != _AUTO_ROLLBACK_REPOSITORY
|
||||
or plan.application_id != _AUTO_ROLLBACK_APPLICATION_ID
|
||||
):
|
||||
job.status = "failed"
|
||||
job.stage = "external_integration_failed"
|
||||
job.result_summary = message
|
||||
job.completed_at = utc_now()
|
||||
return
|
||||
self._attempt_auto_rollback(session, job, plan, reason=message)
|
||||
|
||||
def _attempt_auto_rollback(
|
||||
self,
|
||||
session: Session,
|
||||
job: DeploymentJob,
|
||||
plan: DeploymentPlanV1,
|
||||
*,
|
||||
reason: str,
|
||||
) -> None:
|
||||
"""Restore only the controller-retained prior receipt after verification failure."""
|
||||
policy = evaluate_policy(
|
||||
session,
|
||||
action="deployment.rollback",
|
||||
target=plan.application_id,
|
||||
source="manual",
|
||||
)
|
||||
exact_policies = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == "deployment.rollback")
|
||||
& (Policy.target == plan.application_id)
|
||||
)
|
||||
).all()
|
||||
exact_policy = exact_policies[0] if len(exact_policies) == 1 else None
|
||||
policy_is_exact_and_strict = (
|
||||
exact_policy is not None
|
||||
and exact_policy.manual == "allowed"
|
||||
and exact_policy.voice == "blocked"
|
||||
and exact_policy.risk == "high"
|
||||
and policy.get("policy_id") == exact_policy.id
|
||||
)
|
||||
if policy.get("decision") != "allowed" or not policy_is_exact_and_strict:
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"autoRollback": {"status": "blocked", "reason": "exact-target-policy"},
|
||||
}
|
||||
job.status = "failed"
|
||||
job.stage = "external_integration_failed"
|
||||
job.result_summary = reason
|
||||
job.completed_at = utc_now()
|
||||
return
|
||||
|
||||
secret = get_secret(session, plan.broker_secret_ref)
|
||||
if not secret:
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"autoRollback": {"status": "blocked", "reason": "controller-secret"},
|
||||
}
|
||||
job.status = "failed"
|
||||
job.stage = "external_integration_failed"
|
||||
job.result_summary = reason
|
||||
job.completed_at = utc_now()
|
||||
return
|
||||
|
||||
rollback_key = f"auto-rollback:{job.id}"
|
||||
rollback = self.broker.rollback(plan, secret=secret, idempotency_key=rollback_key)
|
||||
rollback_status = str(rollback.get("status") or "failed")
|
||||
rollback_request = str(rollback.get("request") or "") or None
|
||||
rollback_evidence = {
|
||||
key: rollback[key]
|
||||
for key in _BROKER_EVIDENCE_FIELDS
|
||||
if key in rollback
|
||||
}
|
||||
rollback_evidence["status"] = rollback_status
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"autoRollback": rollback_evidence,
|
||||
}
|
||||
if rollback_request:
|
||||
job.broker_request_id = rollback_request
|
||||
|
||||
if rollback_status == "rolled_back":
|
||||
job.status = "rolled_back"
|
||||
job.stage = "auto_rollback_verified"
|
||||
job.result_summary = (
|
||||
"Post-apply verification failed; the controller restored the prior verified deployment."
|
||||
)
|
||||
job.completed_at = utc_now()
|
||||
elif rollback_status in {"accepted", "already_queued", "running"}:
|
||||
if not rollback_request:
|
||||
job.status = "failed"
|
||||
job.stage = "auto_rollback_failed"
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"autoRollback": {
|
||||
**rollback_evidence,
|
||||
"reason": "missing-controller-request",
|
||||
},
|
||||
}
|
||||
job.result_summary = (
|
||||
"Post-apply verification failed and automatic rollback "
|
||||
"returned no pollable controller request."
|
||||
)
|
||||
job.completed_at = utc_now()
|
||||
return
|
||||
job.status = "rolling_back"
|
||||
job.stage = "auto_rollback_queued"
|
||||
job.result_summary = (
|
||||
"Post-apply verification failed; automatic rollback is in progress."
|
||||
)
|
||||
job.completed_at = None
|
||||
else:
|
||||
job.status = "failed"
|
||||
job.stage = "auto_rollback_failed"
|
||||
job.result_summary = (
|
||||
"Post-apply verification failed and the controller did not verify rollback."
|
||||
)
|
||||
job.completed_at = utc_now()
|
||||
|
||||
def execute(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
plan: DeploymentPlanV1,
|
||||
idempotency_key: str,
|
||||
operation: Literal["preflight", "apply", "rollback"],
|
||||
) -> dict[str, Any]:
|
||||
job = self.create_job(session, plan=plan, idempotency_key=idempotency_key, operation=operation)
|
||||
if job.status != "queued":
|
||||
return self.serialize_job(job, replay=True)
|
||||
|
||||
preflight = self.preflight(session, plan, operation=operation)
|
||||
job.preflight_json = preflight
|
||||
job.updated_at = utc_now()
|
||||
if preflight["status"] != "passed":
|
||||
job.status = "preflight_failed"
|
||||
job.stage = "preflight"
|
||||
job.result_summary = "Deployment preflight was blocked."
|
||||
job.completed_at = utc_now()
|
||||
session.add(job)
|
||||
session.commit()
|
||||
log_event(
|
||||
session,
|
||||
source="deployment-control-plane",
|
||||
identity="operator",
|
||||
action_id=f"deployment.{operation}",
|
||||
target=plan.application_id,
|
||||
target_type="deployment",
|
||||
risk="high",
|
||||
policy_decision="blocked",
|
||||
reason="Typed deployment preflight failed closed.",
|
||||
result="blocked",
|
||||
execution_plan={"planSha256": job.plan_sha256, "operation": operation},
|
||||
post_checks={"preflight": preflight},
|
||||
)
|
||||
return self.serialize_job(job)
|
||||
|
||||
if operation == "preflight":
|
||||
job.status = "preflight_succeeded"
|
||||
job.stage = "preflight"
|
||||
job.result_summary = "Typed deployment preflight passed."
|
||||
job.completed_at = utc_now()
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return self.serialize_job(job)
|
||||
|
||||
secret = get_secret(session, plan.broker_secret_ref)
|
||||
if not secret:
|
||||
raise DeploymentControlError("deployment broker secret was unavailable after preflight")
|
||||
broker_result = (
|
||||
self.broker.apply(plan, secret=secret, idempotency_key=idempotency_key)
|
||||
if operation == "apply"
|
||||
else self.broker.rollback(plan, secret=secret, idempotency_key=idempotency_key)
|
||||
)
|
||||
broker_status = str(broker_result.get("status") or "failed")
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"brokerStatus": broker_status,
|
||||
"brokerEvidence": {
|
||||
key: broker_result[key] for key in _BROKER_EVIDENCE_FIELDS if key in broker_result
|
||||
},
|
||||
}
|
||||
job.broker_request_id = str(broker_result.get("request") or "") or None
|
||||
job.updated_at = utc_now()
|
||||
if broker_status in {"accepted", "already_queued", "running"}:
|
||||
job.status = "applying" if operation == "apply" else "rolling_back"
|
||||
job.stage = "controller_queued"
|
||||
job.result_summary = "Deployment controller accepted the typed request."
|
||||
elif operation == "apply" and broker_status in {"deployed", "already_deployed"}:
|
||||
self._complete_apply_job(session, job, plan)
|
||||
elif operation == "rollback" and broker_status == "rolled_back":
|
||||
job.status = "rolled_back"
|
||||
job.stage = "rollback_verified"
|
||||
job.result_summary = "Deployment controller returned verified rollback evidence."
|
||||
job.completed_at = utc_now()
|
||||
else:
|
||||
job.status = "failed" if operation == "apply" else "rollback_failed"
|
||||
job.stage = "controller_rejected"
|
||||
job.result_summary = "Deployment controller rejected the typed request."
|
||||
job.completed_at = utc_now()
|
||||
session.add(job)
|
||||
session.commit()
|
||||
log_event(
|
||||
session,
|
||||
source="deployment-control-plane",
|
||||
identity="operator",
|
||||
action_id=f"deployment.{operation}",
|
||||
target=plan.application_id,
|
||||
target_type="deployment",
|
||||
risk="high",
|
||||
policy_decision="allowed" if job.status not in {"failed", "rollback_failed"} else "blocked",
|
||||
reason="Typed deployment request sent to the private controller.",
|
||||
result="success" if job.status not in {"failed", "rollback_failed"} else "failed",
|
||||
execution_plan={"planSha256": job.plan_sha256, "operation": operation},
|
||||
after_state={"status": job.status, "stage": job.stage, "brokerRequestId": job.broker_request_id},
|
||||
redacted_metadata={"brokerStatus": broker_status},
|
||||
)
|
||||
return self.serialize_job(job)
|
||||
|
||||
def refresh(self, session: Session, job_id: str) -> dict[str, Any]:
|
||||
job = session.get(DeploymentJob, job_id)
|
||||
if not job:
|
||||
raise DeploymentControlError("deployment job was not found")
|
||||
if job.status in _TERMINAL or not job.broker_request_id:
|
||||
return self.serialize_job(job)
|
||||
plan = DeploymentPlanV1.model_validate(job.plan_json)
|
||||
secret = get_secret(session, plan.broker_secret_ref)
|
||||
if not secret:
|
||||
job.status = "failed"
|
||||
job.stage = "controller_auth_lost"
|
||||
job.result_summary = "Deployment controller secret is no longer available."
|
||||
job.completed_at = utc_now()
|
||||
else:
|
||||
result = self.broker.status(job.broker_request_id, secret=secret)
|
||||
status = str(result.get("status") or "unknown")
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"brokerStatus": status,
|
||||
"brokerEvidence": {
|
||||
key: result[key] for key in _BROKER_EVIDENCE_FIELDS if key in result
|
||||
},
|
||||
}
|
||||
if status in {"deployed", "already_deployed"}:
|
||||
auto_rollback = dict(job.evidence_json or {}).get("autoRollback")
|
||||
if job.status == "rolling_back":
|
||||
job.status = "failed" if isinstance(auto_rollback, dict) else "rollback_failed"
|
||||
job.stage = (
|
||||
"auto_rollback_failed"
|
||||
if isinstance(auto_rollback, dict)
|
||||
else "controller_failed"
|
||||
)
|
||||
job.result_summary = (
|
||||
"Rollback request returned a deployed state and was not accepted as rollback proof."
|
||||
)
|
||||
job.completed_at = utc_now()
|
||||
else:
|
||||
self._complete_apply_job(session, job, plan)
|
||||
elif status == "rolled_back":
|
||||
job.status = "rolled_back"
|
||||
auto_rollback = dict(job.evidence_json or {}).get("autoRollback")
|
||||
if isinstance(auto_rollback, dict):
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"autoRollback": {**auto_rollback, "status": "rolled_back"},
|
||||
}
|
||||
job.stage = "auto_rollback_verified"
|
||||
job.result_summary = (
|
||||
"Post-apply verification failed; the controller verified automatic rollback."
|
||||
)
|
||||
else:
|
||||
job.stage = "rollback_verified"
|
||||
job.result_summary = "Deployment controller verified rollback evidence."
|
||||
job.completed_at = utc_now()
|
||||
elif status in {"failed", "non_forward", "rejected"}:
|
||||
auto_rollback = dict(job.evidence_json or {}).get("autoRollback")
|
||||
if isinstance(auto_rollback, dict) and job.status == "rolling_back":
|
||||
job.status = "failed"
|
||||
job.stage = "auto_rollback_failed"
|
||||
job.evidence_json = {
|
||||
**dict(job.evidence_json or {}),
|
||||
"autoRollback": {**auto_rollback, "status": status},
|
||||
}
|
||||
job.result_summary = (
|
||||
"Post-apply verification failed and automatic rollback was not verified."
|
||||
)
|
||||
else:
|
||||
job.status = "failed" if job.status == "applying" else "rollback_failed"
|
||||
job.stage = "controller_failed"
|
||||
job.result_summary = "Deployment controller reported a typed deployment failure."
|
||||
job.completed_at = utc_now()
|
||||
job.updated_at = utc_now()
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return self.serialize_job(job)
|
||||
|
||||
def list_jobs(self, session: Session, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
rows = session.exec(
|
||||
select(DeploymentJob).order_by(DeploymentJob.created_at.desc()).limit(max(1, min(limit, 250)))
|
||||
).all()
|
||||
return [self.serialize_job(row) for row in rows]
|
||||
@@ -0,0 +1,537 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_REPOSITORY = re.compile(r"^jens/[a-z0-9_.-]+$")
|
||||
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
_BRANCH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
|
||||
_SLUG = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,62}$")
|
||||
_COMPOSE_PROJECT = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
|
||||
_CONTAINER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_MAX_CONTAINERS = 32
|
||||
|
||||
# Fixed application-owned mutation. Callers never provide a command or host
|
||||
# path. Repository-derived topology is validated before it reaches this script,
|
||||
# then re-bound to the exact immutable Git revision inside the controller.
|
||||
_REGISTER_SCRIPT = r'''
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
path = Path("/opt/gitea-deploy/projects.json")
|
||||
host_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
repository, revision, branch, raw_project = sys.argv[1:5]
|
||||
project = json.loads(raw_project)
|
||||
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise SystemExit(20)
|
||||
info = path.stat()
|
||||
if not stat.S_ISREG(info.st_mode):
|
||||
raise SystemExit(21)
|
||||
|
||||
try:
|
||||
if host_path.is_symlink() or not host_path.is_file() or not os.path.samefile(path, host_path):
|
||||
raise SystemExit(36)
|
||||
except OSError:
|
||||
raise SystemExit(36)
|
||||
mutation_path = host_path
|
||||
|
||||
if project.get("branch") != branch:
|
||||
raise SystemExit(37)
|
||||
compose_files = project.get("compose_files")
|
||||
if not isinstance(compose_files, list) or len(compose_files) != 1:
|
||||
raise SystemExit(38)
|
||||
compose_file = compose_files[0]
|
||||
if not isinstance(compose_file, str) or not compose_file:
|
||||
raise SystemExit(38)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(24)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
try:
|
||||
commit_result = git_run("rev-parse", f"{revision}^{{commit}}")
|
||||
except Exception:
|
||||
raise SystemExit(25)
|
||||
commit = commit_result.stdout.strip()
|
||||
if commit_result.returncode != 0 or commit != revision:
|
||||
raise SystemExit(25)
|
||||
|
||||
try:
|
||||
branch_result = git_run("rev-parse", f"refs/heads/{branch}")
|
||||
except Exception:
|
||||
raise SystemExit(26)
|
||||
branch_head = branch_result.stdout.strip()
|
||||
if branch_result.returncode != 0 or not branch_head:
|
||||
raise SystemExit(26)
|
||||
|
||||
try:
|
||||
ancestry = git_run("merge-base", "--is-ancestor", revision, branch_head)
|
||||
except Exception:
|
||||
raise SystemExit(27)
|
||||
if ancestry.returncode != 0:
|
||||
raise SystemExit(27)
|
||||
|
||||
try:
|
||||
compose = git_run("cat-file", "-e", f"{revision}:{compose_file}")
|
||||
except Exception:
|
||||
raise SystemExit(28)
|
||||
if compose.returncode != 0:
|
||||
raise SystemExit(28)
|
||||
|
||||
try:
|
||||
lock_handle = lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
with lock_handle as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
try:
|
||||
projects = json.loads(mutation_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
if not isinstance(projects, dict):
|
||||
raise SystemExit(22)
|
||||
|
||||
current = projects.get(repository)
|
||||
changed = current is None
|
||||
if current is not None:
|
||||
if not isinstance(current, dict):
|
||||
raise SystemExit(23)
|
||||
allowed_extra_keys = {
|
||||
"env_source",
|
||||
"environment",
|
||||
"gitea_repository_id",
|
||||
"gitea_webhook_secret_file",
|
||||
"validation",
|
||||
}
|
||||
extra_keys = set(current) - set(project)
|
||||
if not extra_keys.issubset(allowed_extra_keys):
|
||||
raise SystemExit(23)
|
||||
canonical_current = {key: current.get(key) for key in project}
|
||||
if canonical_current != project:
|
||||
raise SystemExit(23)
|
||||
env_source = current.get("env_source")
|
||||
if env_source is not None:
|
||||
if (
|
||||
not isinstance(env_source, str)
|
||||
or not env_source.startswith("/srv/example/appdata/")
|
||||
or len(env_source) > 512
|
||||
or ".." in Path(env_source).parts
|
||||
):
|
||||
raise SystemExit(23)
|
||||
environment = current.get("environment")
|
||||
if environment is not None:
|
||||
if not isinstance(environment, dict) or len(environment) > 64:
|
||||
raise SystemExit(23)
|
||||
for key, value in environment.items():
|
||||
if (
|
||||
not isinstance(key, str)
|
||||
or not key
|
||||
or len(key) > 128
|
||||
or not isinstance(value, str)
|
||||
or len(value) > 4096
|
||||
):
|
||||
raise SystemExit(23)
|
||||
runnerless_keys = {
|
||||
"gitea_repository_id",
|
||||
"gitea_webhook_secret_file",
|
||||
"validation",
|
||||
}
|
||||
present_runnerless_keys = runnerless_keys.intersection(current)
|
||||
if present_runnerless_keys and present_runnerless_keys != runnerless_keys:
|
||||
raise SystemExit(23)
|
||||
repository_id = current.get("gitea_repository_id")
|
||||
if repository_id is not None and (
|
||||
not isinstance(repository_id, int)
|
||||
or isinstance(repository_id, bool)
|
||||
or repository_id <= 0
|
||||
):
|
||||
raise SystemExit(23)
|
||||
webhook_secret_file = current.get("gitea_webhook_secret_file")
|
||||
if webhook_secret_file is not None:
|
||||
expected_secret_file = repository.replace("/", "__")
|
||||
if webhook_secret_file != expected_secret_file:
|
||||
raise SystemExit(23)
|
||||
validation = current.get("validation")
|
||||
if validation is not None:
|
||||
expected_validation = {
|
||||
"validation_gate_version": 1,
|
||||
"required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 3600 if repository == "jens/mediarr" else 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
if validation != expected_validation:
|
||||
raise SystemExit(23)
|
||||
|
||||
if changed:
|
||||
backup = mutation_path.with_name(
|
||||
f"projects.json.appops-backup-{int(time.time())}-{os.getpid()}"
|
||||
)
|
||||
try:
|
||||
shutil.copy2(mutation_path, backup)
|
||||
except Exception:
|
||||
raise SystemExit(32)
|
||||
|
||||
updated = dict(projects)
|
||||
updated[repository] = project
|
||||
temporary = mutation_path.with_name(f".{mutation_path.name}.{os.getpid()}.tmp")
|
||||
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
|
||||
try:
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.S_IMODE(info.st_mode))
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(33)
|
||||
|
||||
try:
|
||||
os.replace(temporary, mutation_path)
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(34)
|
||||
|
||||
directory_fd = None
|
||||
try:
|
||||
directory_fd = os.open(str(mutation_path.parent), os.O_RDONLY)
|
||||
os.fsync(directory_fd)
|
||||
except Exception:
|
||||
raise SystemExit(35)
|
||||
finally:
|
||||
if directory_fd is not None:
|
||||
os.close(directory_fd)
|
||||
|
||||
print("registered" if changed else "already-registered", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def _safe_compose_file(value: str) -> str:
|
||||
path = str(value or "").strip().replace("\\", "/")
|
||||
if not path or path.startswith("/") or len(path) > 240:
|
||||
raise ValueError("Compose file must be a bounded repository-relative path")
|
||||
parts = path.split("/")
|
||||
if any(part in {"", ".", ".."} for part in parts):
|
||||
raise ValueError("Compose file path cannot contain empty or traversal segments")
|
||||
if not path.lower().endswith((".yml", ".yaml")):
|
||||
raise ValueError("Compose file must use a .yml or .yaml suffix")
|
||||
return path
|
||||
|
||||
|
||||
class DeploymentTargetRegistry:
|
||||
"""Narrow bridge for exact-SHA Compose deployment target onboarding."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
@staticmethod
|
||||
def _slug(repository: str) -> str:
|
||||
return repository.split("/", 1)[1]
|
||||
|
||||
@staticmethod
|
||||
def default_compose_project(repository: str) -> str:
|
||||
raw = DeploymentTargetRegistry._slug(repository).lower()
|
||||
value = re.sub(r"[^a-z0-9_-]+", "-", raw).strip("-_")
|
||||
if not value:
|
||||
raise ValueError("Repository name cannot be converted to a Compose project")
|
||||
return value[:63]
|
||||
|
||||
@staticmethod
|
||||
def _project(
|
||||
repository: str,
|
||||
*,
|
||||
branch: str,
|
||||
compose_file: str,
|
||||
compose_project: str,
|
||||
containers: list[str],
|
||||
health_timeout: int,
|
||||
verify_oci_revision: bool,
|
||||
) -> dict[str, object]:
|
||||
slug = DeploymentTargetRegistry._slug(repository)
|
||||
return {
|
||||
"branch": branch,
|
||||
"slug": slug,
|
||||
"source_dir": f"/mnt/cache/appdata/gitea-deploy-runners/sources/{slug}",
|
||||
"exact_source": True,
|
||||
"compose_files": [compose_file],
|
||||
"compose_project": compose_project,
|
||||
"containers": containers,
|
||||
"verify_oci_revision": verify_oci_revision,
|
||||
"health_timeout": health_timeout,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def validate(
|
||||
repository: str,
|
||||
revision: str,
|
||||
*,
|
||||
branch: str = "main",
|
||||
compose_file: str = "docker-compose.yml",
|
||||
compose_project: str | None = None,
|
||||
containers: list[str] | None = None,
|
||||
health_timeout: int = 180,
|
||||
) -> tuple[str, str, str, str, str, list[str], int]:
|
||||
normalized_repository = str(repository or "").strip().lower()
|
||||
normalized_revision = str(revision or "").strip().lower()
|
||||
normalized_branch = str(branch or "").strip()
|
||||
if not _REPOSITORY.fullmatch(normalized_repository):
|
||||
raise ValueError("Repository must be a canonical Jens owner/name identifier")
|
||||
slug = DeploymentTargetRegistry._slug(normalized_repository)
|
||||
if not _SLUG.fullmatch(slug):
|
||||
raise ValueError("Repository name cannot be used as a bounded deployment slug")
|
||||
if not _REVISION.fullmatch(normalized_revision):
|
||||
raise ValueError("Revision must be a full lowercase 40-character Git SHA")
|
||||
if (
|
||||
not _BRANCH.fullmatch(normalized_branch)
|
||||
or ".." in normalized_branch
|
||||
or "//" in normalized_branch
|
||||
or normalized_branch.endswith("/")
|
||||
):
|
||||
raise ValueError("Default branch is outside the bounded Git ref contract")
|
||||
normalized_compose_file = _safe_compose_file(compose_file)
|
||||
normalized_project = str(
|
||||
compose_project
|
||||
or DeploymentTargetRegistry.default_compose_project(normalized_repository)
|
||||
).strip().lower()
|
||||
if not _COMPOSE_PROJECT.fullmatch(normalized_project):
|
||||
raise ValueError("Compose project is outside the bounded naming contract")
|
||||
normalized_containers = [str(item or "").strip() for item in (containers or [slug])]
|
||||
if not normalized_containers or len(normalized_containers) > _MAX_CONTAINERS:
|
||||
raise ValueError("Deployment target must contain between 1 and 32 containers")
|
||||
if any(not _CONTAINER.fullmatch(item) for item in normalized_containers):
|
||||
raise ValueError("Container name is outside the bounded naming contract")
|
||||
if len(set(normalized_containers)) != len(normalized_containers):
|
||||
raise ValueError("Container names must be unique")
|
||||
timeout = int(health_timeout)
|
||||
if timeout < 30 or timeout > 900:
|
||||
raise ValueError("Health timeout must be between 30 and 900 seconds")
|
||||
return (
|
||||
normalized_repository,
|
||||
normalized_revision,
|
||||
normalized_branch,
|
||||
normalized_compose_file,
|
||||
normalized_project,
|
||||
normalized_containers,
|
||||
timeout,
|
||||
)
|
||||
|
||||
def register_compose_target(
|
||||
self,
|
||||
repository: str,
|
||||
revision: str,
|
||||
*,
|
||||
branch: str = "main",
|
||||
compose_file: str = "docker-compose.yml",
|
||||
compose_project: str | None = None,
|
||||
containers: list[str] | None = None,
|
||||
health_timeout: int = 180,
|
||||
verify_oci_revision: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
(
|
||||
repository,
|
||||
revision,
|
||||
branch,
|
||||
compose_file,
|
||||
compose_project,
|
||||
containers,
|
||||
health_timeout,
|
||||
) = self.validate(
|
||||
repository,
|
||||
revision,
|
||||
branch=branch,
|
||||
compose_file=compose_file,
|
||||
compose_project=compose_project,
|
||||
containers=containers,
|
||||
health_timeout=health_timeout,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
return {"status": "invalid", "reason": str(exc)}
|
||||
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": redact_message(
|
||||
str(status.get("reason") or status.get("status") or "unavailable")
|
||||
),
|
||||
}
|
||||
controller = dict(status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": "deployment controller is not running",
|
||||
}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not container_id:
|
||||
return {
|
||||
"status": "connector_unavailable",
|
||||
"reason": "Portainer deployment control is unavailable",
|
||||
}
|
||||
|
||||
project = self._project(
|
||||
repository,
|
||||
branch=branch,
|
||||
compose_file=compose_file,
|
||||
compose_project=compose_project,
|
||||
containers=containers,
|
||||
health_timeout=health_timeout,
|
||||
verify_oci_revision=bool(verify_oci_revision),
|
||||
)
|
||||
command = [
|
||||
"python3",
|
||||
"-c",
|
||||
_REGISTER_SCRIPT,
|
||||
repository,
|
||||
revision,
|
||||
branch,
|
||||
json.dumps(project, separators=(",", ":"), sort_keys=True),
|
||||
]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": command,
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {
|
||||
"status": "error",
|
||||
"reason": "deployment target registration could not start",
|
||||
}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 25
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {
|
||||
"status": "error",
|
||||
"reason": "deployment target registration timed out",
|
||||
}
|
||||
if exit_code == 23:
|
||||
return {
|
||||
"status": "conflict",
|
||||
"reason": "repository already has a different deployment target",
|
||||
}
|
||||
preflight_reasons = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
21: "deployment projects registry is not a regular file",
|
||||
22: "deployment projects registry does not contain an object",
|
||||
24: "mounted bare repository is unavailable",
|
||||
25: "exact revision cannot be resolved in the mounted bare repository",
|
||||
26: "default branch cannot be resolved in the mounted bare repository",
|
||||
27: "exact revision is not an ancestor of the default branch in the mounted bare repository",
|
||||
28: "configured Compose file is absent at the exact revision",
|
||||
29: "deployment registry lock file cannot be opened",
|
||||
30: "deployment registry lock cannot be acquired",
|
||||
31: "deployment projects registry cannot be read as JSON",
|
||||
32: "deployment projects registry backup cannot be created",
|
||||
33: "deployment projects registry replacement file cannot be durably written",
|
||||
34: "deployment projects registry replacement cannot be installed atomically",
|
||||
35: "deployment projects registry directory durability sync failed after replacement",
|
||||
36: "deployment projects registry host alias is unavailable or does not match the controller registry",
|
||||
37: "controller project branch does not match the requested default branch",
|
||||
38: "controller project Compose contract is invalid",
|
||||
}
|
||||
if exit_code in preflight_reasons:
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": preflight_reasons[exit_code],
|
||||
}
|
||||
if exit_code != 0:
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": "controller rejected the bounded repository/revision preflight; projects.json was not changed",
|
||||
}
|
||||
return {
|
||||
"status": "registered",
|
||||
"repository": repository,
|
||||
"revision": revision,
|
||||
"branch": branch,
|
||||
"slug": self._slug(repository),
|
||||
"compose_file": compose_file,
|
||||
"compose_project": compose_project,
|
||||
"container": containers[0] if len(containers) == 1 else None,
|
||||
"containers": containers,
|
||||
}
|
||||
|
||||
def register_single_container(
|
||||
self,
|
||||
repository: str,
|
||||
revision: str,
|
||||
*,
|
||||
branch: str = "main",
|
||||
compose_file: str = "docker-compose.yml",
|
||||
) -> dict[str, Any]:
|
||||
slug = str(repository or "").strip().lower().split("/", 1)[-1]
|
||||
return self.register_compose_target(
|
||||
repository,
|
||||
revision,
|
||||
branch=branch,
|
||||
compose_file=compose_file,
|
||||
containers=[slug],
|
||||
)
|
||||
@@ -0,0 +1,327 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..db import session as db_session
|
||||
from ..db.models import Policy
|
||||
from .audit_logger import log_event
|
||||
from .gitea_project_auto_registration import GiteaProjectAutoRegistration
|
||||
from .inventory_discovery import run_discovery
|
||||
from .mcp_hub_deployment_reconciliation import McpHubDeploymentReconciliation
|
||||
from .media_codec_dashboard_deployment_reconciliation import (
|
||||
MediaCodecDashboardDeploymentReconciliation,
|
||||
)
|
||||
from .privatemuse_deployment_enrollment import PrivateMuseDeploymentEnrollment
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MCP_HUB_APPLICATION_ID = "itworx-mcp-hub"
|
||||
_MCP_HUB_REPOSITORY = "jens/itworx_mcp_hub"
|
||||
_MCP_HUB_DEPLOYMENT_ACTIONS = ("deployment.apply", "deployment.rollback")
|
||||
|
||||
|
||||
def _reconcile_mcp_hub_exact_policies(session: Session) -> dict[str, object]:
|
||||
"""Create only the exact MCP Hub deployment policies after state proof."""
|
||||
|
||||
existing_by_action: dict[str, Policy | None] = {}
|
||||
for action in _MCP_HUB_DEPLOYMENT_ACTIONS:
|
||||
matches = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == action) & (Policy.target == _MCP_HUB_APPLICATION_ID)
|
||||
)
|
||||
).all()
|
||||
if len(matches) > 1:
|
||||
return {
|
||||
"status": "conflict",
|
||||
"action": action,
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
existing = matches[0] if matches else None
|
||||
if existing is not None and (
|
||||
existing.manual != "allowed"
|
||||
or existing.voice != "blocked"
|
||||
or existing.risk != "high"
|
||||
):
|
||||
return {
|
||||
"status": "conflict",
|
||||
"action": action,
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
existing_by_action[action] = existing
|
||||
|
||||
policy_ids: list[str] = []
|
||||
created = 0
|
||||
for action in _MCP_HUB_DEPLOYMENT_ACTIONS:
|
||||
existing = existing_by_action[action]
|
||||
if existing is not None:
|
||||
policy_ids.append(existing.id)
|
||||
continue
|
||||
suffix = action.rsplit(".", 1)[-1]
|
||||
policy_id = f"mcp-hub-reconciled-{suffix}"
|
||||
if session.get(Policy, policy_id) is not None:
|
||||
return {
|
||||
"status": "conflict",
|
||||
"action": action,
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
session.add(
|
||||
Policy(
|
||||
id=policy_id,
|
||||
action=action,
|
||||
target=_MCP_HUB_APPLICATION_ID,
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason=(
|
||||
"Exact MCP Hub deployment policy created only after "
|
||||
"fail-closed runtime state reconciliation."
|
||||
),
|
||||
provenance_json={
|
||||
"createdBy": "mcp-hub-state-reconciliation/v1",
|
||||
"repository": _MCP_HUB_REPOSITORY,
|
||||
"policyTemplateVersion": "exact-target/v1",
|
||||
},
|
||||
)
|
||||
)
|
||||
policy_ids.append(policy_id)
|
||||
created += 1
|
||||
session.commit()
|
||||
return {"status": "ready", "created": created, "policyIds": policy_ids}
|
||||
|
||||
|
||||
def _ensure_mcp_hub_exact_policies() -> dict[str, object]:
|
||||
with Session(db_session.engine) as session:
|
||||
return _reconcile_mcp_hub_exact_policies(session)
|
||||
|
||||
|
||||
def _audit_privatemuse_enrollment(enrollment: dict[str, object]) -> None:
|
||||
status = str(enrollment.get("status") or "error")
|
||||
failure_reason = str(enrollment.get("reason") or "")
|
||||
reason = "Bounded startup reconciliation for the existing PrivateMuse deployment target."
|
||||
if failure_reason:
|
||||
reason = f"{reason} Enrollment result: {status}; {failure_reason}"
|
||||
with Session(db_session.engine) as session:
|
||||
log_event(
|
||||
session,
|
||||
source="system",
|
||||
identity="appops-startup",
|
||||
action_id="deployment.target.enroll",
|
||||
target="jens/privatemuse",
|
||||
target_type="deployment-target",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
reason=reason,
|
||||
result="success" if status == "enrolled" else "failed",
|
||||
after_state={
|
||||
"status": status,
|
||||
"repository": enrollment.get("repository", "jens/privatemuse"),
|
||||
"legacyRevision": enrollment.get("legacy_revision", ""),
|
||||
"composeProject": enrollment.get("compose_project", ""),
|
||||
"containers": enrollment.get("containers", []),
|
||||
},
|
||||
redacted_metadata={"failureReason": failure_reason} if failure_reason else None,
|
||||
)
|
||||
|
||||
|
||||
def _audit_mcp_hub_reconciliation(result: dict[str, object]) -> None:
|
||||
status = str(result.get("status") or "error")
|
||||
failure_reason = str(result.get("reason") or "")
|
||||
reason = "Fail-closed startup reconciliation for the verified MCP Hub deployment state."
|
||||
if failure_reason:
|
||||
reason = f"{reason} Reconciliation result: {status}; {failure_reason}"
|
||||
with Session(db_session.engine) as session:
|
||||
log_event(
|
||||
session,
|
||||
source="system",
|
||||
identity="appops-startup",
|
||||
action_id="deployment.state.reconcile",
|
||||
target="jens/itworx_mcp_hub",
|
||||
target_type="deployment-target",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
reason=reason,
|
||||
result="success" if status == "reconciled" else "failed",
|
||||
after_state={
|
||||
"status": status,
|
||||
"repository": result.get("repository", "jens/itworx_mcp_hub"),
|
||||
"composeProject": result.get("composeProject", "itworx-mcp-hub"),
|
||||
"policyStatus": result.get("policyStatus", "not_checked"),
|
||||
"policyIds": result.get("policyIds", []),
|
||||
"policiesCreated": result.get("policiesCreated", 0),
|
||||
},
|
||||
redacted_metadata={"failureReason": failure_reason} if failure_reason else None,
|
||||
)
|
||||
|
||||
|
||||
def _audit_media_codec_dashboard_reconciliation(result: dict[str, object]) -> None:
|
||||
status = str(result.get("status") or "error")
|
||||
failure_reason = str(result.get("reason") or "")
|
||||
reason = "Fail-closed startup reconciliation for the verified Media Codec Dashboard deployment state."
|
||||
if failure_reason:
|
||||
reason = f"{reason} Reconciliation result: {status}; {failure_reason}"
|
||||
with Session(db_session.engine) as session:
|
||||
log_event(
|
||||
session,
|
||||
source="system",
|
||||
identity="appops-startup",
|
||||
action_id="deployment.state.reconcile",
|
||||
target="jens/media-codec-dashboard",
|
||||
target_type="deployment-target",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
reason=reason,
|
||||
result="success" if status == "reconciled" else "failed",
|
||||
after_state={
|
||||
"status": status,
|
||||
"repository": result.get("repository", "jens/media-codec-dashboard"),
|
||||
"liveRevision": result.get("live_revision", ""),
|
||||
"previousRecordedRevision": result.get("previous_recorded_revision", ""),
|
||||
"container": result.get("container", ""),
|
||||
},
|
||||
redacted_metadata={"failureReason": failure_reason} if failure_reason else None,
|
||||
)
|
||||
|
||||
|
||||
def _sync_gitea_projects() -> dict[str, object]:
|
||||
with Session(db_session.engine) as session:
|
||||
result = GiteaProjectAutoRegistration().sync(session)
|
||||
summary = result.get("summary") if isinstance(result.get("summary"), dict) else {}
|
||||
changed = int(summary.get("changedThisSync", 0) or 0)
|
||||
if changed:
|
||||
log_event(
|
||||
session,
|
||||
source="system",
|
||||
identity="appops-project-sync",
|
||||
action_id="deployment.targets.reconcile",
|
||||
target="jens/*",
|
||||
target_type="deployment-project-catalog",
|
||||
risk="medium",
|
||||
policy_decision="allowed",
|
||||
reason="Automatic Gitea project registration reconciled changed owned repositories.",
|
||||
result="success" if result.get("status") == "ok" else "failed",
|
||||
after_state={
|
||||
"status": result.get("status"),
|
||||
"total": summary.get("total", 0),
|
||||
"registered": summary.get("registered", 0),
|
||||
"knownUnconfigured": summary.get("knownUnconfigured", 0),
|
||||
"conflicts": summary.get("conflicts", 0),
|
||||
"errors": summary.get("errors", 0),
|
||||
"changed": changed,
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _run_gitea_project_sync() -> None:
|
||||
try:
|
||||
result = await asyncio.to_thread(_sync_gitea_projects)
|
||||
summary = result.get("summary") if isinstance(result.get("summary"), dict) else {}
|
||||
if result.get("status") != "ok":
|
||||
logger.warning(
|
||||
"Gitea project registration sync degraded: status=%s reason=%s",
|
||||
result.get("status"),
|
||||
result.get("reason", "unspecified"),
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Gitea project registration sync complete: total=%s registered=%s changed=%s",
|
||||
summary.get("total", 0),
|
||||
summary.get("registered", 0),
|
||||
summary.get("changedThisSync", 0),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - catalog sync must never take down AppOps.
|
||||
logger.exception("Gitea project registration sync failed")
|
||||
|
||||
|
||||
async def run_periodic_discovery(interval_seconds: int) -> None:
|
||||
"""Reconcile deployment project catalog, bounded target state and inventory."""
|
||||
await _run_gitea_project_sync()
|
||||
|
||||
try:
|
||||
enrollment = await asyncio.to_thread(PrivateMuseDeploymentEnrollment().run)
|
||||
await asyncio.to_thread(_audit_privatemuse_enrollment, enrollment)
|
||||
if enrollment.get("status") != "enrolled":
|
||||
logger.warning(
|
||||
"PrivateMuse deployment enrollment did not complete: status=%s reason=%s",
|
||||
enrollment.get("status"),
|
||||
enrollment.get("reason", "unspecified"),
|
||||
)
|
||||
else:
|
||||
logger.info("PrivateMuse deployment target is enrolled")
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - enrollment failure must not take down AppOps.
|
||||
logger.exception("PrivateMuse deployment enrollment failed")
|
||||
|
||||
try:
|
||||
reconciliation = await asyncio.to_thread(McpHubDeploymentReconciliation().run)
|
||||
if reconciliation.get("status") == "reconciled":
|
||||
policy_result = await asyncio.to_thread(_ensure_mcp_hub_exact_policies)
|
||||
reconciliation = {
|
||||
**reconciliation,
|
||||
"policyStatus": policy_result.get("status", "error"),
|
||||
"policyIds": policy_result.get("policyIds", []),
|
||||
"policiesCreated": policy_result.get("created", 0),
|
||||
}
|
||||
if policy_result.get("status") != "ready":
|
||||
reconciliation = {
|
||||
**reconciliation,
|
||||
"status": "policy_conflict",
|
||||
"reason": "exact MCP Hub deployment policy reconciliation failed closed",
|
||||
}
|
||||
await asyncio.to_thread(_audit_mcp_hub_reconciliation, reconciliation)
|
||||
if reconciliation.get("status") != "reconciled":
|
||||
logger.warning(
|
||||
"MCP Hub deployment reconciliation did not complete: status=%s reason=%s",
|
||||
reconciliation.get("status"),
|
||||
reconciliation.get("reason", "unspecified"),
|
||||
)
|
||||
else:
|
||||
logger.info("MCP Hub deployment state is reconciled")
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - reconciliation failure must not take down AppOps.
|
||||
logger.exception("MCP Hub deployment reconciliation failed")
|
||||
|
||||
try:
|
||||
reconciliation = await asyncio.to_thread(
|
||||
MediaCodecDashboardDeploymentReconciliation().run
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
_audit_media_codec_dashboard_reconciliation, reconciliation
|
||||
)
|
||||
if reconciliation.get("status") != "reconciled":
|
||||
logger.warning(
|
||||
"Media Codec Dashboard deployment reconciliation did not complete: status=%s reason=%s",
|
||||
reconciliation.get("status"),
|
||||
reconciliation.get("reason", "unspecified"),
|
||||
)
|
||||
else:
|
||||
logger.info("Media Codec Dashboard deployment state is reconciled")
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - reconciliation failure must not take down AppOps.
|
||||
logger.exception("Media Codec Dashboard deployment reconciliation failed")
|
||||
|
||||
if interval_seconds <= 0:
|
||||
return
|
||||
|
||||
await asyncio.sleep(min(10, interval_seconds))
|
||||
while True:
|
||||
try:
|
||||
await asyncio.to_thread(run_discovery)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - discovery must never take down the gateway.
|
||||
logger.exception("Scheduled inventory discovery failed")
|
||||
await _run_gitea_project_sync()
|
||||
await asyncio.sleep(interval_seconds)
|
||||
@@ -0,0 +1,986 @@
|
||||
"""Bounded enrollment orchestration for a new managed application identity.
|
||||
|
||||
The plane is intentionally not a generic infrastructure API. It creates a
|
||||
single controller target from a trusted profile, generates write-only
|
||||
credentials, and only then creates exact-target deployment policies. Every
|
||||
resource identifier and error returned from this module is sanitized.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..connectors.authentik import AuthentikConnector
|
||||
from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting, Enrollment, EnrollmentJob, Policy
|
||||
from ..schemas.enrollment import EnrollmentPlanV1, EnrollmentTrustConfigV1
|
||||
from .audit_logger import log_event
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
from .enrollment_profiles import PROFILE_VERSION, profile_contract, profile_summary
|
||||
from .secret_store import (
|
||||
delete_secret,
|
||||
get_secret,
|
||||
has_secret,
|
||||
secret_storage_status,
|
||||
set_secret,
|
||||
)
|
||||
|
||||
TRUST_SETTING_KEY = "deployment_enrollment:trust"
|
||||
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
_TERMINAL = {"ready", "failed", "compensation_required", "preflight_failed"}
|
||||
_CONTROLLER_EVIDENCE = {"status", "targetId", "repository", "applicationId", "appdataPath"}
|
||||
_PROXY_EVIDENCE = {"success", "status", "hostId", "hostname"}
|
||||
_OIDC_EVIDENCE = {"success", "status", "applicationId", "providerId", "clientSecretCreated"}
|
||||
|
||||
|
||||
class EnrollmentError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _canonical(value: EnrollmentPlanV1) -> tuple[dict[str, Any], str]:
|
||||
payload = value.model_dump(mode="json")
|
||||
digest = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
return payload, digest
|
||||
|
||||
|
||||
def _idempotency_digest(operation: str, value: str) -> str:
|
||||
if not _IDEMPOTENCY_KEY.fullmatch(value):
|
||||
raise EnrollmentError("idempotency key must contain 8 to 128 safe characters")
|
||||
return hashlib.sha256(f"{operation}:{value}".encode()).hexdigest()
|
||||
|
||||
|
||||
def _check(
|
||||
code: str,
|
||||
status: Literal["passed", "blocked", "warning", "unknown"],
|
||||
detail: str,
|
||||
) -> dict[str, str]:
|
||||
return {"code": code, "status": status, "detail": detail}
|
||||
|
||||
|
||||
def _secret_ref(application_id: str, kind: str) -> str:
|
||||
return f"ENROLL_{application_id.upper().replace('-', '_')}_{kind}"
|
||||
|
||||
|
||||
def _credential() -> str:
|
||||
# 64 URL-safe characters provide a high-entropy printable HMAC key while
|
||||
# remaining suitable for the controller's mode-0600 credential file.
|
||||
return secrets.token_urlsafe(48)
|
||||
|
||||
|
||||
def _fingerprint(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _enrollment_secret_store_ready(storage: dict[str, Any]) -> bool:
|
||||
"""Enrollment always needs a configured production-grade key.
|
||||
|
||||
The general secret store intentionally supports a development fallback for
|
||||
offline UI work. Enrollment generates controller and OIDC credentials, so
|
||||
it must never accept that deterministic fallback as a write target.
|
||||
"""
|
||||
return bool(storage.get("productionReady"))
|
||||
|
||||
|
||||
class EnrollmentPlane:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
broker: DeploymentBrokerClient | None = None,
|
||||
portainer: PortainerConnector | None = None,
|
||||
npm: NginxProxyManagerConnector | None = None,
|
||||
authentik: AuthentikConnector | None = None,
|
||||
):
|
||||
self.broker = broker or DeploymentBrokerClient()
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
self.npm = npm or NginxProxyManagerConnector()
|
||||
self.authentik = authentik or AuthentikConnector()
|
||||
|
||||
@staticmethod
|
||||
def get_trust_config(session: Session) -> EnrollmentTrustConfigV1 | None:
|
||||
row = session.get(AppSetting, TRUST_SETTING_KEY)
|
||||
if not row or row.is_secret:
|
||||
return None
|
||||
try:
|
||||
return EnrollmentTrustConfigV1.model_validate(row.value_json)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def set_trust_config(session: Session, value: EnrollmentTrustConfigV1) -> dict[str, Any]:
|
||||
row = session.get(AppSetting, TRUST_SETTING_KEY) or AppSetting(
|
||||
key=TRUST_SETTING_KEY, is_secret=False
|
||||
)
|
||||
row.value_json = value.model_dump(mode="json")
|
||||
row.is_secret = False
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return EnrollmentPlane.trust_status(session)
|
||||
|
||||
@staticmethod
|
||||
def trust_status(session: Session) -> dict[str, Any]:
|
||||
trust = EnrollmentPlane.get_trust_config(session)
|
||||
storage = secret_storage_status()
|
||||
if trust is None:
|
||||
return {
|
||||
"status": "needs_setup",
|
||||
"blockers": ["enrollment_trust_not_configured"],
|
||||
"configured": False,
|
||||
"secretStore": storage,
|
||||
}
|
||||
return {
|
||||
"status": "ready" if _enrollment_secret_store_ready(storage) else "blocked",
|
||||
"blockers": [] if _enrollment_secret_store_ready(storage) else ["secret_store_not_production_ready"],
|
||||
"configured": True,
|
||||
"controllerIdentity": trust.controller_identity,
|
||||
"trustedAppdataRoot": trust.trusted_appdata_root,
|
||||
"approvedNetworks": list(trust.approved_networks),
|
||||
"allowedDomainSuffixes": list(trust.allowed_domain_suffixes),
|
||||
"autonomousProfiles": list(trust.autonomous_profiles),
|
||||
"repopulseProxyAccessListId": trust.repopulse_proxy_access_list_id,
|
||||
"oidcConfigured": bool(trust.oidc_issuer and trust.administrator_group),
|
||||
"bootstrapSecretConfigured": has_secret(session, trust.bootstrap_secret_ref),
|
||||
"secretStore": storage,
|
||||
}
|
||||
|
||||
def capabilities(self, session: Session) -> dict[str, Any]:
|
||||
trust = self.get_trust_config(session)
|
||||
storage = secret_storage_status()
|
||||
configured = trust is not None
|
||||
bootstrap = bool(trust and has_secret(session, trust.bootstrap_secret_ref))
|
||||
controller_capability: dict[str, Any] = {}
|
||||
if bootstrap and self.broker.is_configured() and trust is not None:
|
||||
bootstrap_secret = get_secret(session, trust.bootstrap_secret_ref, touch=False)
|
||||
if bootstrap_secret:
|
||||
controller_capability = self.broker.enrollment_capabilities(secret=bootstrap_secret)
|
||||
broker_ready = bool(
|
||||
bootstrap
|
||||
and self.broker.is_configured()
|
||||
and controller_capability.get("status") == "ready"
|
||||
)
|
||||
npm_ready = self.npm.is_configured()
|
||||
oidc_configured = bool(
|
||||
trust
|
||||
and trust.oidc_issuer
|
||||
and trust.administrator_group
|
||||
and trust.support_agent_group
|
||||
and trust.read_only_group
|
||||
and self.authentik.is_configured()
|
||||
)
|
||||
certificate_ready = bool(npm_ready)
|
||||
|
||||
def item(ready: bool, code: str) -> dict[str, str]:
|
||||
return {"status": "ready" if ready else "unavailable", "blockerCode": "" if ready else code}
|
||||
|
||||
return {
|
||||
"apiVersion": "itworx.enrollment-capabilities/v1",
|
||||
"trust": item(configured, "enrollment_trust_not_configured"),
|
||||
"targetEnrollment": item(broker_ready, "deployment_executor_unavailable"),
|
||||
"secretBootstrap": item(_enrollment_secret_store_ready(storage), "secret_store_not_production_ready"),
|
||||
"runtimeDeployment": item(broker_ready, "deployment_executor_unavailable"),
|
||||
"reverseProxyProvisioning": item(npm_ready, "npm_credentials_missing"),
|
||||
"tlsProvisioning": item(certificate_ready, "certificate_provisioning_unavailable"),
|
||||
"oidcProvisioning": item(oidc_configured, "oidc_profile_unavailable"),
|
||||
"policyProvisioning": item(configured, "policy_template_unavailable"),
|
||||
"rollback": item(broker_ready, "deployment_executor_unavailable"),
|
||||
"verification": item(broker_ready, "deployment_executor_unavailable"),
|
||||
"controller": {
|
||||
"status": str(controller_capability.get("status") or "unavailable"),
|
||||
"reason": str(controller_capability.get("reason") or "deployment_executor_unavailable"),
|
||||
},
|
||||
"profiles": [profile_summary(name) for name in ("internal-webapp", "internal-webapp-oidc", "internal-service", "repopulse-authenticated")],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _allowed_hostname(hostname: str, trust: EnrollmentTrustConfigV1) -> bool:
|
||||
return any(hostname == suffix or hostname.endswith(f".{suffix}") for suffix in trust.allowed_domain_suffixes)
|
||||
|
||||
@staticmethod
|
||||
def _existing(session: Session, application_id: str, repository: str) -> tuple[Enrollment | None, Enrollment | None]:
|
||||
by_application = session.exec(
|
||||
select(Enrollment).where(Enrollment.application_id == application_id)
|
||||
).first()
|
||||
by_repository = session.exec(
|
||||
select(Enrollment).where(Enrollment.repository == repository)
|
||||
).first()
|
||||
return by_application, by_repository
|
||||
|
||||
def _controller_payload(
|
||||
self,
|
||||
plan: EnrollmentPlanV1,
|
||||
*,
|
||||
enrollment_id: str,
|
||||
target_credential: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
profile = profile_contract(plan.profile)
|
||||
assert profile is not None # validated before this method is called
|
||||
payload: dict[str, Any] = {
|
||||
"api_version": "itworx.controller-enrollment/v1",
|
||||
"enrollment_id": enrollment_id,
|
||||
"application_id": plan.application_id,
|
||||
"repository": plan.repository,
|
||||
"revision": plan.revision,
|
||||
"profile": plan.profile,
|
||||
"container_name": plan.target.container_name,
|
||||
"compose_file": plan.target.compose_file,
|
||||
"appdata_path": plan.appdata_path,
|
||||
"data_path": plan.data_path,
|
||||
"runtime_uid": plan.storage.runtime_uid,
|
||||
"runtime_gid": plan.storage.runtime_gid,
|
||||
"network": plan.network.docker_network,
|
||||
"internal_port": plan.network.internal_port,
|
||||
"proxy_hostname": plan.exposure.hostname if plan.exposure else None,
|
||||
"image_repository": plan.artifact.image_repository,
|
||||
"image_tag": plan.artifact.image_tag,
|
||||
"health_path": plan.artifact.health_path,
|
||||
"runtime_environment_keys": list(profile["runtimeEnvironmentKeys"]),
|
||||
}
|
||||
if target_credential is not None:
|
||||
payload["target_credential"] = target_credential
|
||||
return payload
|
||||
|
||||
def preflight(self, session: Session, plan: EnrollmentPlanV1) -> dict[str, Any]:
|
||||
checks: list[dict[str, str]] = []
|
||||
trust = self.get_trust_config(session)
|
||||
profile = profile_contract(plan.profile)
|
||||
if trust is None:
|
||||
checks.append(_check("enrollment_trust_not_configured", "blocked", "Root enrollment trust boundaries are not configured."))
|
||||
else:
|
||||
checks.append(_check("enrollment_trust", "passed", "Root-owned enrollment boundaries are configured."))
|
||||
if plan.target.target_identity == trust.controller_identity:
|
||||
checks.append(_check("controller_identity", "passed", "Requested controller matches the trusted enrollment target."))
|
||||
else:
|
||||
checks.append(_check("deployment_executor_unavailable", "blocked", "Requested controller is not the configured enrollment target."))
|
||||
if plan.network.docker_network in trust.approved_networks:
|
||||
checks.append(_check("network", "passed", "Requested Docker network is within the trusted profile boundary."))
|
||||
else:
|
||||
checks.append(_check("network_unavailable", "blocked", "Requested Docker network is not approved for enrollment."))
|
||||
if plan.profile in trust.autonomous_profiles:
|
||||
checks.append(_check("autonomous_profile", "passed", "Deployment profile is approved for autonomous enrollment."))
|
||||
else:
|
||||
checks.append(_check("policy_template_unavailable", "blocked", "Deployment profile is not approved for autonomous enrollment."))
|
||||
|
||||
if profile is None:
|
||||
checks.append(_check("policy_template_unavailable", "blocked", "Requested server-side deployment profile is unavailable."))
|
||||
else:
|
||||
checks.append(_check("deployment_profile", "passed", "Server-side deployment profile is available."))
|
||||
|
||||
if plan.profile == "repopulse-authenticated":
|
||||
gitea_token = get_secret(session, "REPOPULSE_GITEA_TOKEN", touch=False)
|
||||
checks.append(_check(
|
||||
"repopulse_gitea_token_unavailable",
|
||||
"passed" if gitea_token else "blocked",
|
||||
"The existing encrypted RepoPulse Gitea token is available by reference."
|
||||
if gitea_token else "The existing encrypted RepoPulse Gitea token is unavailable.",
|
||||
))
|
||||
if has_secret(session, _secret_ref("repopulse", "MAINTENANCE_TOKEN")):
|
||||
checks.append(_check(
|
||||
"repopulse_maintenance_token_collision", "blocked",
|
||||
"An existing write-only RepoPulse maintenance token cannot be adopted by a new enrollment.",
|
||||
))
|
||||
else:
|
||||
checks.append(_check(
|
||||
"repopulse_maintenance_token", "passed",
|
||||
"A new write-only RepoPulse maintenance token can be generated for this enrollment.",
|
||||
))
|
||||
if trust is None or trust.repopulse_proxy_access_list_id is None:
|
||||
checks.append(_check(
|
||||
"repopulse_access_list_unavailable", "blocked",
|
||||
"A positive root-owned RepoPulse NPM access-list ID is not configured.",
|
||||
))
|
||||
else:
|
||||
verify_access_list = getattr(self.npm, "validate_managed_access_list", None)
|
||||
access_list = verify_access_list(trust.repopulse_proxy_access_list_id) if callable(verify_access_list) else {}
|
||||
verified = (
|
||||
access_list.get("success") is True
|
||||
and access_list.get("status") == "verified"
|
||||
and access_list.get("accessListId") == trust.repopulse_proxy_access_list_id
|
||||
)
|
||||
checks.append(_check(
|
||||
"repopulse_access_list_unavailable",
|
||||
"passed" if verified else "blocked",
|
||||
"The root-owned RepoPulse NPM access list has active authorization entries and requires all checks."
|
||||
if verified else "The root-owned RepoPulse NPM access list could not be verified.",
|
||||
))
|
||||
|
||||
by_application, by_repository = self._existing(session, plan.application_id, plan.repository)
|
||||
if by_application:
|
||||
status = "warning" if by_application.plan_sha256 == _canonical(plan)[1] else "blocked"
|
||||
code = "enrollment_replay" if status == "warning" else "target_already_exists"
|
||||
checks.append(_check(code, status, "This application identity is already enrolled."))
|
||||
else:
|
||||
checks.append(_check("application_identity", "passed", "Application identity is not yet enrolled."))
|
||||
if by_repository and (not by_application or by_repository.id != by_application.id):
|
||||
checks.append(_check("target_already_exists", "blocked", "Repository is already bound to a different managed application."))
|
||||
|
||||
containers = self.portainer.list_containers()
|
||||
if containers.get("status") != "ok":
|
||||
checks.append(_check("container_inventory_unavailable", "blocked", "Portainer inventory is unavailable for collision protection."))
|
||||
else:
|
||||
names = {str(item.get("name") or "").casefold() for item in containers.get("containers", [])}
|
||||
if plan.target.container_name.casefold() in names:
|
||||
checks.append(_check("unmanaged_resource_collision", "blocked", "Container identity already exists and cannot be claimed by enrollment."))
|
||||
else:
|
||||
checks.append(_check("container_identity", "passed", "Container identity is free."))
|
||||
|
||||
if plan.exposure:
|
||||
if trust and self._allowed_hostname(plan.exposure.hostname, trust):
|
||||
checks.append(_check("hostname_suffix", "passed", "Hostname is inside an allowed domain suffix."))
|
||||
else:
|
||||
checks.append(_check("hostname_not_allowed", "blocked", "Hostname is outside the configured allowed suffixes."))
|
||||
hosts = self.npm.proxy_hosts()
|
||||
if not hosts.get("success"):
|
||||
checks.append(_check("npm_credentials_missing" if not self.npm.is_configured() else "npm_executor_unavailable", "blocked", "NPM inventory is unavailable for takeover protection."))
|
||||
else:
|
||||
existing = [item for item in hosts.get("items", []) if plan.exposure.hostname in {str(value).lower() for value in item.get("domainNames", [])}]
|
||||
checks.append(_check("hostname_collision", "blocked" if existing else "passed", "Hostname is already owned and cannot be taken over." if existing else "Hostname is free in NPM."))
|
||||
|
||||
if plan.authentication.mode == "native-oidc":
|
||||
groups_ready = bool(trust and trust.administrator_group and trust.support_agent_group and trust.read_only_group and trust.oidc_issuer)
|
||||
if not groups_ready or not self.authentik.is_configured():
|
||||
checks.append(_check("oidc_profile_unavailable", "blocked", "Configured OIDC profile or Authentik role groups are unavailable."))
|
||||
else:
|
||||
assert trust is not None and plan.authentication.roles is not None
|
||||
expected_roles = {
|
||||
"administrator": trust.administrator_group,
|
||||
"support_agent": trust.support_agent_group,
|
||||
"read_only": trust.read_only_group,
|
||||
}
|
||||
requested_roles = plan.authentication.roles.model_dump(mode="json")
|
||||
checks.append(_check(
|
||||
"oidc_role_model",
|
||||
"passed" if requested_roles == expected_roles else "blocked",
|
||||
"Requested application roles match the root-owned Authentik role model."
|
||||
if requested_roles == expected_roles
|
||||
else "Requested application roles do not match the root-owned Authentik role model.",
|
||||
))
|
||||
auth_status = self.authentik.test_connection()
|
||||
if auth_status.get("status") != "connected":
|
||||
checks.append(_check("oidc_provider", "blocked", "Authentik is unavailable."))
|
||||
else:
|
||||
template = self.authentik.deployment_oidc_template_status(
|
||||
role_claim=trust.oidc_role_claim
|
||||
)
|
||||
if template.get("status") != "ready":
|
||||
checks.append(_check("oidc_profile_unavailable", "blocked", "The server-owned Authentik OIDC template is incomplete."))
|
||||
applications = self.authentik.applications()
|
||||
groups = self.authentik.groups()
|
||||
if not applications.get("success") or not groups.get("success"):
|
||||
checks.append(_check("oidc_provider", "blocked", "Authentik applications cannot be inspected for takeover protection."))
|
||||
elif any(str(item.get("slug") or "") == plan.application_id for item in applications.get("items", [])):
|
||||
checks.append(_check("unmanaged_resource_collision", "blocked", "An existing Authentik application cannot be claimed by a new enrollment."))
|
||||
else:
|
||||
checks.append(_check("oidc_provider", "passed", "Authentik is available and the application identity is free."))
|
||||
required_groups = {
|
||||
trust.administrator_group if trust else "",
|
||||
trust.support_agent_group if trust else "",
|
||||
trust.read_only_group if trust else "",
|
||||
}
|
||||
found_groups = {str(item.get("name") or "") for item in groups.get("items", [])}
|
||||
checks.append(_check(
|
||||
"oidc_role_groups",
|
||||
"passed" if required_groups.issubset(found_groups) else "blocked",
|
||||
"Configured Authentik role groups are available." if required_groups.issubset(found_groups) else "One or more configured Authentik role groups are missing.",
|
||||
))
|
||||
|
||||
storage = secret_storage_status()
|
||||
storage_ready = _enrollment_secret_store_ready(storage)
|
||||
checks.append(_check("secret_store", "passed" if storage_ready else "blocked", "Production-grade write-only secret storage is ready." if storage_ready else "Enrollment requires APPOPS_SECRET_KEY from a protected environment variable or file; the development fallback is not permitted."))
|
||||
if trust:
|
||||
checks.append(_check("controller_bootstrap_secret", "passed" if has_secret(session, trust.bootstrap_secret_ref) else "blocked", "Controller bootstrap secret is available by reference." if has_secret(session, trust.bootstrap_secret_ref) else "Controller bootstrap secret reference is not configured."))
|
||||
if has_secret(session, trust.bootstrap_secret_ref):
|
||||
controller = self.broker.enrollment_preflight(
|
||||
self._controller_payload(plan, enrollment_id="enr-00000000000000000000"),
|
||||
secret=get_secret(session, trust.bootstrap_secret_ref, touch=False) or "",
|
||||
)
|
||||
code = str(controller.get("reason") or "deployment_executor_unavailable")
|
||||
checks.append(_check("controller_target", "passed" if controller.get("status") in {"preflight_ok", "ready"} else "blocked", "Controller accepted the bounded enrollment preflight." if controller.get("status") in {"preflight_ok", "ready"} else f"Controller enrollment preflight is unavailable ({code})."))
|
||||
else:
|
||||
checks.append(_check("controller_target", "blocked", "Controller bootstrap credential is unavailable."))
|
||||
|
||||
blockers = [item["code"] for item in checks if item["status"] == "blocked"]
|
||||
warnings = [item["code"] for item in checks if item["status"] == "warning"]
|
||||
planned_resources = {
|
||||
"applicationId": plan.application_id,
|
||||
"controllerTarget": plan.target.target_identity,
|
||||
"appdataPath": plan.appdata_path,
|
||||
"dataPath": plan.data_path,
|
||||
"network": plan.network.docker_network,
|
||||
"hostname": plan.exposure.hostname if plan.exposure else None,
|
||||
"oidc": plan.authentication.mode == "native-oidc",
|
||||
}
|
||||
return {
|
||||
"status": "ready" if not blockers else "blocked",
|
||||
"blockers": blockers,
|
||||
"warnings": warnings,
|
||||
"checks": checks,
|
||||
"plannedResources": planned_resources,
|
||||
"plannedMutations": [] if blockers else ["controller_target_register", "secret_generate", *( ["reverse_proxy_ensure"] if plan.exposure else []), *( ["oidc_application_ensure"] if plan.authentication.mode == "native-oidc" else []), *( ["runtime_environment_seal"] if profile and profile["runtimeEnvironmentKeys"] else []), "exact_policy_install"],
|
||||
"plannedSecretRefs": [
|
||||
_secret_ref(plan.application_id, "TARGET_HMAC"),
|
||||
*( [_secret_ref(plan.application_id, "OIDC_CLIENT_SECRET")] if plan.authentication.mode == "native-oidc" else []),
|
||||
*( ["REPOPULSE_GITEA_TOKEN", _secret_ref("repopulse", "MAINTENANCE_TOKEN")] if plan.profile == "repopulse-authenticated" else []),
|
||||
],
|
||||
"plannedPolicies": [f"enrollment-{plan.application_id}-deployment-apply", f"enrollment-{plan.application_id}-deployment-rollback"],
|
||||
}
|
||||
|
||||
def _create_enrollment(self, session: Session, plan: EnrollmentPlanV1, plan_sha256: str, trust: EnrollmentTrustConfigV1) -> Enrollment:
|
||||
enrollment = Enrollment(
|
||||
id=f"enr-{uuid.uuid4().hex[:20]}",
|
||||
application_id=plan.application_id,
|
||||
repository=plan.repository,
|
||||
revision=plan.revision,
|
||||
profile=plan.profile,
|
||||
controller_identity=trust.controller_identity,
|
||||
plan_sha256=plan_sha256,
|
||||
status="planned",
|
||||
target_secret_ref=_secret_ref(plan.application_id, "TARGET_HMAC"),
|
||||
oidc_secret_ref=_secret_ref(plan.application_id, "OIDC_CLIENT_SECRET") if plan.authentication.mode == "native-oidc" else None,
|
||||
plan_json=plan.model_dump(mode="json"),
|
||||
provenance_json={"profileVersion": PROFILE_VERSION, "createdBy": "enrollment-plane/v1", "sourceRevision": plan.revision},
|
||||
)
|
||||
session.add(enrollment)
|
||||
session.commit()
|
||||
session.refresh(enrollment)
|
||||
return enrollment
|
||||
|
||||
def _create_job(self, session: Session, enrollment: Enrollment, idempotency_key: str) -> EnrollmentJob:
|
||||
digest = _idempotency_digest("enroll", idempotency_key)
|
||||
existing = session.exec(select(EnrollmentJob).where(EnrollmentJob.idempotency_sha256 == digest)).first()
|
||||
if existing:
|
||||
if existing.enrollment_id != enrollment.id:
|
||||
raise EnrollmentError("idempotency key was already used for a different enrollment")
|
||||
return existing
|
||||
job = EnrollmentJob(
|
||||
id=f"enroll-{uuid.uuid4().hex[:16]}",
|
||||
enrollment_id=enrollment.id,
|
||||
application_id=enrollment.application_id,
|
||||
idempotency_sha256=digest,
|
||||
status="queued",
|
||||
stage="planned",
|
||||
)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def serialize_enrollment(enrollment: Enrollment) -> dict[str, Any]:
|
||||
plan = dict(enrollment.plan_json or {})
|
||||
exposure = plan.get("exposure") if isinstance(plan.get("exposure"), dict) else {}
|
||||
authentication = plan.get("authentication") if isinstance(plan.get("authentication"), dict) else {}
|
||||
return {
|
||||
"id": enrollment.id,
|
||||
"applicationId": enrollment.application_id,
|
||||
"repository": enrollment.repository,
|
||||
"revision": enrollment.revision,
|
||||
"profile": enrollment.profile,
|
||||
"controllerIdentity": enrollment.controller_identity,
|
||||
"status": enrollment.status,
|
||||
"publicUrl": f"https://{exposure.get('hostname')}" if exposure.get("hostname") else None,
|
||||
"authentication": authentication.get("mode", "none"),
|
||||
"resources": enrollment.resources_json or {},
|
||||
"targetSecretRef": enrollment.target_secret_ref,
|
||||
"oidcSecretRef": enrollment.oidc_secret_ref,
|
||||
"maintenanceSecretRef": (
|
||||
_secret_ref("repopulse", "MAINTENANCE_TOKEN")
|
||||
if enrollment.profile == "repopulse-authenticated" else None
|
||||
),
|
||||
"lastVerifiedAt": enrollment.last_verified_at.isoformat() if enrollment.last_verified_at else None,
|
||||
"createdAt": enrollment.created_at.isoformat(),
|
||||
"updatedAt": enrollment.updated_at.isoformat(),
|
||||
"completedAt": enrollment.completed_at.isoformat() if enrollment.completed_at else None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def serialize_job(job: EnrollmentJob, *, replay: bool = False) -> dict[str, Any]:
|
||||
return {
|
||||
"id": job.id,
|
||||
"enrollmentId": job.enrollment_id,
|
||||
"applicationId": job.application_id,
|
||||
"operation": job.operation,
|
||||
"status": job.status,
|
||||
"stage": job.stage,
|
||||
"preflight": job.preflight_json or {},
|
||||
"evidence": job.evidence_json or {},
|
||||
"result": job.result_summary,
|
||||
"createdAt": job.created_at.isoformat(),
|
||||
"updatedAt": job.updated_at.isoformat(),
|
||||
"completedAt": job.completed_at.isoformat() if job.completed_at else None,
|
||||
"idempotentReplay": replay,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _checkpoint(
|
||||
session: Session,
|
||||
enrollment: Enrollment,
|
||||
job: EnrollmentJob,
|
||||
*,
|
||||
stage: str,
|
||||
evidence: dict[str, Any],
|
||||
) -> None:
|
||||
"""Durably record intent and receipts around every external boundary.
|
||||
|
||||
This is deliberately conservative: after an interrupted external call,
|
||||
a same-plan request is surfaced as recovery-required instead of being
|
||||
replayed against NPM, Authentik or the controller without proof of its
|
||||
prior effect. Values in ``evidence`` are identifiers/status only.
|
||||
"""
|
||||
now = utc_now()
|
||||
enrollment.status = "provisioning"
|
||||
enrollment.resources_json = {**dict(enrollment.resources_json or {}), **evidence}
|
||||
enrollment.updated_at = now
|
||||
job.status = "running"
|
||||
job.stage = stage
|
||||
job.evidence_json = {**dict(job.evidence_json or {}), **evidence}
|
||||
job.updated_at = now
|
||||
session.add(enrollment)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
|
||||
def _install_policies(self, session: Session, enrollment: Enrollment, plan: EnrollmentPlanV1) -> list[str]:
|
||||
provenance = {
|
||||
"enrollmentId": enrollment.id,
|
||||
"profileVersion": PROFILE_VERSION,
|
||||
"createdBy": "enrollment-plane/v1",
|
||||
"policyTemplateVersion": "exact-target/v1",
|
||||
"target": plan.application_id,
|
||||
"sourceRevision": plan.revision,
|
||||
"createdAt": utc_now().isoformat(),
|
||||
}
|
||||
identifiers: list[str] = []
|
||||
for action, suffix in (("deployment.apply", "deployment-apply"), ("deployment.rollback", "deployment-rollback")):
|
||||
policy_id = f"enrollment-{plan.application_id}-{suffix}"
|
||||
existing = session.get(Policy, policy_id)
|
||||
if existing is not None:
|
||||
# Enrollment never adopts or repurposes a pre-existing policy,
|
||||
# even where its visible action/target happen to match. A
|
||||
# replay is handled before this method; this branch is a
|
||||
# collision, not a safe recovery path.
|
||||
raise EnrollmentError("exact policy identifier is already owned and cannot be adopted")
|
||||
existing = Policy(
|
||||
id=policy_id,
|
||||
action=action,
|
||||
target=plan.application_id,
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Enrollment-generated exact-target deployment policy.",
|
||||
)
|
||||
existing.provenance_json = provenance
|
||||
session.add(existing)
|
||||
identifiers.append(policy_id)
|
||||
session.commit()
|
||||
return identifiers
|
||||
|
||||
def _compensate(
|
||||
self,
|
||||
session: Session,
|
||||
enrollment: Enrollment,
|
||||
evidence: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Reverse only resources this execution proved it created.
|
||||
|
||||
A failed compensation stays visible as ``compensation_required``. It
|
||||
never attempts to adopt, overwrite or delete an earlier resource.
|
||||
"""
|
||||
plan = EnrollmentPlanV1.model_validate(enrollment.plan_json)
|
||||
outcomes: dict[str, str] = {}
|
||||
oidc = evidence.get("oidc") if isinstance(evidence.get("oidc"), dict) else {}
|
||||
if oidc.get("status") == "created":
|
||||
remove = getattr(self.authentik, "remove_deployment_oidc_application", None)
|
||||
result = remove(
|
||||
application_id=oidc.get("applicationId"),
|
||||
provider_id=oidc.get("providerId"),
|
||||
application_slug=plan.application_id,
|
||||
launch_url=plan.public_url or "",
|
||||
) if callable(remove) else {"success": False, "status": "unsupported"}
|
||||
outcomes["oidc"] = str(result.get("status") or "failed")
|
||||
elif oidc.get("status") == "partial_failure":
|
||||
outcomes["oidc"] = "ownership_unverified"
|
||||
|
||||
proxy = evidence.get("proxy") if isinstance(evidence.get("proxy"), dict) else {}
|
||||
if proxy.get("status") == "created":
|
||||
remove = getattr(self.npm, "remove_managed_proxy_host", None)
|
||||
result = remove(host_id=proxy.get("hostId"), ownership_id=enrollment.id) if callable(remove) else {"success": False, "status": "unsupported"}
|
||||
outcomes["proxy"] = str(result.get("status") or "failed")
|
||||
elif proxy.get("status") == "ownership_unverified":
|
||||
outcomes["proxy"] = "ownership_unverified"
|
||||
|
||||
controller = evidence.get("controller") if isinstance(evidence.get("controller"), dict) else {}
|
||||
if controller.get("status") in {"enrolled", "already_enrolled"}:
|
||||
trust = self.get_trust_config(session)
|
||||
bootstrap = get_secret(session, trust.bootstrap_secret_ref, touch=False) if trust else None
|
||||
if bootstrap:
|
||||
result = self.broker.unenroll_target(
|
||||
{"enrollment_id": enrollment.id, "application_id": enrollment.application_id, "repository": enrollment.repository},
|
||||
secret=bootstrap,
|
||||
)
|
||||
outcomes["controller"] = str(result.get("status") or "failed")
|
||||
else:
|
||||
outcomes["controller"] = "bootstrap_secret_unavailable"
|
||||
elif controller.get("status") == "partial_failure":
|
||||
outcomes["controller"] = "ownership_unverified"
|
||||
|
||||
completed = all(value in {"removed", "rolled_back"} for value in outcomes.values())
|
||||
if completed and outcomes:
|
||||
delete_secret(session, enrollment.target_secret_ref)
|
||||
if enrollment.oidc_secret_ref:
|
||||
delete_secret(session, enrollment.oidc_secret_ref)
|
||||
if plan.profile == "repopulse-authenticated":
|
||||
delete_secret(session, _secret_ref("repopulse", "MAINTENANCE_TOKEN"))
|
||||
return {"completed": completed, "outcomes": outcomes}
|
||||
|
||||
def _fail(
|
||||
self,
|
||||
session: Session,
|
||||
enrollment: Enrollment,
|
||||
job: EnrollmentJob,
|
||||
*,
|
||||
stage: str,
|
||||
message: str,
|
||||
evidence: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
compensation = (
|
||||
self._compensate(session, enrollment, evidence)
|
||||
if evidence.get("controller")
|
||||
else {"completed": True, "outcomes": {}}
|
||||
)
|
||||
compensation_required = not bool(compensation.get("completed"))
|
||||
enrollment.status = "compensation_required" if compensation_required else "failed"
|
||||
enrollment.resources_json = {**dict(enrollment.resources_json or {}), **evidence, "compensation": compensation}
|
||||
enrollment.updated_at = utc_now()
|
||||
enrollment.completed_at = utc_now()
|
||||
job.status = "compensation_required" if compensation_required else "failed"
|
||||
job.stage = stage
|
||||
job.evidence_json = {**dict(job.evidence_json or {}), **evidence, "compensation": compensation}
|
||||
job.result_summary = message
|
||||
job.updated_at = utc_now()
|
||||
job.completed_at = utc_now()
|
||||
session.add(enrollment)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
log_event(
|
||||
session,
|
||||
source="enrollment-plane",
|
||||
identity="operator",
|
||||
action_id="target.enroll",
|
||||
target=enrollment.application_id,
|
||||
target_type="enrollment",
|
||||
risk="high",
|
||||
policy_decision="blocked",
|
||||
reason=message,
|
||||
result="failed",
|
||||
execution_plan={"enrollmentId": enrollment.id, "planSha256": enrollment.plan_sha256},
|
||||
after_state={"stage": stage, "status": enrollment.status},
|
||||
)
|
||||
return {"enrollment": self.serialize_enrollment(enrollment), "job": self.serialize_job(job)}
|
||||
|
||||
def enroll(self, session: Session, plan: EnrollmentPlanV1, *, idempotency_key: str) -> dict[str, Any]:
|
||||
_, plan_digest = _canonical(plan)
|
||||
existing, repository_existing = self._existing(session, plan.application_id, plan.repository)
|
||||
if existing and existing.plan_sha256 == plan_digest:
|
||||
prior = session.exec(
|
||||
select(EnrollmentJob).where(EnrollmentJob.enrollment_id == existing.id).order_by(EnrollmentJob.created_at.desc())
|
||||
).first()
|
||||
if prior is None or prior.status in _TERMINAL:
|
||||
return {
|
||||
"enrollment": self.serialize_enrollment(existing),
|
||||
"job": self.serialize_job(prior, replay=True) if prior else None,
|
||||
"idempotentReplay": True,
|
||||
}
|
||||
# A process may have stopped after an external request was sent but
|
||||
# before its response was durably checkpointed. Do not infer the
|
||||
# effect and do not send the request again: retain the exact intent
|
||||
# and require an explicit reconciliation flow.
|
||||
existing.status = "recovery_required"
|
||||
existing.updated_at = utc_now()
|
||||
prior.status = "recovery_required"
|
||||
prior.result_summary = "Enrollment stopped during an external stage; reconcile recorded intent and receipts before retrying."
|
||||
prior.updated_at = utc_now()
|
||||
session.add(existing)
|
||||
session.add(prior)
|
||||
session.commit()
|
||||
return {
|
||||
"enrollment": self.serialize_enrollment(existing),
|
||||
"job": self.serialize_job(prior),
|
||||
"recoveryRequired": True,
|
||||
}
|
||||
if existing or repository_existing:
|
||||
raise EnrollmentError("application identity or repository is already enrolled")
|
||||
|
||||
trust = self.get_trust_config(session)
|
||||
if trust is None:
|
||||
raise EnrollmentError("enrollment trust boundaries are not configured")
|
||||
enrollment = self._create_enrollment(session, plan, plan_digest, trust)
|
||||
job = self._create_job(session, enrollment, idempotency_key)
|
||||
if job.status != "queued":
|
||||
return {"enrollment": self.serialize_enrollment(enrollment), "job": self.serialize_job(job, replay=True)}
|
||||
|
||||
preflight = self.preflight(session, plan)
|
||||
job.preflight_json = preflight
|
||||
job.updated_at = utc_now()
|
||||
session.add(job)
|
||||
session.commit()
|
||||
if preflight["status"] != "ready":
|
||||
enrollment.status = "preflight_failed"
|
||||
enrollment.completed_at = utc_now()
|
||||
enrollment.updated_at = utc_now()
|
||||
job.status = "preflight_failed"
|
||||
job.stage = "preflight"
|
||||
job.result_summary = "Enrollment preflight was blocked."
|
||||
job.completed_at = utc_now()
|
||||
session.add(enrollment)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return {"enrollment": self.serialize_enrollment(enrollment), "job": self.serialize_job(job)}
|
||||
|
||||
repopulse_gitea_token = (
|
||||
get_secret(session, "REPOPULSE_GITEA_TOKEN", touch=False)
|
||||
if plan.profile == "repopulse-authenticated" else None
|
||||
)
|
||||
if plan.profile == "repopulse-authenticated" and not repopulse_gitea_token:
|
||||
return self._fail(
|
||||
session, enrollment, job, stage="preflight",
|
||||
message="The existing encrypted RepoPulse Gitea token became unavailable after preflight.",
|
||||
evidence={},
|
||||
)
|
||||
|
||||
target_secret = _credential()
|
||||
set_secret(
|
||||
session,
|
||||
enrollment.target_secret_ref,
|
||||
target_secret,
|
||||
used_by=[f"enrollment:{enrollment.id}", "deployment-controller"],
|
||||
metadata={"enrollmentId": enrollment.id, "credentialFingerprint": _fingerprint(target_secret), "rotationState": "active"},
|
||||
)
|
||||
oidc_secret: str | None = None
|
||||
if enrollment.oidc_secret_ref:
|
||||
oidc_secret = _credential()
|
||||
set_secret(
|
||||
session,
|
||||
enrollment.oidc_secret_ref,
|
||||
oidc_secret,
|
||||
used_by=[f"enrollment:{enrollment.id}", "authentik", "deployment-controller"],
|
||||
metadata={"enrollmentId": enrollment.id, "credentialFingerprint": _fingerprint(oidc_secret), "rotationState": "active"},
|
||||
)
|
||||
repopulse_maintenance_token: str | None = None
|
||||
if plan.profile == "repopulse-authenticated":
|
||||
repopulse_maintenance_token = _credential()
|
||||
set_secret(
|
||||
session,
|
||||
_secret_ref("repopulse", "MAINTENANCE_TOKEN"),
|
||||
repopulse_maintenance_token,
|
||||
used_by=[f"enrollment:{enrollment.id}", "repopulse-maintenance", "deployment-controller"],
|
||||
metadata={
|
||||
"enrollmentId": enrollment.id,
|
||||
"credentialFingerprint": _fingerprint(repopulse_maintenance_token),
|
||||
"rotationState": "active",
|
||||
},
|
||||
)
|
||||
bootstrap_secret = get_secret(session, trust.bootstrap_secret_ref, touch=False)
|
||||
if not bootstrap_secret:
|
||||
return self._fail(session, enrollment, job, stage="creating_secrets", message="Controller bootstrap secret disappeared after preflight.", evidence={})
|
||||
evidence: dict[str, Any] = {
|
||||
"intent": {
|
||||
"controller": {
|
||||
"enrollmentId": enrollment.id,
|
||||
"applicationId": plan.application_id,
|
||||
"repository": plan.repository,
|
||||
}
|
||||
}
|
||||
}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="registering_target", evidence=evidence
|
||||
)
|
||||
controller_result = self.broker.enroll_target(
|
||||
self._controller_payload(plan, enrollment_id=enrollment.id, target_credential=target_secret),
|
||||
secret=bootstrap_secret,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
evidence["controller"] = {
|
||||
key: controller_result[key] for key in _CONTROLLER_EVIDENCE if key in controller_result
|
||||
}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="registering_target", evidence=evidence
|
||||
)
|
||||
if controller_result.get("status") not in {"enrolled", "already_enrolled"}:
|
||||
return self._fail(session, enrollment, job, stage="registering_target", message="Controller rejected the bounded target enrollment.", evidence=evidence)
|
||||
|
||||
if plan.exposure:
|
||||
evidence["intent"] = {
|
||||
**dict(evidence.get("intent") or {}),
|
||||
"proxy": {"hostname": plan.exposure.hostname, "ownershipId": enrollment.id},
|
||||
}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="provisioning_proxy", evidence=evidence
|
||||
)
|
||||
proxy_arguments = {
|
||||
"hostname": plan.exposure.hostname,
|
||||
"forward_host": plan.application_id,
|
||||
"forward_port": plan.network.internal_port,
|
||||
"certificate_mode": "managed",
|
||||
"force_ssl": True,
|
||||
"http2": True,
|
||||
"upload_limit_mb": 1024 if plan.exposure.upload_profile == "large-upload" else 128,
|
||||
"read_timeout_seconds": 900 if plan.exposure.timeout_profile == "long-running" else 120,
|
||||
"ownership_id": enrollment.id,
|
||||
}
|
||||
if plan.profile == "repopulse-authenticated":
|
||||
access_list_id = trust.repopulse_proxy_access_list_id if trust else None
|
||||
if not access_list_id or access_list_id <= 0:
|
||||
return self._fail(
|
||||
session, enrollment, job,
|
||||
stage="provisioning_proxy",
|
||||
message="RepoPulse requires a verified, positive NPM access-list ID.",
|
||||
evidence=evidence,
|
||||
)
|
||||
proxy_arguments["access_list_id"] = access_list_id
|
||||
proxy_result = self.npm.ensure_managed_proxy_host(
|
||||
**proxy_arguments,
|
||||
)
|
||||
evidence["proxy"] = {key: proxy_result[key] for key in _PROXY_EVIDENCE if key in proxy_result}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="provisioning_proxy", evidence=evidence
|
||||
)
|
||||
if not proxy_result.get("success") or proxy_result.get("status") != "created":
|
||||
return self._fail(session, enrollment, job, stage="provisioning_proxy", message="Managed reverse-proxy provisioning was not verified.", evidence=evidence)
|
||||
|
||||
profile = profile_contract(plan.profile)
|
||||
runtime_values: dict[str, str] = {}
|
||||
if plan.public_url and profile and "APPOPS_PUBLIC_URL" in profile["runtimeEnvironmentKeys"]:
|
||||
runtime_values["APPOPS_PUBLIC_URL"] = plan.public_url
|
||||
if plan.authentication.mode == "native-oidc":
|
||||
assert oidc_secret is not None and enrollment.oidc_secret_ref is not None
|
||||
evidence["intent"] = {
|
||||
**dict(evidence.get("intent") or {}),
|
||||
"oidc": {"applicationSlug": plan.application_id, "launchUrl": plan.public_url},
|
||||
}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="provisioning_identity", evidence=evidence
|
||||
)
|
||||
oidc_result = self.authentik.ensure_deployment_oidc_application(
|
||||
application_slug=plan.application_id,
|
||||
launch_url=plan.public_url or "",
|
||||
redirect_uri=f"{plan.public_url}{plan.authentication.redirect_path}",
|
||||
client_secret=oidc_secret,
|
||||
administrator_group=trust.administrator_group or "",
|
||||
support_agent_group=trust.support_agent_group or "",
|
||||
read_only_group=trust.read_only_group or "",
|
||||
)
|
||||
evidence["oidc"] = {key: oidc_result[key] for key in _OIDC_EVIDENCE if key in oidc_result}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="provisioning_identity", evidence=evidence
|
||||
)
|
||||
if not oidc_result.get("success") or oidc_result.get("status") != "created":
|
||||
return self._fail(session, enrollment, job, stage="provisioning_identity", message="Bounded OIDC application provisioning was not verified.", evidence=evidence)
|
||||
runtime_values.update({
|
||||
"OIDC_CLIENT_ID": plan.application_id,
|
||||
"OIDC_CLIENT_SECRET": oidc_secret,
|
||||
"OIDC_ISSUER": trust.oidc_issuer or "",
|
||||
"OIDC_ROLE_CLAIM": trust.oidc_role_claim,
|
||||
})
|
||||
if plan.profile == "repopulse-authenticated":
|
||||
assert repopulse_gitea_token is not None and repopulse_maintenance_token is not None
|
||||
runtime_values.update({
|
||||
"REPOPULSE_GITEA_URL": "https://gitea.example.com",
|
||||
"REPOPULSE_GITEA_TOKEN": repopulse_gitea_token,
|
||||
"REPOPULSE_MAINTENANCE_TOKEN": repopulse_maintenance_token,
|
||||
})
|
||||
|
||||
if profile and profile["runtimeEnvironmentKeys"]:
|
||||
evidence["intent"] = {
|
||||
**dict(evidence.get("intent") or {}),
|
||||
"runtime": {"applicationId": plan.application_id, "keys": list(runtime_values)},
|
||||
}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="sealing_runtime", evidence=evidence
|
||||
)
|
||||
runtime_result = self.broker.configure_enrollment_runtime(
|
||||
{
|
||||
"enrollment_id": enrollment.id,
|
||||
"application_id": plan.application_id,
|
||||
"repository": plan.repository,
|
||||
"values": runtime_values,
|
||||
},
|
||||
secret=target_secret,
|
||||
)
|
||||
evidence["runtime"] = {key: runtime_result[key] for key in ("status", "enrollmentId", "applicationId") if key in runtime_result}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="sealing_runtime", evidence=evidence
|
||||
)
|
||||
if runtime_result.get("status") not in {"configured", "already_configured"}:
|
||||
return self._fail(session, enrollment, job, stage="sealing_runtime", message="Controller could not seal profile-owned runtime values.", evidence=evidence)
|
||||
|
||||
evidence["intent"] = {
|
||||
**dict(evidence.get("intent") or {}),
|
||||
"policies": [
|
||||
f"enrollment-{plan.application_id}-deployment-apply",
|
||||
f"enrollment-{plan.application_id}-deployment-rollback",
|
||||
],
|
||||
}
|
||||
self._checkpoint(
|
||||
session, enrollment, job, stage="installing_policy", evidence=evidence
|
||||
)
|
||||
try:
|
||||
policy_ids = self._install_policies(session, enrollment, plan)
|
||||
except EnrollmentError as exc:
|
||||
return self._fail(session, enrollment, job, stage="installing_policy", message=str(exc), evidence=evidence)
|
||||
evidence["policies"] = policy_ids
|
||||
enrollment.status = "ready"
|
||||
enrollment.resources_json = {
|
||||
"appdataPath": plan.appdata_path,
|
||||
"dataPath": plan.data_path,
|
||||
"network": plan.network.docker_network,
|
||||
"hostname": plan.exposure.hostname if plan.exposure else None,
|
||||
**evidence,
|
||||
}
|
||||
enrollment.last_verified_at = utc_now()
|
||||
enrollment.updated_at = utc_now()
|
||||
enrollment.completed_at = utc_now()
|
||||
job.status = "ready"
|
||||
job.stage = "ready"
|
||||
job.evidence_json = evidence
|
||||
job.result_summary = "Bounded target, profile resources and exact policies are ready for deployment."
|
||||
job.updated_at = utc_now()
|
||||
job.completed_at = utc_now()
|
||||
session.add(enrollment)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
log_event(
|
||||
session,
|
||||
source="enrollment-plane",
|
||||
identity="operator",
|
||||
action_id="target.enroll",
|
||||
target=plan.application_id,
|
||||
target_type="enrollment",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
reason="Bounded application identity was enrolled from an autonomous profile.",
|
||||
result="success",
|
||||
execution_plan={"enrollmentId": enrollment.id, "planSha256": plan_digest, "profile": plan.profile},
|
||||
after_state={"status": "ready", "policyIds": policy_ids},
|
||||
)
|
||||
return {"enrollment": self.serialize_enrollment(enrollment), "job": self.serialize_job(job)}
|
||||
|
||||
def list_enrollments(self, session: Session, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
rows = session.exec(select(Enrollment).order_by(Enrollment.created_at.desc()).limit(max(1, min(limit, 250)))).all()
|
||||
return [self.serialize_enrollment(row) for row in rows]
|
||||
|
||||
def get_enrollment(self, session: Session, enrollment_id: str) -> dict[str, Any]:
|
||||
row = session.get(Enrollment, enrollment_id)
|
||||
if row is None:
|
||||
raise EnrollmentError("enrollment was not found")
|
||||
jobs = session.exec(
|
||||
select(EnrollmentJob).where(EnrollmentJob.enrollment_id == enrollment_id).order_by(EnrollmentJob.created_at.desc()).limit(20)
|
||||
).all()
|
||||
return {"enrollment": self.serialize_enrollment(row), "jobs": [self.serialize_job(job) for job in jobs]}
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Server-owned enrollment profiles and bounded runtime environment contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
PROFILE_VERSION = "enrollment-profile/v1"
|
||||
|
||||
PROFILES: dict[str, dict[str, Any]] = {
|
||||
"internal-webapp": {
|
||||
"public": True,
|
||||
"authentication": "none",
|
||||
"runtimeEnvironmentKeys": ("APPOPS_PUBLIC_URL",),
|
||||
"requiresNpm": True,
|
||||
"requiresOidc": False,
|
||||
},
|
||||
"internal-webapp-oidc": {
|
||||
"public": True,
|
||||
"authentication": "native-oidc",
|
||||
"runtimeEnvironmentKeys": (
|
||||
"APPOPS_PUBLIC_URL",
|
||||
"OIDC_CLIENT_ID",
|
||||
"OIDC_CLIENT_SECRET",
|
||||
"OIDC_ISSUER",
|
||||
"OIDC_ROLE_CLAIM",
|
||||
),
|
||||
"requiresNpm": True,
|
||||
"requiresOidc": True,
|
||||
},
|
||||
"internal-service": {
|
||||
"public": False,
|
||||
"authentication": "none",
|
||||
"runtimeEnvironmentKeys": (),
|
||||
"requiresNpm": False,
|
||||
"requiresOidc": False,
|
||||
},
|
||||
"repopulse-authenticated": {
|
||||
"public": True,
|
||||
"authentication": "npm-access-list+maintenance-token",
|
||||
"runtimeEnvironmentKeys": (
|
||||
"REPOPULSE_GITEA_URL",
|
||||
"REPOPULSE_GITEA_TOKEN",
|
||||
"REPOPULSE_MAINTENANCE_TOKEN",
|
||||
),
|
||||
"requiresNpm": True,
|
||||
"requiresOidc": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def profile_contract(name: str) -> dict[str, Any] | None:
|
||||
profile = PROFILES.get(name)
|
||||
return dict(profile) if profile else None
|
||||
|
||||
|
||||
def profile_summary(name: str) -> dict[str, Any] | None:
|
||||
profile = profile_contract(name)
|
||||
if profile is None:
|
||||
return None
|
||||
return {"name": name, "version": PROFILE_VERSION, **profile}
|
||||
@@ -0,0 +1,611 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting, Policy
|
||||
from .deployment_target_registry import DeploymentTargetRegistry
|
||||
|
||||
_CATALOG_KEY = "deployment:gitea-project-catalog:v1"
|
||||
_OWNER = "Jens"
|
||||
_DESCRIPTOR = ".itworx/deployment-target.json"
|
||||
_COMPOSE_CANDIDATES = (
|
||||
"docker-compose.yml",
|
||||
"docker-compose.yaml",
|
||||
"compose.yml",
|
||||
"compose.yaml",
|
||||
)
|
||||
# Must remain identical to DeploymentPlanV1.application_id.
|
||||
_APP_ID = re.compile(r"^[a-z][a-z0-9-]{1,62}$")
|
||||
_TRANSIENT_STATES = {"gitea_unavailable", "controller_unavailable", "registration_error"}
|
||||
_RETRY_STATES = _TRANSIENT_STATES | {"controller_conflict", "policy_conflict"}
|
||||
_DEPLOYMENT_ACTIONS = ("deployment.apply", "deployment.rollback", "deployment.runnerless.provision")
|
||||
|
||||
|
||||
class GiteaProjectAutoRegistration:
|
||||
"""Keep AppOps' project catalog aligned with the owned Gitea portfolio.
|
||||
|
||||
Every repository is recorded in the catalog. A repository becomes an
|
||||
executable deployment target only when an exact revision exposes a bounded
|
||||
Compose contract compatible with DeploymentPlanV1. Exact deployment
|
||||
policies are created only after the controller target has been registered
|
||||
successfully. Existing targets and policies are never overwritten.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
target_registry: DeploymentTargetRegistry | None = None,
|
||||
owner: str = _OWNER,
|
||||
) -> None:
|
||||
self.target_registry = target_registry or DeploymentTargetRegistry()
|
||||
self.owner = owner
|
||||
|
||||
@staticmethod
|
||||
def _base_url() -> str:
|
||||
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
||||
parsed = urlsplit(raw)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username:
|
||||
raise RuntimeError("invalid_gitea_url")
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", ""))
|
||||
|
||||
@staticmethod
|
||||
def _token() -> str:
|
||||
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "").strip()
|
||||
if not token:
|
||||
raise RuntimeError("missing_gitea_admin_token")
|
||||
return token
|
||||
|
||||
def _client(self) -> httpx.Client:
|
||||
timeout = max(2.0, min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)))
|
||||
return httpx.Client(
|
||||
timeout=timeout,
|
||||
verify=bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls)),
|
||||
headers={
|
||||
"Authorization": f"token {self._token()}",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
def _get(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
path: str,
|
||||
*,
|
||||
params: dict[str, Any] | None = None,
|
||||
allow_not_found: bool = False,
|
||||
) -> Any:
|
||||
try:
|
||||
response = client.get(f"{self._base_url()}{path}", params=params)
|
||||
except (httpx.RequestError, ValueError) as exc:
|
||||
raise RuntimeError(redact_message(str(exc))) from exc
|
||||
if allow_not_found and response.status_code == 404:
|
||||
return None
|
||||
if response.status_code in {401, 403}:
|
||||
raise RuntimeError("gitea_admin_access_denied")
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"gitea_project_catalog_http_{response.status_code}")
|
||||
try:
|
||||
return response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("gitea_project_catalog_invalid_response") from exc
|
||||
|
||||
def _repositories(self, client: httpx.Client) -> list[dict[str, Any]]:
|
||||
owner = quote(self.owner, safe="")
|
||||
repositories: list[dict[str, Any]] = []
|
||||
for page in range(1, 11):
|
||||
payload = self._get(
|
||||
client,
|
||||
f"/api/v1/users/{owner}/repos",
|
||||
params={"page": page, "limit": 50},
|
||||
)
|
||||
if not isinstance(payload, list):
|
||||
raise RuntimeError("gitea_project_catalog_invalid_repository_list")
|
||||
items = [item for item in payload if isinstance(item, dict)]
|
||||
repositories.extend(items)
|
||||
if len(items) < 50:
|
||||
break
|
||||
return repositories[:500]
|
||||
|
||||
def _branch_head(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
*,
|
||||
repository_name: str,
|
||||
branch: str,
|
||||
) -> str:
|
||||
owner = quote(self.owner, safe="")
|
||||
repository = quote(repository_name, safe="")
|
||||
branch_ref = quote(branch, safe="")
|
||||
payload = self._get(
|
||||
client,
|
||||
f"/api/v1/repos/{owner}/{repository}/branches/{branch_ref}",
|
||||
)
|
||||
commit = payload.get("commit") if isinstance(payload, dict) else None
|
||||
revision = ""
|
||||
if isinstance(commit, dict):
|
||||
revision = str(commit.get("id") or commit.get("sha") or "").strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", revision):
|
||||
raise RuntimeError("gitea_default_branch_head_unavailable")
|
||||
return revision
|
||||
|
||||
def _contents(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
*,
|
||||
repository_name: str,
|
||||
revision: str,
|
||||
path: str = "",
|
||||
allow_not_found: bool = False,
|
||||
) -> Any:
|
||||
owner = quote(self.owner, safe="")
|
||||
repository = quote(repository_name, safe="")
|
||||
suffix = f"/{quote(path, safe='/')}" if path else ""
|
||||
return self._get(
|
||||
client,
|
||||
f"/api/v1/repos/{owner}/{repository}/contents{suffix}",
|
||||
params={"ref": revision},
|
||||
allow_not_found=allow_not_found,
|
||||
)
|
||||
|
||||
def _read_text(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
*,
|
||||
repository_name: str,
|
||||
revision: str,
|
||||
path: str,
|
||||
allow_not_found: bool = False,
|
||||
) -> str | None:
|
||||
payload = self._contents(
|
||||
client,
|
||||
repository_name=repository_name,
|
||||
revision=revision,
|
||||
path=path,
|
||||
allow_not_found=allow_not_found,
|
||||
)
|
||||
if payload is None:
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("gitea_project_file_invalid_response")
|
||||
if str(payload.get("encoding") or "").lower() != "base64":
|
||||
raise RuntimeError("gitea_project_file_encoding_unsupported")
|
||||
content = str(payload.get("content") or "").replace("\n", "")
|
||||
try:
|
||||
raw = base64.b64decode(content, validate=True)
|
||||
return raw.decode("utf-8")
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise RuntimeError("gitea_project_file_decode_failed") from exc
|
||||
|
||||
@staticmethod
|
||||
def _application_id(repository_name: str) -> str:
|
||||
raw = repository_name.lower()
|
||||
value = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")[:63]
|
||||
if not _APP_ID.fullmatch(value):
|
||||
raise ValueError("repository name cannot be converted to a DeploymentPlanV1 application id")
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _safe_descriptor(payload: Any, repository_name: str) -> dict[str, Any]:
|
||||
if payload is None:
|
||||
return {}
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("deployment descriptor must contain a JSON object")
|
||||
allowed = {
|
||||
"apiVersion",
|
||||
"applicationId",
|
||||
"composeFile",
|
||||
"composeProject",
|
||||
"healthTimeout",
|
||||
"verifyOciRevision",
|
||||
}
|
||||
unknown = set(payload) - allowed
|
||||
if unknown:
|
||||
raise ValueError("deployment descriptor contains unsupported fields")
|
||||
if payload.get("apiVersion") not in {None, "itworx.deployment-target/v1"}:
|
||||
raise ValueError("deployment descriptor apiVersion is unsupported")
|
||||
application_id = str(
|
||||
payload.get("applicationId")
|
||||
or GiteaProjectAutoRegistration._application_id(repository_name)
|
||||
).strip().lower()
|
||||
if not _APP_ID.fullmatch(application_id):
|
||||
raise ValueError("deployment descriptor applicationId is invalid")
|
||||
compose_file = str(payload.get("composeFile") or "").strip()
|
||||
compose_project = str(payload.get("composeProject") or application_id).strip().lower()
|
||||
if compose_project != application_id:
|
||||
raise ValueError("deployment descriptor composeProject must equal applicationId")
|
||||
timeout = int(payload.get("healthTimeout") or 180)
|
||||
return {
|
||||
"applicationId": application_id,
|
||||
"composeFile": compose_file,
|
||||
"composeProject": compose_project,
|
||||
"healthTimeout": timeout,
|
||||
"verifyOciRevision": bool(payload.get("verifyOciRevision", False)),
|
||||
}
|
||||
|
||||
def _compose_contract(
|
||||
self,
|
||||
client: httpx.Client,
|
||||
*,
|
||||
repository_name: str,
|
||||
revision: str,
|
||||
) -> dict[str, Any] | None:
|
||||
root = self._contents(
|
||||
client,
|
||||
repository_name=repository_name,
|
||||
revision=revision,
|
||||
)
|
||||
if not isinstance(root, list):
|
||||
raise RuntimeError("gitea_project_root_invalid_response")
|
||||
names = {
|
||||
str(item.get("name") or "")
|
||||
for item in root
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
|
||||
descriptor: dict[str, Any] = {}
|
||||
descriptor_present = False
|
||||
if ".itworx" in names:
|
||||
raw_descriptor = self._read_text(
|
||||
client,
|
||||
repository_name=repository_name,
|
||||
revision=revision,
|
||||
path=_DESCRIPTOR,
|
||||
allow_not_found=True,
|
||||
)
|
||||
if raw_descriptor is not None:
|
||||
descriptor_present = True
|
||||
try:
|
||||
descriptor = self._safe_descriptor(json.loads(raw_descriptor), repository_name)
|
||||
except (json.JSONDecodeError, TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid deployment descriptor: {exc}") from exc
|
||||
|
||||
compose_file = str(descriptor.get("composeFile") or "")
|
||||
if not compose_file:
|
||||
compose_file = next((candidate for candidate in _COMPOSE_CANDIDATES if candidate in names), "")
|
||||
if not compose_file:
|
||||
return None
|
||||
|
||||
raw_compose = self._read_text(
|
||||
client,
|
||||
repository_name=repository_name,
|
||||
revision=revision,
|
||||
path=compose_file,
|
||||
)
|
||||
if raw_compose is None:
|
||||
raise ValueError("configured Compose file is unavailable")
|
||||
try:
|
||||
compose = yaml.safe_load(raw_compose)
|
||||
except yaml.YAMLError as exc:
|
||||
raise ValueError("Compose file is not valid YAML") from exc
|
||||
services = compose.get("services") if isinstance(compose, dict) else None
|
||||
if not isinstance(services, dict) or not services:
|
||||
raise ValueError("Compose file does not define services")
|
||||
if len(services) > 8:
|
||||
if descriptor_present:
|
||||
raise ValueError("deployment descriptor exceeds DeploymentPlanV1's eight-service bound")
|
||||
return None
|
||||
|
||||
service_names = [str(name).strip() for name in services]
|
||||
if any(not name for name in service_names):
|
||||
raise ValueError("Compose service name is invalid")
|
||||
|
||||
if descriptor_present:
|
||||
application_id = str(descriptor["applicationId"])
|
||||
if application_id not in service_names:
|
||||
raise ValueError("deployment descriptor applicationId must name a Compose service")
|
||||
else:
|
||||
repository_id = self._application_id(repository_name)
|
||||
compose_name = str(compose.get("name") or "").strip().lower() if isinstance(compose, dict) else ""
|
||||
candidates: list[str] = []
|
||||
for candidate in (repository_id, compose_name):
|
||||
if candidate and _APP_ID.fullmatch(candidate) and candidate in service_names:
|
||||
if candidate not in candidates:
|
||||
candidates.append(candidate)
|
||||
if len(service_names) == 1 and _APP_ID.fullmatch(service_names[0]):
|
||||
if service_names[0] not in candidates:
|
||||
candidates.append(service_names[0])
|
||||
if len(candidates) != 1:
|
||||
# The repository remains cataloged, but deployment requires an
|
||||
# explicit bounded descriptor to select its primary service.
|
||||
return None
|
||||
application_id = candidates[0]
|
||||
|
||||
compose_project = application_id
|
||||
containers: list[str] = []
|
||||
for service_name, raw_service in services.items():
|
||||
service = raw_service if isinstance(raw_service, dict) else {}
|
||||
explicit_name = str(service.get("container_name") or "").strip()
|
||||
container = explicit_name or f"{compose_project}-{service_name}-1"
|
||||
containers.append(container)
|
||||
|
||||
return {
|
||||
"applicationId": application_id,
|
||||
"composeFile": compose_file,
|
||||
"composeProject": compose_project,
|
||||
"containers": containers,
|
||||
"serviceNames": service_names,
|
||||
"healthTimeout": int(descriptor.get("healthTimeout") or 180),
|
||||
"verifyOciRevision": bool(descriptor.get("verifyOciRevision", False)),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _policy_id(application_id: str, action: str) -> str:
|
||||
digest = hashlib.sha256(f"{action}:{application_id}".encode("utf-8")).hexdigest()[:20]
|
||||
suffix = action.rsplit(".", 1)[-1]
|
||||
return f"gitea-auto-{application_id[:32]}-{suffix}-{digest}"
|
||||
|
||||
@staticmethod
|
||||
def _existing_exact_policy_ids(
|
||||
session: Session,
|
||||
*,
|
||||
application_id: str,
|
||||
) -> tuple[bool, list[str]]:
|
||||
identifiers: list[str] = []
|
||||
for action in _DEPLOYMENT_ACTIONS:
|
||||
existing = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == action) & (Policy.target == application_id)
|
||||
)
|
||||
).first()
|
||||
if existing is None or existing.manual != "allowed":
|
||||
return False, []
|
||||
identifiers.append(existing.id)
|
||||
return True, identifiers
|
||||
|
||||
@staticmethod
|
||||
def _ensure_exact_policies(
|
||||
session: Session,
|
||||
*,
|
||||
application_id: str,
|
||||
repository: str,
|
||||
revision: str,
|
||||
) -> tuple[bool, list[str], int]:
|
||||
existing_by_action: dict[str, Policy | None] = {}
|
||||
for action in _DEPLOYMENT_ACTIONS:
|
||||
existing = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == action) & (Policy.target == application_id)
|
||||
)
|
||||
).first()
|
||||
if existing is not None and existing.manual != "allowed":
|
||||
return False, [], 0
|
||||
existing_by_action[action] = existing
|
||||
|
||||
identifiers: list[str] = []
|
||||
created = 0
|
||||
for action in _DEPLOYMENT_ACTIONS:
|
||||
existing = existing_by_action[action]
|
||||
if existing is not None:
|
||||
identifiers.append(existing.id)
|
||||
continue
|
||||
policy_id = GiteaProjectAutoRegistration._policy_id(application_id, action)
|
||||
row = Policy(
|
||||
id=policy_id,
|
||||
action=action,
|
||||
target=application_id,
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Gitea auto-registration generated exact-target deployment policy.",
|
||||
provenance_json={
|
||||
"createdBy": "gitea-project-auto-registration/v1",
|
||||
"repository": repository,
|
||||
"sourceRevision": revision,
|
||||
"createdAt": utc_now().isoformat(),
|
||||
"policyTemplateVersion": "exact-target/v1",
|
||||
},
|
||||
)
|
||||
session.add(row)
|
||||
identifiers.append(policy_id)
|
||||
created += 1
|
||||
session.commit()
|
||||
return True, identifiers, created
|
||||
|
||||
@staticmethod
|
||||
def catalog(session: Session) -> dict[str, Any]:
|
||||
row = session.get(AppSetting, _CATALOG_KEY)
|
||||
if row is None or row.is_secret:
|
||||
return {
|
||||
"status": "never_synced",
|
||||
"owner": _OWNER,
|
||||
"projects": [],
|
||||
"summary": {"total": 0},
|
||||
}
|
||||
return dict(row.value_json or {})
|
||||
|
||||
@staticmethod
|
||||
def _marker(repo: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"updatedAt": str(repo.get("updated_at") or ""),
|
||||
"defaultBranch": str(repo.get("default_branch") or ""),
|
||||
"archived": bool(repo.get("archived")),
|
||||
"empty": bool(repo.get("empty")),
|
||||
"private": bool(repo.get("private")),
|
||||
}
|
||||
|
||||
def sync(self, session: Session, *, force: bool = False) -> dict[str, Any]:
|
||||
previous = self.catalog(session)
|
||||
previous_items = {
|
||||
str(item.get("repository") or ""): item
|
||||
for item in previous.get("projects", [])
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
try:
|
||||
client = self._client()
|
||||
with client:
|
||||
repositories = self._repositories(client)
|
||||
projects: list[dict[str, Any]] = []
|
||||
controller_registered = 0
|
||||
policies_created = 0
|
||||
changed = 0
|
||||
for repo in sorted(
|
||||
repositories,
|
||||
key=lambda item: str(item.get("full_name") or item.get("name") or "").casefold(),
|
||||
):
|
||||
name = str(repo.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
repository = f"jens/{name.lower()}"
|
||||
marker = self._marker(repo)
|
||||
prior = previous_items.get(repository)
|
||||
if (
|
||||
not force
|
||||
and prior is not None
|
||||
and all(prior.get(key) == value for key, value in marker.items())
|
||||
and str(prior.get("state") or "") not in _RETRY_STATES
|
||||
):
|
||||
projects.append(dict(prior))
|
||||
continue
|
||||
|
||||
changed += 1
|
||||
item: dict[str, Any] = {
|
||||
"repository": repository,
|
||||
"name": name,
|
||||
**marker,
|
||||
"state": "known",
|
||||
"deploymentRegistered": False,
|
||||
"policyIds": [],
|
||||
}
|
||||
if marker["archived"]:
|
||||
item["state"] = "archived"
|
||||
projects.append(item)
|
||||
continue
|
||||
branch = str(marker["defaultBranch"] or "").strip()
|
||||
if marker["empty"] or not branch:
|
||||
item["state"] = "empty"
|
||||
projects.append(item)
|
||||
continue
|
||||
|
||||
try:
|
||||
revision = self._branch_head(
|
||||
client,
|
||||
repository_name=name,
|
||||
branch=branch,
|
||||
)
|
||||
item["revision"] = revision
|
||||
contract = self._compose_contract(
|
||||
client,
|
||||
repository_name=name,
|
||||
revision=revision,
|
||||
)
|
||||
if contract is None:
|
||||
item["state"] = "known_unconfigured"
|
||||
projects.append(item)
|
||||
continue
|
||||
application_id = str(contract["applicationId"])
|
||||
item.update(
|
||||
{
|
||||
"applicationId": application_id,
|
||||
"composeFile": contract["composeFile"],
|
||||
"composeProject": contract["composeProject"],
|
||||
"containers": list(contract["containers"]),
|
||||
"serviceNames": list(contract["serviceNames"]),
|
||||
}
|
||||
)
|
||||
result = self.target_registry.register_compose_target(
|
||||
repository,
|
||||
revision,
|
||||
branch=branch,
|
||||
compose_file=str(contract["composeFile"]),
|
||||
compose_project=str(contract["composeProject"]),
|
||||
containers=list(contract["containers"]),
|
||||
health_timeout=int(contract["healthTimeout"]),
|
||||
verify_oci_revision=bool(contract["verifyOciRevision"]),
|
||||
)
|
||||
result_status = str(result.get("status") or "error")
|
||||
if result_status == "registered":
|
||||
policy_ok, policy_ids, created = self._ensure_exact_policies(
|
||||
session,
|
||||
application_id=application_id,
|
||||
repository=repository,
|
||||
revision=revision,
|
||||
)
|
||||
policies_created += created
|
||||
if policy_ok:
|
||||
item["state"] = "registered"
|
||||
item["deploymentRegistered"] = True
|
||||
item["policyIds"] = policy_ids
|
||||
controller_registered += 1
|
||||
else:
|
||||
item["state"] = "policy_conflict"
|
||||
elif result_status == "conflict":
|
||||
policy_ok, policy_ids = self._existing_exact_policy_ids(
|
||||
session,
|
||||
application_id=application_id,
|
||||
)
|
||||
if policy_ok:
|
||||
item["state"] = "registered_existing"
|
||||
item["deploymentRegistered"] = True
|
||||
item["policyIds"] = policy_ids
|
||||
else:
|
||||
item["state"] = "controller_conflict"
|
||||
item["reason"] = str(result.get("reason") or "")[:240]
|
||||
elif result_status in {"controller_unavailable", "connector_unavailable"}:
|
||||
item["state"] = "controller_unavailable"
|
||||
item["reason"] = str(result.get("reason") or "")[:240]
|
||||
else:
|
||||
item["state"] = "registration_error"
|
||||
item["reason"] = str(result.get("reason") or result_status)[:240]
|
||||
except (RuntimeError, TypeError, ValueError, yaml.YAMLError) as exc:
|
||||
item["state"] = "registration_error"
|
||||
item["reason"] = redact_message(str(exc))[:240]
|
||||
projects.append(item)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
return {
|
||||
**previous,
|
||||
"status": "gitea_unavailable",
|
||||
"reason": redact_message(str(exc))[:240],
|
||||
}
|
||||
|
||||
summary = {
|
||||
"total": len(projects),
|
||||
"registered": sum(
|
||||
str(item.get("state")) in {"registered", "registered_existing"}
|
||||
for item in projects
|
||||
),
|
||||
"knownUnconfigured": sum(
|
||||
str(item.get("state")) == "known_unconfigured" for item in projects
|
||||
),
|
||||
"archived": sum(str(item.get("state")) == "archived" for item in projects),
|
||||
"empty": sum(str(item.get("state")) == "empty" for item in projects),
|
||||
"conflicts": sum(
|
||||
str(item.get("state")) in {"controller_conflict", "policy_conflict"}
|
||||
for item in projects
|
||||
),
|
||||
"errors": sum(
|
||||
str(item.get("state")) in _TRANSIENT_STATES for item in projects
|
||||
),
|
||||
"changedThisSync": changed,
|
||||
"controllerTargetsRegisteredThisSync": controller_registered,
|
||||
"policiesCreatedThisSync": policies_created,
|
||||
}
|
||||
value = {
|
||||
"status": "ok",
|
||||
"owner": self.owner,
|
||||
"syncedAt": utc_now().isoformat(),
|
||||
"automatic": True,
|
||||
"projects": projects,
|
||||
"summary": summary,
|
||||
}
|
||||
row = session.get(AppSetting, _CATALOG_KEY) or AppSetting(
|
||||
key=_CATALOG_KEY,
|
||||
is_secret=False,
|
||||
)
|
||||
row.value_json = value
|
||||
row.is_secret = False
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return value
|
||||
@@ -0,0 +1,3 @@
|
||||
|
||||
def run_healthcheck(target):
|
||||
return {'target': target, 'healthy': True}
|
||||
@@ -0,0 +1,197 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.time import utc_now
|
||||
from .secret_store import get_secret
|
||||
from .voice_catalog import list_commands
|
||||
|
||||
|
||||
def _timeout_seconds() -> float:
|
||||
try:
|
||||
return max(2.0, min(20.0, float(settings.apis.get('HOME_ASSISTANT_TIMEOUT_SECONDS') or 8)))
|
||||
except (TypeError, ValueError):
|
||||
return 8.0
|
||||
|
||||
|
||||
def _get_json(url: str, token: str, path: str, verify_tls: bool) -> tuple[Any, str, int | None]:
|
||||
try:
|
||||
with httpx.Client(timeout=_timeout_seconds(), verify=verify_tls) as client:
|
||||
response = client.get(
|
||||
f'{url.rstrip("/")}{path}',
|
||||
headers={
|
||||
'Authorization': f'Bearer {token}',
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
)
|
||||
except httpx.RequestError:
|
||||
return None, 'unreachable', None
|
||||
|
||||
if response.status_code in {401, 403}:
|
||||
return None, 'auth_failed', response.status_code
|
||||
if response.status_code >= 400:
|
||||
return None, 'error', response.status_code
|
||||
try:
|
||||
return response.json(), 'ok', response.status_code
|
||||
except ValueError:
|
||||
return None, 'invalid_response', response.status_code
|
||||
|
||||
|
||||
def _step(step_id: str, label: str, status: str, detail: str) -> dict[str, str]:
|
||||
return {'id': step_id, 'label': label, 'status': status, 'detail': detail}
|
||||
|
||||
|
||||
def build_readiness(session: Session) -> dict[str, Any]:
|
||||
url = str(settings.apis.get('HOME_ASSISTANT_URL') or '').strip().rstrip('/')
|
||||
token = get_secret(session, 'HOME_ASSISTANT_TOKEN', touch=False) or str(settings.apis.get('HOME_ASSISTANT_TOKEN') or '').strip()
|
||||
verify_tls = bool(settings.apis.get('HOME_ASSISTANT_VERIFY_TLS', False))
|
||||
commands = list_commands(session)
|
||||
expected_scripts = sorted({
|
||||
f'script.{command["homeAssistantScriptName"]}'
|
||||
for command in commands
|
||||
if command.get('homeAssistantScriptName')
|
||||
})
|
||||
core_script = 'script.appops_check_setup'
|
||||
|
||||
if not url or not token:
|
||||
missing = [
|
||||
name for name, value in (
|
||||
('HOME_ASSISTANT_URL', url),
|
||||
('HOME_ASSISTANT_TOKEN', token),
|
||||
)
|
||||
if not value
|
||||
]
|
||||
return {
|
||||
'status': 'missing_config',
|
||||
'operational': False,
|
||||
'apiReachable': False,
|
||||
'homeAssistantUrl': url,
|
||||
'missing': missing,
|
||||
'expectedScriptCount': len(expected_scripts),
|
||||
'detectedScriptCount': 0,
|
||||
'detectedScripts': [],
|
||||
'missingScripts': expected_scripts,
|
||||
'conversationAvailable': False,
|
||||
'geminiConversationAvailable': False,
|
||||
'llmExposureStatus': 'manual_verification_required',
|
||||
'steps': [
|
||||
_step('connector', 'Home Assistant connector', 'warning', f'Missing: {", ".join(missing)}'),
|
||||
],
|
||||
'nextActions': [
|
||||
{
|
||||
'id': 'configure_connector',
|
||||
'label': 'Configure Home Assistant connector',
|
||||
'href': '/setup?step=connectors&connector=home_assistant',
|
||||
},
|
||||
],
|
||||
'lastCheckedAt': utc_now().isoformat(),
|
||||
}
|
||||
|
||||
config_payload, config_status, config_code = _get_json(url, token, '/api/config', verify_tls)
|
||||
states_payload, states_status, states_code = _get_json(url, token, '/api/states', verify_tls)
|
||||
api_status = config_status if config_status != 'ok' else states_status
|
||||
if api_status != 'ok':
|
||||
return {
|
||||
'status': api_status,
|
||||
'operational': False,
|
||||
'apiReachable': False,
|
||||
'homeAssistantUrl': url,
|
||||
'missing': [],
|
||||
'statusCode': config_code or states_code,
|
||||
'expectedScriptCount': len(expected_scripts),
|
||||
'detectedScriptCount': 0,
|
||||
'detectedScripts': [],
|
||||
'missingScripts': expected_scripts,
|
||||
'conversationAvailable': False,
|
||||
'geminiConversationAvailable': False,
|
||||
'llmExposureStatus': 'manual_verification_required',
|
||||
'steps': [
|
||||
_step('connector', 'Home Assistant connector', 'complete', 'URL and token are stored.'),
|
||||
_step('api', 'Home Assistant API', 'warning', f'Live API check returned {api_status}.'),
|
||||
],
|
||||
'nextActions': [
|
||||
{
|
||||
'id': 'test_connector',
|
||||
'label': 'Review Home Assistant connector',
|
||||
'href': '/setup?step=connectors&connector=home_assistant',
|
||||
},
|
||||
],
|
||||
'lastCheckedAt': utc_now().isoformat(),
|
||||
}
|
||||
|
||||
components = set()
|
||||
if isinstance(config_payload, dict) and isinstance(config_payload.get('components'), list):
|
||||
components = {str(item) for item in config_payload['components']}
|
||||
entity_ids = {
|
||||
str(item.get('entity_id'))
|
||||
for item in states_payload
|
||||
if isinstance(states_payload, list) and isinstance(item, dict) and item.get('entity_id')
|
||||
}
|
||||
detected_scripts = sorted(entity_id for entity_id in expected_scripts if entity_id in entity_ids)
|
||||
missing_scripts = sorted(set(expected_scripts) - set(detected_scripts))
|
||||
conversation_available = 'conversation' in components
|
||||
gemini_available = 'google_generative_ai_conversation' in components
|
||||
core_script_available = core_script in entity_ids
|
||||
operational = core_script_available and conversation_available and gemini_available
|
||||
coverage = int((len(detected_scripts) / len(expected_scripts)) * 100) if expected_scripts else 100
|
||||
|
||||
steps = [
|
||||
_step('connector', 'Home Assistant connector', 'complete', 'Stored configuration is available.'),
|
||||
_step('api', 'Home Assistant API', 'complete', 'Authenticated API requests succeeded.'),
|
||||
_step(
|
||||
'scripts',
|
||||
'AppOps scripts',
|
||||
'complete' if not missing_scripts else ('partial' if detected_scripts else 'warning'),
|
||||
f'{len(detected_scripts)} of {len(expected_scripts)} allowlisted scripts detected.',
|
||||
),
|
||||
_step(
|
||||
'conversation',
|
||||
'Home Assistant Conversation',
|
||||
'complete' if conversation_available else 'warning',
|
||||
'Conversation component is loaded.' if conversation_available else 'Conversation component was not reported by Home Assistant.',
|
||||
),
|
||||
_step(
|
||||
'gemini',
|
||||
'Google Gemini Conversation',
|
||||
'complete' if gemini_available else 'warning',
|
||||
'Google Gemini Conversation integration is loaded.' if gemini_available else 'Google Gemini Conversation integration was not reported by Home Assistant.',
|
||||
),
|
||||
_step(
|
||||
'exposure',
|
||||
'Scripts exposed to Assist',
|
||||
'manual',
|
||||
'Home Assistant does not expose reliable LLM script-exposure state through this API check; verify this in Assist settings.',
|
||||
),
|
||||
]
|
||||
next_actions = []
|
||||
if missing_scripts:
|
||||
next_actions.append({'id': 'install_scripts', 'label': 'Install or reload AppOps scripts', 'href': '/voice?tab=ha'})
|
||||
if not gemini_available:
|
||||
next_actions.append({'id': 'configure_gemini', 'label': 'Configure Gemini Conversation', 'href': f'{url}/config/integrations'})
|
||||
if not conversation_available:
|
||||
next_actions.append({'id': 'configure_assist', 'label': 'Configure an Assist pipeline', 'href': f'{url}/config/voice-assistants/assistants'})
|
||||
next_actions.append({'id': 'verify_exposure', 'label': 'Verify script exposure to Assist', 'href': f'{url}/config/voice-assistants/expose'})
|
||||
|
||||
return {
|
||||
'status': 'ready' if operational else 'partial',
|
||||
'operational': operational,
|
||||
'apiReachable': True,
|
||||
'homeAssistantUrl': url,
|
||||
'missing': [],
|
||||
'expectedScriptCount': len(expected_scripts),
|
||||
'detectedScriptCount': len(detected_scripts),
|
||||
'scriptCoveragePercent': coverage,
|
||||
'detectedScripts': detected_scripts,
|
||||
'missingScripts': missing_scripts,
|
||||
'coreScriptAvailable': core_script_available,
|
||||
'conversationAvailable': conversation_available,
|
||||
'geminiConversationAvailable': gemini_available,
|
||||
'llmExposureStatus': 'manual_verification_required',
|
||||
'steps': steps,
|
||||
'nextActions': next_actions,
|
||||
'lastCheckedAt': utc_now().isoformat(),
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..db.models import AppSetting, Connector, Policy
|
||||
from ..core.time import utc_now
|
||||
from .inventory_discovery import list_drift_incidents, update_drift_incident_status
|
||||
from .secret_store import secret_storage_status
|
||||
|
||||
_INCIDENT_STATES_SETTING = 'incidents:states'
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def _incident_states(session: Session) -> dict[str, dict[str, Any]]:
|
||||
row = session.get(AppSetting, _INCIDENT_STATES_SETTING)
|
||||
value = row.value_json if row else {}
|
||||
items = value.get('items', {}) if isinstance(value, dict) else {}
|
||||
return {str(key): value for key, value in items.items() if isinstance(value, dict)}
|
||||
|
||||
|
||||
def _set_incident_states(session: Session, states: dict[str, dict[str, Any]]) -> None:
|
||||
row = session.get(AppSetting, _INCIDENT_STATES_SETTING)
|
||||
value = {'items': states}
|
||||
if not row:
|
||||
row = AppSetting(key=_INCIDENT_STATES_SETTING, value_json=value)
|
||||
else:
|
||||
row.value_json = value
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
|
||||
|
||||
def _generated_incidents(session: Session) -> list[dict[str, Any]]:
|
||||
timestamp = _now_iso()
|
||||
incidents: list[dict[str, Any]] = []
|
||||
|
||||
connectors = session.exec(select(Connector).order_by(Connector.id)).all()
|
||||
degraded_connectors = [
|
||||
connector.id
|
||||
for connector in connectors
|
||||
if connector.status in {'missing_config', 'disconnected', 'error', 'auth_failed', 'unreachable'}
|
||||
]
|
||||
if degraded_connectors:
|
||||
incidents.append({
|
||||
'id': 'incident-connectors-config',
|
||||
'severity': 'high' if len(degraded_connectors) >= 5 else 'medium',
|
||||
'status': 'open',
|
||||
'title': 'Connector configuratie of health vraagt aandacht',
|
||||
'detectedTime': timestamp,
|
||||
'evidence': {
|
||||
'source': 'Connector Registry',
|
||||
'check': 'Connector readiness',
|
||||
'observed': f"{len(degraded_connectors)} degraded connector(s): {', '.join(degraded_connectors[:8])}",
|
||||
'expected': 'Alle bekende connectors connected of bewust optioneel.',
|
||||
'timestamp': timestamp,
|
||||
},
|
||||
'suggestedActions': ['connector.configure', 'connector.test'],
|
||||
})
|
||||
|
||||
volume_policy = session.get(Policy, 'policy-volume-delete')
|
||||
if not (volume_policy and volume_policy.voice == 'blocked' and volume_policy.manual == 'blocked'):
|
||||
incidents.append({
|
||||
'id': 'incident-policy-volume-delete',
|
||||
'severity': 'critical',
|
||||
'status': 'open',
|
||||
'title': 'Destructive storage policy niet fail-closed',
|
||||
'detectedTime': timestamp,
|
||||
'evidence': {
|
||||
'source': 'Policy Engine',
|
||||
'check': 'volume.delete guardrail',
|
||||
'observed': 'volume.delete is niet volledig blocked',
|
||||
'expected': 'voice=blocked en manual=blocked',
|
||||
'timestamp': timestamp,
|
||||
},
|
||||
'suggestedActions': ['policy.review'],
|
||||
})
|
||||
|
||||
storage = secret_storage_status()
|
||||
if not storage.get('productionReady'):
|
||||
incidents.append({
|
||||
'id': 'incident-secret-storage',
|
||||
'severity': 'high',
|
||||
'status': 'open',
|
||||
'title': 'Secret storage niet production-ready',
|
||||
'detectedTime': timestamp,
|
||||
'evidence': {
|
||||
'source': 'Secret Store',
|
||||
'check': 'Encrypted-at-rest readiness',
|
||||
'observed': 'Secret storage mist production-ready configuratie.',
|
||||
'expected': 'Encrypted secret storage met persistente sleutel.',
|
||||
'timestamp': timestamp,
|
||||
},
|
||||
'suggestedActions': ['secrets.health'],
|
||||
})
|
||||
|
||||
return incidents
|
||||
|
||||
|
||||
def list_incidents(session: Session) -> list[dict[str, Any]]:
|
||||
states = _incident_states(session)
|
||||
incidents: list[dict[str, Any]] = []
|
||||
for incident in _generated_incidents(session):
|
||||
state = states.get(incident['id'], {})
|
||||
status = str(state.get('status') or 'open')
|
||||
if status == 'resolved':
|
||||
continue
|
||||
incident['status'] = status if status in {'open', 'acknowledged', 'muted'} else 'open'
|
||||
incidents.append(incident)
|
||||
incidents.extend(list_drift_incidents(session))
|
||||
return incidents
|
||||
|
||||
|
||||
def update_incident_status(session: Session, incident_id: str, status: str) -> dict[str, Any]:
|
||||
if status not in {'open', 'acknowledged', 'muted', 'resolved'}:
|
||||
status = 'acknowledged'
|
||||
|
||||
drift_update = update_drift_incident_status(session, incident_id, status)
|
||||
if drift_update:
|
||||
return drift_update
|
||||
|
||||
current_ids = {incident['id'] for incident in _generated_incidents(session)}
|
||||
states = _incident_states(session)
|
||||
if incident_id not in current_ids and incident_id not in states:
|
||||
return {'success': False, 'message': 'incident_not_found', 'result': 'failed', 'status': 'failed'}
|
||||
|
||||
states[incident_id] = {'status': status, 'updatedAt': _now_iso()}
|
||||
_set_incident_states(session, states)
|
||||
session.commit()
|
||||
return {
|
||||
'success': True,
|
||||
'message': f'Incident {incident_id} status updated',
|
||||
'result': 'success',
|
||||
'status': 'success',
|
||||
}
|
||||
|
||||
|
||||
def count_open_incidents(session: Session) -> int:
|
||||
return len([incident for incident in list_incidents(session) if incident.get('status') == 'open'])
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import JobRun
|
||||
|
||||
|
||||
def list_jobs(session: Session):
|
||||
return session.exec(select(JobRun).order_by(JobRun.started_at.desc())).all()
|
||||
|
||||
|
||||
def get_job(session: Session, job_id: str) -> JobRun | None:
|
||||
return session.get(JobRun, job_id)
|
||||
|
||||
|
||||
def create_job(
|
||||
session: Session,
|
||||
*,
|
||||
action_id: str,
|
||||
target: str,
|
||||
source: str,
|
||||
risk: str,
|
||||
policy_decision: str,
|
||||
steps: list[str],
|
||||
audit_event_id: str | None = None,
|
||||
) -> JobRun:
|
||||
job = prepare_job(
|
||||
action_id=action_id,
|
||||
target=target,
|
||||
source=source,
|
||||
risk=risk,
|
||||
policy_decision=policy_decision,
|
||||
steps=steps,
|
||||
audit_event_id=audit_event_id,
|
||||
)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def prepare_job(
|
||||
*,
|
||||
action_id: str,
|
||||
target: str,
|
||||
source: str,
|
||||
risk: str,
|
||||
policy_decision: str,
|
||||
steps: list[str],
|
||||
audit_event_id: str | None = None,
|
||||
) -> JobRun:
|
||||
"""Build a job without committing it so admission can be atomic."""
|
||||
now = utc_now()
|
||||
return JobRun(
|
||||
id=f'job-{uuid.uuid4().hex[:12]}',
|
||||
type=action_id,
|
||||
status='queued',
|
||||
source=source,
|
||||
action_id=action_id,
|
||||
target=target,
|
||||
risk=risk,
|
||||
policy_decision=policy_decision,
|
||||
steps_json=json.dumps(steps),
|
||||
progress=0,
|
||||
started_at=now,
|
||||
audit_event_id=audit_event_id,
|
||||
)
|
||||
|
||||
|
||||
def start_job(session: Session, job: JobRun) -> JobRun:
|
||||
job.status = 'running'
|
||||
job.started_at = job.started_at or utc_now()
|
||||
job.progress = 5
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def complete_job(session: Session, job: JobRun, *, status: str, result_summary: str) -> JobRun:
|
||||
job.status = status
|
||||
job.finished_at = utc_now()
|
||||
job.result_summary = result_summary
|
||||
job.progress = 100 if status == 'succeeded' else job.progress
|
||||
if job.started_at:
|
||||
job.duration_ms = int((job.finished_at - job.started_at).total_seconds() * 1000)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def cancel_job(session: Session, job: JobRun, *, result_summary: str = 'Cancelled by operator.') -> JobRun:
|
||||
job.status = 'cancelled'
|
||||
job.finished_at = utc_now()
|
||||
job.result_summary = result_summary
|
||||
if job.started_at:
|
||||
job.duration_ms = int((job.finished_at - job.started_at).total_seconds() * 1000)
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def set_job_progress(session: Session, job: JobRun, progress: int, result_summary: str | None = None) -> JobRun:
|
||||
job.progress = max(0, min(100, int(progress)))
|
||||
if result_summary:
|
||||
job.result_summary = result_summary
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return job
|
||||
@@ -0,0 +1,396 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
from threading import RLock
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy import delete, distinct, func
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.redaction import REDACTION_PATTERNS, redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import App, AppSetting, AuditEvent, KnowledgeChunk, NotificationEvent, Policy
|
||||
|
||||
|
||||
_WORD = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{1,}")
|
||||
_HEADING = re.compile(r"^(#{1,4})\s+(.+?)\s*$", re.MULTILINE)
|
||||
_BLOCKED_FILES: set[str] = set()
|
||||
_INDEX_MANIFEST_KEY = "knowledge:index_manifest"
|
||||
_MANIFEST_CACHE: dict[str, Any] = {}
|
||||
_PERSISTED_CACHE: dict[str, Any] = {}
|
||||
_INDEX_LOCK = RLock()
|
||||
_STOPWORDS = {
|
||||
"aan", "als", "bij", "dat", "de", "den", "der", "die", "dit", "een", "en", "for", "het",
|
||||
"hoe", "in", "is", "met", "naar", "of", "om", "op", "te", "the", "to", "van", "voor", "wat",
|
||||
}
|
||||
|
||||
|
||||
def _tokens(value: str) -> list[str]:
|
||||
return [token.casefold() for token in _WORD.findall(value) if token.casefold() not in _STOPWORDS]
|
||||
|
||||
|
||||
def _semantic_features(value: str) -> set[str]:
|
||||
normalized = " ".join(_tokens(value))
|
||||
words = normalized.split()
|
||||
features = {f"word:{word}" for word in words}
|
||||
features.update(f"pair:{words[index]}_{words[index + 1]}" for index in range(len(words) - 1))
|
||||
for word in words:
|
||||
if len(word) >= 5:
|
||||
features.update(f"tri:{word[index:index + 3]}" for index in range(len(word) - 2))
|
||||
return features
|
||||
|
||||
|
||||
def _expanded_query(value: str) -> str:
|
||||
normalized = value.casefold()
|
||||
expansions: list[str] = []
|
||||
if "authentic" in normalized:
|
||||
expansions.append("operator authentication private deployment security beveiliging")
|
||||
if "destruct" in normalized or "geblokkeerd" in normalized:
|
||||
expansions.append("destructive blocked policy guardrail blokkeren acties")
|
||||
if ("secret" in normalized or "geheim" in normalized) and ("encrypt" in normalized or "opgeslagen" in normalized):
|
||||
expansions.append("secret storage encrypted-at-rest Fernet SQLite")
|
||||
return " ".join([value, *expansions])
|
||||
|
||||
|
||||
def _safe_text(value: str) -> str:
|
||||
value = re.sub(r"<script\b[^>]*>.*?</script>", " ", value, flags=re.IGNORECASE | re.DOTALL)
|
||||
value = redact_message(value)
|
||||
return re.sub(r"\s+", " ", value).strip()
|
||||
|
||||
|
||||
def _split_markdown(source_uri: str, raw: str) -> Iterable[tuple[str, str, str]]:
|
||||
matches = list(_HEADING.finditer(raw))
|
||||
title = _safe_text(Path(source_uri).stem.replace("_", " ").replace("-", " ").title())[:200]
|
||||
if matches:
|
||||
title = _safe_text(matches[0].group(2))[:200]
|
||||
boundaries = matches or [re.match(r"", raw)]
|
||||
for index, match in enumerate(boundaries):
|
||||
start = match.end() if match else 0
|
||||
end = boundaries[index + 1].start() if index + 1 < len(boundaries) else len(raw)
|
||||
section = _safe_text(match.group(2))[:200] if match and match.lastindex else title
|
||||
body = _safe_text(raw[start:end])
|
||||
if not body:
|
||||
continue
|
||||
for offset in range(0, len(body), 1800):
|
||||
content = body[offset:offset + 2200].strip()
|
||||
if len(content) >= 40:
|
||||
yield title, section, content
|
||||
|
||||
|
||||
def _knowledge_files(root: Path) -> list[Path]:
|
||||
candidates = [root / "README.md", root / "CHANGELOG.md"]
|
||||
docs = root / "docs"
|
||||
if docs.is_dir():
|
||||
candidates.extend(sorted(docs.glob("*.md")))
|
||||
return [
|
||||
path for path in candidates[:160]
|
||||
if path.is_file() and path.name.casefold() not in _BLOCKED_FILES and path.stat().st_size <= 1_000_000
|
||||
]
|
||||
|
||||
|
||||
def _source_manifest(root: Path) -> dict[str, Any]:
|
||||
files = _knowledge_files(root)
|
||||
stat_signature = tuple(
|
||||
(path.relative_to(root).as_posix(), path.stat().st_size, path.stat().st_mtime_ns)
|
||||
for path in files
|
||||
)
|
||||
cached = _MANIFEST_CACHE.get(str(root))
|
||||
if cached and cached.get("statSignature") == stat_signature:
|
||||
return dict(cached["manifest"])
|
||||
digest = hashlib.sha256()
|
||||
for path in files:
|
||||
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(path.read_bytes())
|
||||
digest.update(b"\0")
|
||||
manifest = {"hash": digest.hexdigest(), "sourceCount": len(files)}
|
||||
_MANIFEST_CACHE[str(root)] = {"statSignature": stat_signature, "manifest": manifest}
|
||||
return dict(manifest)
|
||||
|
||||
|
||||
def _redaction_match_count(value: str) -> int:
|
||||
return sum(len(pattern.findall(value)) for pattern in REDACTION_PATTERNS)
|
||||
|
||||
|
||||
def _compact_excerpt(value: str, limit: int = 360) -> str:
|
||||
sentences = re.split(r"(?<=[.!?])\s+", value.strip())
|
||||
summary = " ".join(sentences[:2]).strip()
|
||||
candidate = summary or value.strip()
|
||||
return candidate if len(candidate) <= limit else f"{candidate[:limit - 1].rstrip()}…"
|
||||
|
||||
|
||||
def _rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
|
||||
root = (root or Path.cwd()).resolve()
|
||||
manifest = _source_manifest(root)
|
||||
rows: list[KnowledgeChunk] = []
|
||||
sources: set[str] = set()
|
||||
redacted_matches = 0
|
||||
for path in _knowledge_files(root):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
source_uri = f"appops-doc://{relative}"
|
||||
raw = path.read_text(encoding="utf-8", errors="replace")
|
||||
redacted_matches += _redaction_match_count(raw)
|
||||
source_has_chunks = False
|
||||
for ordinal, (title, section, content) in enumerate(_split_markdown(relative, raw)):
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
rows.append(KnowledgeChunk(
|
||||
id=f"kc-{hashlib.sha256(f'{source_uri}:{ordinal}:{digest}'.encode()).hexdigest()[:20]}",
|
||||
source_uri=source_uri,
|
||||
source_type="documentation",
|
||||
title=title,
|
||||
section=section,
|
||||
content=content,
|
||||
content_hash=digest,
|
||||
))
|
||||
source_has_chunks = True
|
||||
if source_has_chunks:
|
||||
sources.add(source_uri)
|
||||
session.exec(delete(KnowledgeChunk))
|
||||
session.add_all(rows)
|
||||
setting = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
||||
if not setting:
|
||||
setting = AppSetting(key=_INDEX_MANIFEST_KEY, is_secret=False)
|
||||
generated_at = datetime.now(UTC).isoformat()
|
||||
setting.value_json = {
|
||||
"manifestHash": manifest["hash"],
|
||||
"sourceCount": len(sources),
|
||||
"chunkCount": len(rows),
|
||||
"generatedAt": generated_at,
|
||||
"redactedMatches": redacted_matches,
|
||||
"secretScanStatus": "passed_with_redactions" if redacted_matches else "passed",
|
||||
}
|
||||
setting.updated_at = utc_now()
|
||||
session.add(setting)
|
||||
session.commit()
|
||||
_PERSISTED_CACHE.clear()
|
||||
return {
|
||||
"status": "ready",
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"chunks": len(rows),
|
||||
"sources": len(sources),
|
||||
"generatedAt": generated_at,
|
||||
"manifestHash": manifest["hash"],
|
||||
"redactedMatches": redacted_matches,
|
||||
"secretScanStatus": setting.value_json["secretScanStatus"],
|
||||
}
|
||||
|
||||
|
||||
def rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
|
||||
with _INDEX_LOCK:
|
||||
return _rebuild_index(session, root)
|
||||
|
||||
|
||||
def index_status(session: Session) -> dict[str, Any]:
|
||||
chunks, sources, last_indexed = session.exec(
|
||||
select(
|
||||
func.count(KnowledgeChunk.id),
|
||||
func.count(distinct(KnowledgeChunk.source_uri)),
|
||||
func.max(KnowledgeChunk.updated_at),
|
||||
)
|
||||
).one()
|
||||
manifest = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
||||
metadata = manifest.value_json if manifest and isinstance(manifest.value_json, dict) else {}
|
||||
return {
|
||||
"status": "ready" if chunks else "empty",
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"chunks": chunks,
|
||||
"sources": sources,
|
||||
"lastIndexedAt": metadata.get("generatedAt") or last_indexed,
|
||||
"secretsIndexed": False,
|
||||
"secretScanStatus": metadata.get("secretScanStatus", "unknown"),
|
||||
"redactedMatches": int(metadata.get("redactedMatches", 0)),
|
||||
"manifestHash": metadata.get("manifestHash"),
|
||||
"generationMode": "extractive_grounded",
|
||||
}
|
||||
|
||||
|
||||
def ensure_index_current(session: Session, root: Path | None = None) -> dict[str, Any]:
|
||||
with _INDEX_LOCK:
|
||||
root = (root or Path.cwd()).resolve()
|
||||
current = _source_manifest(root)
|
||||
stored = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
||||
stored_hash = stored.value_json.get("manifestHash") if stored and isinstance(stored.value_json, dict) else None
|
||||
has_chunks = session.exec(select(KnowledgeChunk.id).limit(1)).first() is not None
|
||||
if not has_chunks or stored_hash != current["hash"]:
|
||||
return _rebuild_index(session, root)
|
||||
return index_status(session)
|
||||
|
||||
|
||||
def _runtime_chunks(session: Session) -> list[dict[str, str]]:
|
||||
chunks: list[dict[str, str]] = []
|
||||
for app in session.exec(select(App).order_by(App.name)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-app-{app.id}", "source_uri": f"appops://apps/{app.id}", "source_type": "live_app",
|
||||
"title": app.name, "section": "Actuele appstatus",
|
||||
"content": _safe_text(
|
||||
f"App {app.name}; categorie {app.category}; status {app.status}; criticality {app.criticality}; "
|
||||
f"connector {app.connector_status}; container {app.container_status}; protected {app.protected}; review {app.needs_review}."
|
||||
),
|
||||
})
|
||||
for policy in session.exec(select(Policy).order_by(Policy.action)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-policy-{policy.id}", "source_uri": f"appops://policies/{policy.id}", "source_type": "live_policy",
|
||||
"title": policy.action, "section": "Actuele policy",
|
||||
"content": _safe_text(
|
||||
f"Actie {policy.action}; target {policy.target}; manual {policy.manual}; voice {policy.voice}; "
|
||||
f"risk {policy.risk}; reden {policy.reason or 'geen'}"
|
||||
),
|
||||
})
|
||||
for event in session.exec(select(AuditEvent).order_by(AuditEvent.timestamp.desc()).limit(80)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-audit-{event.id}", "source_uri": f"appops://audit/{event.id}", "source_type": "live_audit",
|
||||
"title": event.action_id, "section": "Recent audit-event",
|
||||
"content": _safe_text(
|
||||
f"{event.timestamp.isoformat()} actie {event.action_id}; target {event.target}; bron {event.source}; "
|
||||
f"policy {event.policy_decision}; resultaat {event.result}; reden {event.reason or 'geen'}"
|
||||
),
|
||||
})
|
||||
for event in session.exec(select(NotificationEvent).order_by(NotificationEvent.created_at.desc()).limit(40)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-notification-{event.id}", "source_uri": f"appops://notifications/{event.id}", "source_type": "live_notification",
|
||||
"title": _safe_text(event.title), "section": "Recente notificatie",
|
||||
"content": _safe_text(f"{event.severity} {event.source} {event.category or ''}: {event.message}; status {event.status}"),
|
||||
})
|
||||
return chunks
|
||||
|
||||
|
||||
def query(session: Session, question: str, *, limit: int = 6, root: Path | None = None) -> dict[str, Any]:
|
||||
question = _safe_text(question)[:500]
|
||||
if len(question) < 2:
|
||||
raise ValueError("question_required")
|
||||
status = ensure_index_current(session, root)
|
||||
cache_key = str(status.get("manifestHash") or status.get("lastIndexedAt") or "empty")
|
||||
if _PERSISTED_CACHE.get("key") != cache_key:
|
||||
persisted = [
|
||||
{
|
||||
"id": row.id, "source_uri": row.source_uri, "source_type": row.source_type,
|
||||
"title": row.title, "section": row.section or row.title, "content": row.content,
|
||||
}
|
||||
for row in session.exec(select(KnowledgeChunk)).all()
|
||||
]
|
||||
_PERSISTED_CACHE.update({
|
||||
"key": cache_key,
|
||||
"items": persisted,
|
||||
"terms": [Counter(_tokens(f"{item['title']} {item['section']} {item['content']}")) for item in persisted],
|
||||
"features": [_semantic_features(f"{item['title']} {item['section']} {item['content']}") for item in persisted],
|
||||
})
|
||||
persisted = list(_PERSISTED_CACHE.get("items", []))
|
||||
persisted_terms = list(_PERSISTED_CACHE.get("terms", []))
|
||||
persisted_features = list(_PERSISTED_CACHE.get("features", []))
|
||||
candidates = persisted + _runtime_chunks(session)
|
||||
expanded_question = _expanded_query(question)
|
||||
query_terms = Counter(_tokens(expanded_question))
|
||||
if not query_terms:
|
||||
raise ValueError("question_has_no_search_terms")
|
||||
document_frequency: Counter[str] = Counter()
|
||||
tokenized: list[tuple[dict[str, str], Counter[str], set[str]]] = []
|
||||
for index, item in enumerate(candidates):
|
||||
terms = persisted_terms[index] if index < len(persisted_terms) else Counter(
|
||||
_tokens(f"{item['title']} {item['section']} {item['content']}")
|
||||
)
|
||||
features = persisted_features[index] if index < len(persisted_features) else _semantic_features(
|
||||
f"{item['title']} {item['section']} {item['content']}"
|
||||
)
|
||||
tokenized.append((item, terms, features))
|
||||
document_frequency.update(terms.keys())
|
||||
scored: list[tuple[float, dict[str, str]]] = []
|
||||
total = max(1, len(tokenized))
|
||||
query_features = _semantic_features(expanded_question)
|
||||
for item, terms, features in tokenized:
|
||||
score = 0.0
|
||||
title_terms = set(_tokens(f"{item['title']} {item['section']}"))
|
||||
for term, query_count in query_terms.items():
|
||||
if not terms[term]:
|
||||
continue
|
||||
idf = math.log((total + 1) / (document_frequency[term] + 1)) + 1
|
||||
score += query_count * (1 + math.log(terms[term])) * idf
|
||||
if term in title_terms:
|
||||
score += 1.5 * idf
|
||||
overlap = len(query_features.intersection(features))
|
||||
semantic_score = overlap / max(1, math.sqrt(len(query_features) * len(features)))
|
||||
score += semantic_score * 4.0
|
||||
if score > 0.15:
|
||||
scored.append((score, item))
|
||||
selected: list[tuple[float, dict[str, str]]] = []
|
||||
source_counts: Counter[str] = Counter()
|
||||
requested_limit = max(1, min(limit, 10))
|
||||
for score, item in sorted(scored, key=lambda value: value[0], reverse=True):
|
||||
source_uri = item["source_uri"]
|
||||
if source_counts[source_uri] >= 2:
|
||||
continue
|
||||
selected.append((score, item))
|
||||
source_counts[source_uri] += 1
|
||||
if len(selected) >= requested_limit:
|
||||
break
|
||||
results = []
|
||||
for rank, (score, item) in enumerate(selected, start=1):
|
||||
results.append({
|
||||
"id": item["id"], "rank": rank, "title": item["title"], "section": item["section"],
|
||||
"uri": item["source_uri"], "sourceType": item["source_type"], "score": round(score, 3),
|
||||
"excerpt": item["content"][:700],
|
||||
})
|
||||
if not results:
|
||||
answer = "Er is geen voldoende relevante, brongebonden context gevonden. Er wordt geen antwoord gegokt."
|
||||
else:
|
||||
evidence = "\n".join(
|
||||
f"[{result['rank']}] {result['title']}: {_compact_excerpt(result['excerpt'])}"
|
||||
for result in results[:3]
|
||||
)
|
||||
answer = f"Gevonden broncontext:\n{evidence}"
|
||||
return {
|
||||
"question": question,
|
||||
"answer": answer,
|
||||
"confidence": "high" if len(results) >= 3 else "medium" if results else "none",
|
||||
"citations": results,
|
||||
"guardrails": {
|
||||
"readOnly": True,
|
||||
"secretsExcluded": True,
|
||||
"instructionsFromSourcesIgnored": True,
|
||||
"executionViaPolicyOnly": True,
|
||||
},
|
||||
"retrieval": {
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
||||
"externalEmbeddingProvider": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_RETRIEVAL_EVALUATION_CASES = (
|
||||
("Hoe werkt private operator authenticatie?", ("PRIVATE_DEPLOYMENT_SECURITY", "API_CONTRACT")),
|
||||
("Hoe controleer en herstel ik een SQLite backup?", ("BACKUP_RESTORE", "RELEASE_CHECKLIST")),
|
||||
("Hoe zijn destructieve acties geblokkeerd?", ("API_CONTRACT", "HARDENING")),
|
||||
("Hoe koppelen MCP, RAG en n8n?", ("MCP_RAG_N8N_INTEGRATION",)),
|
||||
("Hoe worden connector secrets encrypted-at-rest opgeslagen?", ("SECRET_STORAGE",)),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_retrieval(session: Session) -> dict[str, Any]:
|
||||
cases = []
|
||||
for question, expected_sources in _RETRIEVAL_EVALUATION_CASES:
|
||||
result = query(session, question, limit=5)
|
||||
uris = [str(item.get("uri") or "") for item in result.get("citations", [])]
|
||||
passed = any(expected.lower() in uri.lower() for expected in expected_sources for uri in uris)
|
||||
cases.append({
|
||||
"question": question,
|
||||
"passed": passed,
|
||||
"expectedSourceHints": list(expected_sources),
|
||||
"topSourceUris": uris[:3],
|
||||
"citationCount": len(uris),
|
||||
})
|
||||
passed_count = sum(1 for case in cases if case["passed"])
|
||||
return {
|
||||
"status": "passed" if passed_count == len(cases) else "needs_tuning",
|
||||
"evaluationVersion": 2,
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
||||
"passed": passed_count,
|
||||
"total": len(cases),
|
||||
"passRate": round(passed_count / max(1, len(cases)), 3),
|
||||
"cases": cases,
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from threading import Lock
|
||||
from time import monotonic
|
||||
from time import perf_counter
|
||||
from typing import Callable
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..api.routes_observability import network_exposure
|
||||
from ..core.config import settings
|
||||
from ..db import session as db_session
|
||||
from ..mcp_server import persist_tool_metrics, run_read_only_canary
|
||||
from . import media_pipeline_service, notification_service, safety_center_service
|
||||
from .connector_health import refresh_connector_statuses
|
||||
from .request_metrics import persist_request_metrics
|
||||
from .runtime_service import verify_latest_backup_drill
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_STATUS_LOCK = Lock()
|
||||
_TASK_STATUS: dict[str, dict[str, object]] = {}
|
||||
|
||||
|
||||
def maintenance_status() -> dict[str, object]:
|
||||
with _STATUS_LOCK:
|
||||
tasks = {name: dict(status) for name, status in _TASK_STATUS.items()}
|
||||
return {
|
||||
"status": "running",
|
||||
"tasks": tasks,
|
||||
"intervalsSeconds": {
|
||||
"notificationEvaluation": settings.notification_interval_seconds,
|
||||
"connectorHealth": settings.connector_health_interval_seconds,
|
||||
"observabilityPersistence": settings.observability_persist_interval_seconds,
|
||||
"backupRestoreDrill": settings.restore_drill_interval_seconds,
|
||||
"mcpCanary": settings.mcp_canary_interval_seconds if settings.mcp_enabled else 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def warm_operational_caches() -> None:
|
||||
with Session(db_session.engine) as session:
|
||||
safety_center_service.get_dashboard(session)
|
||||
media_pipeline_service.summary(session)
|
||||
network_exposure(False, session)
|
||||
|
||||
|
||||
def evaluate_notifications_and_retries() -> None:
|
||||
with Session(db_session.engine) as session:
|
||||
notification_service.retry_failed_deliveries(session)
|
||||
notification_service.evaluate_source(session, "all")
|
||||
|
||||
|
||||
def reconcile_connector_health() -> None:
|
||||
with Session(db_session.engine) as session:
|
||||
refresh_connector_statuses(session)
|
||||
|
||||
|
||||
def persist_observability() -> None:
|
||||
with Session(db_session.engine) as session:
|
||||
persist_request_metrics(session)
|
||||
persist_tool_metrics(session)
|
||||
|
||||
|
||||
def run_restore_drill() -> None:
|
||||
with Session(db_session.engine) as session:
|
||||
verify_latest_backup_drill(session)
|
||||
|
||||
|
||||
async def _safe_thread(label: str, function: Callable[[], object]) -> None:
|
||||
started = perf_counter()
|
||||
try:
|
||||
await asyncio.to_thread(function)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - maintenance failures must not stop the API.
|
||||
logger.exception("Scheduled %s failed", label)
|
||||
state = {"status": "failed", "error": "scheduled_task_failed"}
|
||||
else:
|
||||
state = {"status": "ok"}
|
||||
state.update(
|
||||
{
|
||||
"lastRunAt": datetime.now(UTC).isoformat(),
|
||||
"durationMs": round((perf_counter() - started) * 1000, 2),
|
||||
}
|
||||
)
|
||||
with _STATUS_LOCK:
|
||||
_TASK_STATUS[label] = state
|
||||
|
||||
|
||||
async def run_periodic_maintenance() -> None:
|
||||
await asyncio.sleep(2)
|
||||
if settings.connector_health_interval_seconds:
|
||||
await _safe_thread("connector health reconciliation", reconcile_connector_health)
|
||||
await _safe_thread("operational cache warmup", warm_operational_caches)
|
||||
|
||||
now = monotonic()
|
||||
next_notification = now + min(15, settings.notification_interval_seconds) if settings.notification_interval_seconds else float("inf")
|
||||
next_connector_health = now + settings.connector_health_interval_seconds if settings.connector_health_interval_seconds else float("inf")
|
||||
next_observability = now + min(10, settings.observability_persist_interval_seconds) if settings.observability_persist_interval_seconds else float("inf")
|
||||
next_restore = now + min(60, settings.restore_drill_interval_seconds) if settings.restore_drill_interval_seconds else float("inf")
|
||||
next_canary = now + min(10, settings.mcp_canary_interval_seconds) if settings.mcp_enabled and settings.mcp_canary_interval_seconds else float("inf")
|
||||
|
||||
while True:
|
||||
now = monotonic()
|
||||
if now >= next_notification:
|
||||
await _safe_thread("notification evaluation", evaluate_notifications_and_retries)
|
||||
next_notification = monotonic() + settings.notification_interval_seconds
|
||||
if now >= next_connector_health:
|
||||
await _safe_thread("connector health reconciliation", reconcile_connector_health)
|
||||
next_connector_health = monotonic() + settings.connector_health_interval_seconds
|
||||
if now >= next_canary:
|
||||
await _safe_thread("MCP read-only canary", run_read_only_canary)
|
||||
next_canary = monotonic() + settings.mcp_canary_interval_seconds
|
||||
if now >= next_observability:
|
||||
await _safe_thread("observability persistence", persist_observability)
|
||||
next_observability = monotonic() + settings.observability_persist_interval_seconds
|
||||
if now >= next_restore:
|
||||
await _safe_thread("backup restore drill", run_restore_drill)
|
||||
next_restore = monotonic() + settings.restore_drill_interval_seconds
|
||||
await asyncio.sleep(5)
|
||||
@@ -0,0 +1,691 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/itworx_mcp_hub"
|
||||
_COMPOSE_PROJECT = "itworx-mcp-hub"
|
||||
|
||||
# One-time fail-closed re-attestation for the existing MCP Hub runtime. The
|
||||
# script never builds, restarts, recreates, stops or deploys a container. It
|
||||
# repairs only controller state after proving the exact target, Git ancestry,
|
||||
# complete Compose topology and every currently running image identity.
|
||||
_RECONCILE_SCRIPT = r'''
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
repository, expected_compose_project = sys.argv[1:3]
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
owned_image_source = "https://gitea.example.com/jens/itworx-mcp-hub"
|
||||
|
||||
if repository != "jens/itworx_mcp_hub" or expected_compose_project != "itworx-mcp-hub":
|
||||
raise SystemExit(40)
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(20)
|
||||
try:
|
||||
if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path):
|
||||
raise SystemExit(36)
|
||||
except OSError:
|
||||
raise SystemExit(36)
|
||||
|
||||
try:
|
||||
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
if not isinstance(project, dict):
|
||||
raise SystemExit(22)
|
||||
expected_containers = project.get("containers")
|
||||
expected_fixed = {
|
||||
"itworx-mcp-hub-connector-appops-1",
|
||||
"itworx-mcp-hub-connector-gitea-1",
|
||||
"itworx-mcp-hub-connector-knowledge-1",
|
||||
"itworx-mcp-hub-connector-mobilityops-1",
|
||||
"itworx-mcp-hub-connector-projectbrain-1",
|
||||
"itworx-mcp-hub-connector-stitch-1",
|
||||
"itworx-mcp-hub-connector-unraid-1",
|
||||
"itworx-mcp-hub-contextforge-1",
|
||||
"itworx-mcp-hub-control-api-1",
|
||||
"itworx-mcp-hub-dataplane-router-1",
|
||||
"itworx-mcp-hub-docker-read-broker-1",
|
||||
"itworx-mcp-hub-postgres-1",
|
||||
"itworx-mcp-hub-redis-1",
|
||||
"itworx-mcp-hub-web-1",
|
||||
"itworx-mcp-hub-worker-1",
|
||||
}
|
||||
if (
|
||||
project.get("branch") != "main"
|
||||
or project.get("slug") != expected_compose_project
|
||||
or project.get("compose_project") != expected_compose_project
|
||||
or not isinstance(expected_containers, list)
|
||||
or set(expected_containers) != expected_fixed
|
||||
):
|
||||
raise SystemExit(23)
|
||||
|
||||
state_path = state_root / f"{expected_compose_project}.json"
|
||||
project_lock_path = state_root / "locks" / f"{expected_compose_project}.lock"
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")
|
||||
canonical_bare_repo = canonical_repository_root / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir() or bare_repo.is_symlink():
|
||||
raise SystemExit(24)
|
||||
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def canonical_git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={canonical_bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def canonical_image_ancestry_is_proven(recorded_revision, image_revision, expected_main):
|
||||
if not canonical_bare_repo.is_dir() or canonical_bare_repo.is_symlink():
|
||||
return False
|
||||
resolved_recorded = canonical_git_run("rev-parse", f"{recorded_revision}^{{commit}}")
|
||||
resolved_image = canonical_git_run("rev-parse", f"{image_revision}^{{commit}}")
|
||||
resolved_main = canonical_git_run("rev-parse", "refs/heads/main")
|
||||
if not (
|
||||
resolved_recorded.returncode == 0
|
||||
and resolved_recorded.stdout.strip() == recorded_revision
|
||||
and resolved_image.returncode == 0
|
||||
and resolved_image.stdout.strip() == image_revision
|
||||
and resolved_main.returncode == 0
|
||||
and resolved_main.stdout.strip() == expected_main
|
||||
):
|
||||
return False
|
||||
return (
|
||||
canonical_git_run(
|
||||
"merge-base", "--is-ancestor", recorded_revision, image_revision
|
||||
).returncode
|
||||
== 0
|
||||
and canonical_git_run(
|
||||
"merge-base", "--is-ancestor", image_revision, expected_main
|
||||
).returncode
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
main_revision = main.stdout.strip()
|
||||
if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None:
|
||||
raise SystemExit(26)
|
||||
|
||||
|
||||
def main_declares_external_image(image_ref):
|
||||
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{1,300}@sha256:[0-9a-f]{64}", image_ref) is None:
|
||||
return False
|
||||
for compose_path in (
|
||||
"infrastructure/compose/docker-compose.yml",
|
||||
"infrastructure/compose/docker-compose.prod.yml",
|
||||
):
|
||||
source = git_run("show", f"{main_revision}:{compose_path}")
|
||||
if source.returncode != 0:
|
||||
return False
|
||||
for line in source.stdout.splitlines():
|
||||
if line.strip() in {f"image: {image_ref}", f'image: "{image_ref}"', f"image: '{image_ref}'"}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main_declares_sessionless_dataplane_image(container_name, image_ref):
|
||||
if (
|
||||
container_name != "itworx-mcp-hub-dataplane-router-1"
|
||||
or image_ref != "itworx-mcp-hub/dataplane:sessionless-final"
|
||||
):
|
||||
return False
|
||||
sessionless_commit = "b519a8c4cf17cd9df00a575acfe096838aab7789"
|
||||
resolved = git_run("rev-parse", f"{sessionless_commit}^{{commit}}")
|
||||
if resolved.returncode != 0 or resolved.stdout.strip() != sessionless_commit:
|
||||
return False
|
||||
if git_run("merge-base", "--is-ancestor", sessionless_commit, main_revision).returncode != 0:
|
||||
return False
|
||||
|
||||
router = git_run(
|
||||
"show",
|
||||
f"{sessionless_commit}:services/dataplane_router/router.py",
|
||||
)
|
||||
tests = git_run(
|
||||
"show",
|
||||
f"{sessionless_commit}:tests/test_dataplane_session_headers.py",
|
||||
)
|
||||
evidence = git_run(
|
||||
"show",
|
||||
f"{main_revision}:evidence/mcp-session-boundary-20260916/SUMMARY.md",
|
||||
)
|
||||
if router.returncode != 0 or tests.returncode != 0 or evidence.returncode != 0:
|
||||
return False
|
||||
|
||||
router_contract = (
|
||||
'MCP_TRANSPORT_STATE_HEADERS = frozenset({b"mcp-session-id"})',
|
||||
'if "mcp-session-id" in response.headers:',
|
||||
'del response.headers["mcp-session-id"]',
|
||||
)
|
||||
test_contract = (
|
||||
"2025-11-25",
|
||||
"2026-07-28",
|
||||
"mcp-session-id",
|
||||
)
|
||||
evidence_contract = (
|
||||
sessionless_commit,
|
||||
"itworx-mcp-hub/dataplane:sessionless-final",
|
||||
"permanently enforce stateless public transport",
|
||||
)
|
||||
return (
|
||||
all(fragment in router.stdout for fragment in router_contract)
|
||||
and all(fragment in tests.stdout for fragment in test_contract)
|
||||
and all(fragment in evidence.stdout for fragment in evidence_contract)
|
||||
)
|
||||
|
||||
|
||||
def main_declares_contextforge_compat_image(container_name, image_ref):
|
||||
if container_name != "itworx-mcp-hub-contextforge-1":
|
||||
return False
|
||||
manifest = git_run(
|
||||
"show",
|
||||
f"{main_revision}:infrastructure/images/contextforge/compat-image.env",
|
||||
)
|
||||
builder = git_run(
|
||||
"show",
|
||||
f"{main_revision}:scripts/build_contextforge_compat_image.sh",
|
||||
)
|
||||
if manifest.returncode != 0 or builder.returncode != 0:
|
||||
return False
|
||||
|
||||
values = {}
|
||||
for raw_line in manifest.stdout.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
return False
|
||||
key, value = line.split("=", 1)
|
||||
if key in values:
|
||||
return False
|
||||
values[key] = value.strip().strip('"').strip("'")
|
||||
|
||||
expected_keys = {
|
||||
"CONTEXTFORGE_UPSTREAM_COMMIT",
|
||||
"CONTEXTFORGE_ALEMBIC_HEAD",
|
||||
"CONTEXTFORGE_MCP_SDK",
|
||||
"CONTEXTFORGE_LOCAL_IMAGE",
|
||||
}
|
||||
upstream_commit = str(values.get("CONTEXTFORGE_UPSTREAM_COMMIT") or "")
|
||||
alembic_head = str(values.get("CONTEXTFORGE_ALEMBIC_HEAD") or "")
|
||||
mcp_sdk = str(values.get("CONTEXTFORGE_MCP_SDK") or "")
|
||||
local_image = str(values.get("CONTEXTFORGE_LOCAL_IMAGE") or "")
|
||||
if (
|
||||
set(values) != expected_keys
|
||||
or re.fullmatch(r"[0-9a-f]{40}", upstream_commit) is None
|
||||
or re.fullmatch(r"[0-9a-f]{12}", alembic_head) is None
|
||||
or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", mcp_sdk) is None
|
||||
or local_image != image_ref
|
||||
or local_image != f"itworx-mcp-hub/contextforge-schema:{alembic_head}"
|
||||
):
|
||||
return False
|
||||
|
||||
required_builder_contract = (
|
||||
'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"',
|
||||
'git -C "$SRC" checkout --detach FETCH_HEAD',
|
||||
'MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"',
|
||||
'alembic -c mcpgateway/alembic.ini heads',
|
||||
'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"',
|
||||
)
|
||||
return all(fragment in builder.stdout for fragment in required_builder_contract)
|
||||
|
||||
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
||||
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
with project_lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
with global_lock:
|
||||
try:
|
||||
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
try:
|
||||
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
if current_projects.get(repository) != project:
|
||||
raise SystemExit(37)
|
||||
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(42)
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(42)
|
||||
if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository:
|
||||
raise SystemExit(43)
|
||||
|
||||
recorded_live = str(state.get("live_revision") or state.get("revision") or "")
|
||||
recorded_status = str(state.get("status") or "")
|
||||
recorded_live_status = str(state.get("live_status") or "")
|
||||
last_attempt = dict(state.get("last_attempt") or {})
|
||||
attempt_revision = str(last_attempt.get("revision") or "")
|
||||
attempt_status = str(last_attempt.get("status") or "")
|
||||
last_attempt_message = str(last_attempt.get("message") or "")
|
||||
receipt = state.get("live_receipt")
|
||||
recorded_artifacts = state.get("live_artifacts")
|
||||
|
||||
already_reconciled = (
|
||||
state.get("live_legacy_provenance") is True
|
||||
and recorded_status == "success"
|
||||
and recorded_live_status == "healthy"
|
||||
and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None
|
||||
and isinstance(recorded_artifacts, dict)
|
||||
)
|
||||
if not already_reconciled and (
|
||||
recorded_status != "failed"
|
||||
or recorded_live_status not in {"healthy", "unknown"}
|
||||
or state.get("live_legacy_provenance") is not False
|
||||
or attempt_status != "failed"
|
||||
or re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None
|
||||
or re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is None
|
||||
or attempt_revision == recorded_live
|
||||
or not isinstance(receipt, dict)
|
||||
or receipt.get("schema") != "gitea-deploy-receipt.v1"
|
||||
or receipt.get("repository") != repository
|
||||
or receipt.get("revision") != recorded_live
|
||||
or not isinstance(recorded_artifacts, dict)
|
||||
):
|
||||
raise SystemExit(52)
|
||||
|
||||
# deploy.py deliberately downgrades the prior live status to unknown
|
||||
# when capture_compose_rollback() fails before production mutation.
|
||||
# Accept that state only when the durable attempt message is one of the
|
||||
# bounded snapshot-stage failure classes and no rollback/source-restore
|
||||
# error was recorded. The runtime is still re-proven container by
|
||||
# container below before any controller state is changed.
|
||||
snapshot_failure_prefixes = (
|
||||
"Refusing to mutate a Compose project with unrecorded containers:",
|
||||
"Deployment state claims a live revision but all production containers are missing",
|
||||
"Cannot capture a complete rollback snapshot; containers are missing:",
|
||||
"Existing production containers require a valid healthy live deployment state",
|
||||
"Container ",
|
||||
"Existing deployment state is not bound to its live revision",
|
||||
"Recorded deployment receipt ",
|
||||
"Existing deployment no longer matches its verified live receipt",
|
||||
"Expected containers are not healthy/running:",
|
||||
"Compose project container set does not match the allowlisted runtime:",
|
||||
"Containers are linked to the wrong Compose project:",
|
||||
"Running container image IDs do not match the verified artifacts:",
|
||||
"Immutable image provenance does not match the requested deployment:",
|
||||
)
|
||||
if (
|
||||
not already_reconciled
|
||||
and recorded_live_status == "unknown"
|
||||
and (
|
||||
not last_attempt_message.startswith(snapshot_failure_prefixes)
|
||||
or "; rollback error:" in last_attempt_message
|
||||
or "; source rollback error:" in last_attempt_message
|
||||
)
|
||||
):
|
||||
raise SystemExit(55)
|
||||
|
||||
resolved_live = git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
||||
if resolved_live.returncode != 0 or resolved_live.stdout.strip() != recorded_live:
|
||||
raise SystemExit(25)
|
||||
if git_run("merge-base", "--is-ancestor", recorded_live, main_revision).returncode != 0:
|
||||
raise SystemExit(52)
|
||||
if not already_reconciled:
|
||||
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision:
|
||||
raise SystemExit(25)
|
||||
if git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision).returncode != 0:
|
||||
raise SystemExit(52)
|
||||
if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(52)
|
||||
|
||||
listed = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"--all",
|
||||
"--filter",
|
||||
f"label=com.docker.compose.project={expected_compose_project}",
|
||||
"--format",
|
||||
"{{.Names}}",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
project_names = [line.strip() for line in listed.stdout.splitlines() if line.strip()]
|
||||
if listed.returncode != 0 or set(project_names) != set(expected_containers):
|
||||
raise SystemExit(47)
|
||||
|
||||
old_containers = recorded_artifacts.get("containers")
|
||||
if not isinstance(old_containers, dict) or set(old_containers) != set(expected_containers):
|
||||
raise SystemExit(52)
|
||||
|
||||
container_artifacts = {}
|
||||
drift_found = False
|
||||
for name in expected_containers:
|
||||
inspected = subprocess.run(
|
||||
["docker", "inspect", name],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(44)
|
||||
try:
|
||||
container = json.loads(inspected.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(44)
|
||||
runtime_state = dict(container.get("State") or {})
|
||||
health = str(dict(runtime_state.get("Health") or {}).get("Status") or "none")
|
||||
labels = dict(dict(container.get("Config") or {}).get("Labels") or {})
|
||||
config = dict(container.get("Config") or {})
|
||||
image_id = str(container.get("Image") or "")
|
||||
image_ref = str(config.get("Image") or "")
|
||||
if not runtime_state.get("Running") or health in {"starting", "unhealthy"}:
|
||||
raise SystemExit(45)
|
||||
if labels.get("com.docker.compose.project") != expected_compose_project:
|
||||
raise SystemExit(47)
|
||||
if re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None or not image_ref:
|
||||
raise SystemExit(46)
|
||||
|
||||
image_result = subprocess.run(
|
||||
["docker", "image", "inspect", image_id],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if image_result.returncode != 0:
|
||||
raise SystemExit(50)
|
||||
try:
|
||||
image = json.loads(image_result.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(50)
|
||||
image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {})
|
||||
image_source = str(image_labels.get("org.opencontainers.image.source") or "")
|
||||
image_revision = str(image_labels.get("org.opencontainers.image.revision") or "")
|
||||
|
||||
recorded = old_containers.get(name)
|
||||
if not isinstance(recorded, dict):
|
||||
raise SystemExit(52)
|
||||
recorded_image_id = str(recorded.get("image_id") or "")
|
||||
recorded_container_id = str(recorded.get("container_id") or "")
|
||||
if re.fullmatch(r"sha256:[0-9a-f]{64}", recorded_image_id) is None:
|
||||
raise SystemExit(52)
|
||||
|
||||
image_changed = image_id != recorded_image_id
|
||||
if image_changed:
|
||||
contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref)
|
||||
sessionless_dataplane_image = (
|
||||
not already_reconciled
|
||||
and main_declares_sessionless_dataplane_image(name, image_ref)
|
||||
)
|
||||
if contextforge_compat_image or sessionless_dataplane_image:
|
||||
pass
|
||||
elif image_source == owned_image_source:
|
||||
if re.fullmatch(r"[0-9a-f]{40}", image_revision) is None:
|
||||
raise SystemExit(51)
|
||||
resolved_image = git_run("rev-parse", f"{image_revision}^{{commit}}")
|
||||
mounted_recorded_to_image = git_run("merge-base", "--is-ancestor", recorded_live, image_revision)
|
||||
mounted_image_to_main = git_run("merge-base", "--is-ancestor", image_revision, main_revision)
|
||||
mounted_ancestry_proven = (
|
||||
resolved_image.returncode == 0
|
||||
and resolved_image.stdout.strip() == image_revision
|
||||
and mounted_recorded_to_image.returncode == 0
|
||||
and mounted_image_to_main.returncode == 0
|
||||
)
|
||||
if not mounted_ancestry_proven and not canonical_image_ancestry_is_proven(
|
||||
recorded_live, image_revision, main_revision
|
||||
):
|
||||
raise SystemExit(51)
|
||||
elif not main_declares_external_image(image_ref):
|
||||
raise SystemExit(51)
|
||||
|
||||
container_id = str(container.get("Id") or "")
|
||||
if not container_id:
|
||||
raise SystemExit(46)
|
||||
if image_id != recorded_image_id or container_id != recorded_container_id:
|
||||
drift_found = True
|
||||
|
||||
container_artifacts[name] = {
|
||||
"container_id": container_id,
|
||||
"image_id": image_id,
|
||||
"image_ref": image_ref,
|
||||
"container_revision": str(labels.get("org.opencontainers.image.revision") or ""),
|
||||
"image_revision": image_revision,
|
||||
"image_source": image_source,
|
||||
"image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""),
|
||||
"image_version": str(image_labels.get("org.opencontainers.image.version") or ""),
|
||||
"image_created": str(image_labels.get("org.opencontainers.image.created") or ""),
|
||||
}
|
||||
|
||||
if already_reconciled:
|
||||
if drift_found:
|
||||
raise SystemExit(52)
|
||||
print("already-reconciled", flush=True)
|
||||
raise SystemExit(0)
|
||||
if not drift_found:
|
||||
raise SystemExit(52)
|
||||
|
||||
artifacts = {"containers": container_artifacts, "compose_sha256": ""}
|
||||
now = int(time.time())
|
||||
payload = {
|
||||
"schema_version": 3,
|
||||
"repository": repository,
|
||||
"slug": expected_compose_project,
|
||||
"compose_project": expected_compose_project,
|
||||
"revision": recorded_live,
|
||||
"live_revision": recorded_live,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_artifacts": artifacts,
|
||||
"live_receipt": {},
|
||||
"rollback_snapshot": {},
|
||||
"status": "success",
|
||||
"message": "Verified mixed MCP Hub runtime reconciled as a legacy provenance baseline",
|
||||
"updated_at_epoch": now,
|
||||
"artifacts": artifacts,
|
||||
"last_attempt": {
|
||||
"revision": recorded_live,
|
||||
"status": "success",
|
||||
"message": "Verified MCP Hub runtime state reconciliation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {"reconciled_failed_revision": attempt_revision},
|
||||
},
|
||||
"previous_revision": recorded_live,
|
||||
"rollback_restored_revision": "",
|
||||
}
|
||||
|
||||
backup = state_path.with_name(f"{state_path.name}.mcp-hub-reconcile-backup-{now}-{os.getpid()}")
|
||||
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.mcp-hub-reconcile.tmp")
|
||||
try:
|
||||
mode = stat.S_IMODE(state_path.stat().st_mode)
|
||||
shutil.copy2(state_path, backup)
|
||||
with temporary.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, mode)
|
||||
os.replace(temporary, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(53)
|
||||
|
||||
try:
|
||||
verify = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(54)
|
||||
verified_containers = dict(dict(verify.get("live_artifacts") or {}).get("containers") or {})
|
||||
if not (
|
||||
verify.get("repository") == repository
|
||||
and verify.get("live_revision") == recorded_live
|
||||
and verify.get("live_status") == "healthy"
|
||||
and verify.get("live_legacy_provenance") is True
|
||||
and verify.get("status") == "success"
|
||||
and set(verified_containers) == set(expected_containers)
|
||||
and all(str(dict(verified_containers[name]).get("image_id") or "") == str(container_artifacts[name]["image_id"]) for name in expected_containers)
|
||||
):
|
||||
raise SystemExit(54)
|
||||
|
||||
print("reconciled", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class McpHubDeploymentReconciliation:
|
||||
"""Re-attest only the exact proven MCP Hub state drift, without touching runtime."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": redact_message(
|
||||
str(status.get("reason") or status.get("status") or "unavailable")
|
||||
),
|
||||
}
|
||||
controller = dict(status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": "deployment controller is not running",
|
||||
}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not container_id:
|
||||
return {
|
||||
"status": "connector_unavailable",
|
||||
"reason": "Portainer deployment control is unavailable",
|
||||
}
|
||||
|
||||
command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _COMPOSE_PROJECT]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": command,
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "MCP Hub state reconciliation could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
deadline = time.monotonic() + 60
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "MCP Hub state reconciliation timed out"}
|
||||
reasons = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
22: "MCP Hub deployment target is missing",
|
||||
23: "MCP Hub deployment target differs from the bounded production topology",
|
||||
24: "mounted MCP Hub bare repository is unavailable",
|
||||
25: "MCP Hub deployment revisions cannot be resolved",
|
||||
26: "MCP Hub main cannot be resolved",
|
||||
29: "deployment lock files cannot be opened",
|
||||
30: "deployment locks cannot be acquired",
|
||||
31: "deployment projects registry cannot be read",
|
||||
36: "deployment projects registry host alias is unavailable or mismatched",
|
||||
37: "MCP Hub deployment target changed while reconciliation was waiting",
|
||||
40: "fixed MCP Hub reconciliation contract was altered",
|
||||
42: "MCP Hub deployment state file is unavailable or invalid",
|
||||
43: "MCP Hub deployment state is not a modern controller state",
|
||||
44: "a live MCP Hub container cannot be inspected",
|
||||
45: "a live MCP Hub container is not running and healthy",
|
||||
46: "a live MCP Hub container has an invalid immutable image identity",
|
||||
47: "live MCP Hub Compose project differs from the controller target",
|
||||
50: "a live MCP Hub image cannot be inspected",
|
||||
51: "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain",
|
||||
52: "controller state drift is not the exact known recoverable MCP Hub state",
|
||||
53: "reconciled MCP Hub deployment state could not be written durably",
|
||||
54: "reconciled MCP Hub deployment state did not verify after write",
|
||||
55: "unknown MCP Hub live status was not produced by a proven recoverable snapshot-stage failure",
|
||||
}
|
||||
if exit_code != 0:
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": reasons.get(exit_code, "MCP Hub deployment reconciliation failed closed"),
|
||||
}
|
||||
return {
|
||||
"status": "reconciled",
|
||||
"repository": _REPOSITORY,
|
||||
"composeProject": _COMPOSE_PROJECT,
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..core.redaction import redact_message, redact_value
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting
|
||||
from .action_execution import (
|
||||
execute_approved_action,
|
||||
execute_governed_action,
|
||||
mcp_execution_supported,
|
||||
)
|
||||
from .action_registry import ACTION_RISK, ACTIONS, action_capability
|
||||
from .approval_service import (
|
||||
approval_fingerprint,
|
||||
get_approval,
|
||||
mark_approval_executed,
|
||||
)
|
||||
from .audit_logger import log_event
|
||||
from .job_runner import cancel_job, get_job
|
||||
from .policy_engine import evaluate_policy
|
||||
|
||||
_IDEMPOTENCY_LOCK = Lock()
|
||||
_IDEMPOTENCY_PREFIX = "mcp:operation:idempotency:"
|
||||
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
|
||||
|
||||
def _reconciliation_required_response(
|
||||
*,
|
||||
action: str,
|
||||
target: str,
|
||||
risk: str,
|
||||
original_policy_decision: str | None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"action": action,
|
||||
"target": target,
|
||||
"risk": risk,
|
||||
"policyDecision": "blocked",
|
||||
"originalPolicyDecision": original_policy_decision,
|
||||
"created": False,
|
||||
"executed": False,
|
||||
"executionSupported": mcp_execution_supported(action),
|
||||
"executionState": "reconciliation_required",
|
||||
"reason": (
|
||||
"A previous MCP attempt crossed the durable execution boundary but did not "
|
||||
"persist a terminal response. Re-execution is refused to prevent a duplicate mutation."
|
||||
),
|
||||
"nextStep": (
|
||||
"Inspect the target, audit trail and any job state to reconcile the prior attempt; "
|
||||
"do not retry this idempotency key blindly."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def request_operation(
|
||||
session: Session,
|
||||
*,
|
||||
action: str,
|
||||
target: str,
|
||||
reason: str,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Execute an MCP mutation immediately when central policy permits it.
|
||||
|
||||
MCP is intentionally a no-human-approval surface. A policy decision of
|
||||
``approval_required`` is accepted only when AppOps has a typed executor for the
|
||||
exact action; it is then executed directly through that executor and audited with
|
||||
the original decision preserved as metadata. ``blocked`` actions stay blocked.
|
||||
Actions without a typed executor fail closed instead of creating an Approval row.
|
||||
|
||||
A durable idempotency reservation is committed before any side effect. Once the
|
||||
reservation enters ``executing``, a replay without a terminal response is refused
|
||||
as ``reconciliation_required`` rather than risking a second mutation.
|
||||
"""
|
||||
|
||||
safe_action = action.strip()[:160]
|
||||
safe_target = target.strip()[:240]
|
||||
safe_reason = redact_message(reason.strip())[:1000]
|
||||
if safe_action not in ACTIONS:
|
||||
raise ValueError("unknown_action")
|
||||
if not safe_target:
|
||||
raise ValueError("target_required")
|
||||
if not safe_reason:
|
||||
raise ValueError("reason_required")
|
||||
if not _IDEMPOTENCY_KEY.fullmatch(idempotency_key):
|
||||
raise ValueError("invalid_idempotency_key")
|
||||
|
||||
capability = action_capability(safe_action)
|
||||
policy = evaluate_policy(
|
||||
session,
|
||||
action=safe_action,
|
||||
target=safe_target,
|
||||
source="automation",
|
||||
)
|
||||
risk = str(policy.get("risk") or ACTION_RISK.get(safe_action, "critical"))
|
||||
if capability == "read_only":
|
||||
return {
|
||||
"action": safe_action,
|
||||
"target": safe_target,
|
||||
"risk": risk,
|
||||
"policyDecision": policy["decision"],
|
||||
"created": False,
|
||||
"executed": False,
|
||||
"executionSupported": False,
|
||||
"nextStep": "Use the corresponding read-only MCP tool.",
|
||||
}
|
||||
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"action": safe_action,
|
||||
"target": safe_target,
|
||||
"reason": safe_reason,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
row_key = (
|
||||
f"{_IDEMPOTENCY_PREFIX}{hashlib.sha256(idempotency_key.encode()).hexdigest()}"
|
||||
)
|
||||
|
||||
with _IDEMPOTENCY_LOCK:
|
||||
row = session.get(AppSetting, row_key)
|
||||
if row:
|
||||
if row.value_json.get("requestDigest") != digest:
|
||||
raise ValueError("idempotency_key_reused_with_different_request")
|
||||
if row.value_json.get("response") is not None:
|
||||
return {**row.value_json.get("response", {}), "idempotentReplay": True}
|
||||
if row.value_json.get("state") == "executing":
|
||||
response = _reconciliation_required_response(
|
||||
action=safe_action,
|
||||
target=safe_target,
|
||||
risk=risk,
|
||||
original_policy_decision=row.value_json.get(
|
||||
"originalPolicyDecision"
|
||||
),
|
||||
)
|
||||
return {**response, "idempotentReplay": True}
|
||||
else:
|
||||
row = AppSetting(
|
||||
key=row_key,
|
||||
value_json={
|
||||
"requestDigest": digest,
|
||||
"state": "reserved",
|
||||
"reservedAt": utc_now().isoformat(),
|
||||
"action": safe_action,
|
||||
"target": safe_target,
|
||||
},
|
||||
is_secret=False,
|
||||
)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
if policy["decision"] == "blocked":
|
||||
audit = log_event(
|
||||
session,
|
||||
source="automation",
|
||||
identity="chatgpt-mcp",
|
||||
action_id=safe_action,
|
||||
target=safe_target,
|
||||
risk=risk,
|
||||
policy_decision="blocked",
|
||||
reason=policy["reason"],
|
||||
result="blocked",
|
||||
redacted_metadata={"integration": "mcp", "request": "operation"},
|
||||
)
|
||||
response = {
|
||||
"action": safe_action,
|
||||
"target": safe_target,
|
||||
"risk": risk,
|
||||
"policyDecision": "blocked",
|
||||
"reason": policy["reason"],
|
||||
"created": False,
|
||||
"executed": False,
|
||||
"executionSupported": False,
|
||||
"auditId": audit.id,
|
||||
"nextStep": "Blocked by AppOps policy. MCP does not create approvals.",
|
||||
}
|
||||
elif not mcp_execution_supported(safe_action):
|
||||
reason_text = (
|
||||
"MCP no-human-approval mode has no typed executor for this action; "
|
||||
"the operation is blocked instead of creating an approval request."
|
||||
)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="automation",
|
||||
identity="chatgpt-mcp",
|
||||
action_id=safe_action,
|
||||
target=safe_target,
|
||||
risk=risk,
|
||||
policy_decision="blocked",
|
||||
reason=reason_text,
|
||||
result="blocked",
|
||||
redacted_metadata={
|
||||
"integration": "mcp",
|
||||
"request": "operation",
|
||||
"originalPolicyDecision": policy["decision"],
|
||||
},
|
||||
)
|
||||
response = {
|
||||
"action": safe_action,
|
||||
"target": safe_target,
|
||||
"risk": risk,
|
||||
"policyDecision": "blocked",
|
||||
"originalPolicyDecision": policy["decision"],
|
||||
"reason": reason_text,
|
||||
"created": False,
|
||||
"executed": False,
|
||||
"executionSupported": False,
|
||||
"auditId": audit.id,
|
||||
"nextStep": "No MCP approval will be requested; add a typed executor if this action should become autonomous.",
|
||||
}
|
||||
else:
|
||||
original_decision = str(policy["decision"])
|
||||
authorization_reason = safe_reason
|
||||
if original_decision == "approval_required":
|
||||
authorization_reason = redact_message(
|
||||
f"{safe_reason} MCP autonomous execution accepted the policy's approval-required posture without a human approval step: {policy['reason']}"
|
||||
)[:1000]
|
||||
|
||||
# This commit is the side-effect boundary. If the process dies after this
|
||||
# point and before a terminal response is stored, a replay sees
|
||||
# state=executing and refuses to mutate again.
|
||||
row.value_json = {
|
||||
**row.value_json,
|
||||
"state": "executing",
|
||||
"executingAt": utc_now().isoformat(),
|
||||
"originalPolicyDecision": original_decision,
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
result = execute_governed_action(
|
||||
session,
|
||||
action=safe_action,
|
||||
target=safe_target,
|
||||
source="chatgpt-mcp",
|
||||
risk=risk,
|
||||
reason=authorization_reason,
|
||||
original_policy_decision=original_decision,
|
||||
)
|
||||
response = {
|
||||
**result,
|
||||
"action": safe_action,
|
||||
"target": safe_target,
|
||||
"risk": risk,
|
||||
"policyDecision": str(result.get("policyDecision") or "allowed"),
|
||||
"originalPolicyDecision": original_decision,
|
||||
"created": False,
|
||||
"executed": True,
|
||||
"executionSupported": True,
|
||||
"nextStep": (
|
||||
"Use server.job.status for the returned job if further progress inspection is needed."
|
||||
if result.get("jobId")
|
||||
else "Operation completed through the typed MCP executor."
|
||||
),
|
||||
}
|
||||
|
||||
row.value_json = {
|
||||
**row.value_json,
|
||||
"state": "completed",
|
||||
"completedAt": utc_now().isoformat(),
|
||||
"response": response,
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return {**response, "idempotentReplay": False}
|
||||
|
||||
|
||||
def operation_status(session: Session, approval_id: str) -> dict[str, Any]:
|
||||
"""Legacy status reader for approval records created before MCP no-approval mode."""
|
||||
|
||||
approval = get_approval(session, approval_id.strip())
|
||||
if not approval:
|
||||
raise ValueError("approval_not_found")
|
||||
return redact_value(
|
||||
{
|
||||
"approvalId": approval.id,
|
||||
"action": approval.action,
|
||||
"target": approval.target,
|
||||
"risk": approval.risk,
|
||||
"status": approval.status,
|
||||
"actionFingerprint": approval.action_fingerprint,
|
||||
"expiresAt": approval.expires_at.isoformat()
|
||||
if approval.expires_at
|
||||
else None,
|
||||
"executionSupported": mcp_execution_supported(approval.action),
|
||||
"jobId": approval.job_id,
|
||||
"auditId": approval.audit_event_id,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def execute_operation(
|
||||
session: Session,
|
||||
*,
|
||||
approval_id: str,
|
||||
expected_action_fingerprint: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Legacy executor for approval records created before MCP no-approval mode."""
|
||||
|
||||
approval = get_approval(session, approval_id.strip())
|
||||
if not approval:
|
||||
raise ValueError("approval_not_found")
|
||||
if approval.status == "executed":
|
||||
return operation_status(session, approval.id)
|
||||
if approval.status != "approved":
|
||||
raise ValueError("approval_not_approved")
|
||||
if approval.expires_at is not None and approval.expires_at <= utc_now():
|
||||
raise ValueError("approval_expired")
|
||||
expected = approval_fingerprint(
|
||||
approval.action,
|
||||
approval.target,
|
||||
approval.source,
|
||||
approval.risk,
|
||||
approval.request_context_json,
|
||||
)
|
||||
supplied = expected_action_fingerprint.strip()
|
||||
if not supplied or supplied != expected or approval.action_fingerprint != expected:
|
||||
raise ValueError("approval_context_mismatch")
|
||||
if not mcp_execution_supported(approval.action):
|
||||
raise ValueError("operation_executor_unavailable")
|
||||
|
||||
result = execute_approved_action(session, approval)
|
||||
if result.get("success"):
|
||||
mark_approval_executed(
|
||||
session,
|
||||
approval,
|
||||
job_id=str(result["jobId"]),
|
||||
audit_event_id=str(result["auditId"]),
|
||||
)
|
||||
return redact_value({**result, "approvalId": approval.id})
|
||||
|
||||
|
||||
def job_status(session: Session, job_id: str) -> dict[str, Any]:
|
||||
job = get_job(session, job_id.strip())
|
||||
if not job:
|
||||
raise ValueError("job_not_found")
|
||||
return redact_value(job.model_dump(mode="json"))
|
||||
|
||||
|
||||
def cancel_operation_job(session: Session, job_id: str) -> dict[str, Any]:
|
||||
job = get_job(session, job_id.strip())
|
||||
if not job:
|
||||
raise ValueError("job_not_found")
|
||||
from .privatemuse_deployment_recovery import (
|
||||
privatemuse_recovery_job_cancellation_reason,
|
||||
)
|
||||
|
||||
cancellation_reason = privatemuse_recovery_job_cancellation_reason(session, job)
|
||||
if not cancellation_reason:
|
||||
from .workmesh_first_deployment import (
|
||||
workmesh_first_deploy_job_cancellation_reason,
|
||||
)
|
||||
|
||||
cancellation_reason = workmesh_first_deploy_job_cancellation_reason(session, job)
|
||||
if cancellation_reason:
|
||||
return {
|
||||
"jobId": job.id,
|
||||
"status": job.status,
|
||||
"cancelled": False,
|
||||
"changed": False,
|
||||
"reason": cancellation_reason,
|
||||
}
|
||||
if job.status in {"succeeded", "failed", "cancelled"}:
|
||||
return {
|
||||
"jobId": job.id,
|
||||
"status": job.status,
|
||||
"cancelled": job.status == "cancelled",
|
||||
"changed": False,
|
||||
}
|
||||
previous = job.status
|
||||
updated = cancel_job(session, job, result_summary="Cancelled through governed MCP.")
|
||||
audit = log_event(
|
||||
session,
|
||||
source="automation",
|
||||
identity="chatgpt-mcp",
|
||||
action_id="job.cancel",
|
||||
target=updated.id,
|
||||
target_type="job",
|
||||
risk="low",
|
||||
policy_decision="allowed",
|
||||
reason="A governed MCP client cancelled a queued or running AppOps job.",
|
||||
result="success",
|
||||
before_state={"status": previous},
|
||||
after_state={"status": updated.status},
|
||||
redacted_metadata={"jobId": updated.id, "actionId": updated.action_id},
|
||||
)
|
||||
return {
|
||||
"jobId": updated.id,
|
||||
"status": updated.status,
|
||||
"cancelled": True,
|
||||
"changed": True,
|
||||
"auditId": audit.id,
|
||||
}
|
||||
@@ -0,0 +1,768 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/media-codec-dashboard"
|
||||
_CONTAINER = "Media-Codec-Dashboard"
|
||||
|
||||
# Fail-closed reconciliation for the verified production runtime.
|
||||
# It does not deploy, restart, recreate or stop any container. It only repairs
|
||||
# controller state after proving the exact live container/image/mount topology
|
||||
# and immutable runtime revision from independent OCI labels plus Gitea ancestry.
|
||||
_RECONCILE_SCRIPT = r'''
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
repository, expected_container = sys.argv[1:3]
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
state_path = state_root / "media-codec-dashboard.json"
|
||||
project_lock_path = state_root / "locks" / "media-codec-dashboard.lock"
|
||||
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
||||
|
||||
if repository != "jens/media-codec-dashboard":
|
||||
raise SystemExit(40)
|
||||
if expected_container != "Media-Codec-Dashboard":
|
||||
raise SystemExit(40)
|
||||
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(20)
|
||||
try:
|
||||
if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path):
|
||||
raise SystemExit(36)
|
||||
except OSError:
|
||||
raise SystemExit(36)
|
||||
|
||||
try:
|
||||
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
if not isinstance(project, dict):
|
||||
raise SystemExit(22)
|
||||
if project.get("branch") != "main" or project.get("slug") != "media-codec-dashboard":
|
||||
raise SystemExit(23)
|
||||
if project.get("compose_project") != "media-codec-dashboard" or project.get("containers") != [expected_container]:
|
||||
raise SystemExit(23)
|
||||
compose_files = project.get("compose_files")
|
||||
if not isinstance(compose_files, list) or "docker-compose.yml" not in compose_files:
|
||||
raise SystemExit(23)
|
||||
source_dir = Path(str(project.get("source_dir") or ""))
|
||||
allowed_roots = (Path("/mnt/cache/appdata/gitea-deploy-runners/sources"), Path("/srv/example/appdata"))
|
||||
if not source_dir.is_absolute() or not any(root == source_dir or root in source_dir.parents for root in allowed_roots):
|
||||
raise SystemExit(23)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")
|
||||
canonical_bare_repo = canonical_repository_root / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(24)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
|
||||
def canonical_git_run(*args):
|
||||
return subprocess.run(["git", f"--git-dir={canonical_bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
|
||||
def _is_current_verified_receipt_snapshot_baseline(state, repository, expected_container, live_revision, attempt_revision):
|
||||
attempt = state.get("last_attempt")
|
||||
artifacts = state.get("live_artifacts")
|
||||
receipt = state.get("live_receipt")
|
||||
if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict):
|
||||
return False
|
||||
containers = artifacts.get("containers")
|
||||
receipt_containers = receipt.get("containers")
|
||||
return (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == repository
|
||||
and state.get("revision") == live_revision
|
||||
and state.get("live_revision") == live_revision
|
||||
and state.get("status") == "failed"
|
||||
and state.get("live_status") == "unknown"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
and attempt.get("revision") == attempt_revision
|
||||
and attempt.get("status") == "failed"
|
||||
and attempt.get("message") == "Existing production containers require a valid healthy live deployment state"
|
||||
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None
|
||||
and isinstance(containers, dict)
|
||||
and set(containers) == {expected_container}
|
||||
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
||||
and receipt.get("repository") == repository
|
||||
and receipt.get("revision") == live_revision
|
||||
and receipt.get("live") is None
|
||||
and isinstance(receipt_containers, dict)
|
||||
and set(receipt_containers) == {expected_container}
|
||||
and receipt_containers == containers
|
||||
)
|
||||
|
||||
def _is_exact_legacy_state_ahead_baseline(
|
||||
state,
|
||||
repository,
|
||||
expected_container,
|
||||
recorded_live,
|
||||
live_revision,
|
||||
container_artifact,
|
||||
git_facts,
|
||||
):
|
||||
attempt = state.get("last_attempt")
|
||||
artifacts = state.get("live_artifacts")
|
||||
receipt = state.get("live_receipt")
|
||||
if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict):
|
||||
return False
|
||||
metadata = attempt.get("metadata")
|
||||
build = receipt.get("build")
|
||||
if not isinstance(metadata, dict) or not isinstance(build, dict):
|
||||
return False
|
||||
artifact_containers = artifacts.get("containers")
|
||||
receipt_containers = receipt.get("containers")
|
||||
if not isinstance(artifact_containers, dict) or not isinstance(receipt_containers, dict):
|
||||
return False
|
||||
stored_artifact = artifact_containers.get(expected_container)
|
||||
if not isinstance(stored_artifact, dict) or receipt_containers != artifact_containers:
|
||||
return False
|
||||
|
||||
def _same_immutable_artifact_after_container_recreate(stored, current):
|
||||
expected_fields = {
|
||||
"container_id",
|
||||
"image_id",
|
||||
"image_ref",
|
||||
"container_revision",
|
||||
"image_revision",
|
||||
"image_source",
|
||||
"image_tree",
|
||||
"image_version",
|
||||
"image_created",
|
||||
}
|
||||
immutable_fields = expected_fields - {"container_id"}
|
||||
return (
|
||||
isinstance(current, dict)
|
||||
and set(stored) == expected_fields
|
||||
and set(current) == expected_fields
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(stored.get("container_id") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(current.get("container_id") or "")) is not None
|
||||
and all(stored.get(field) == current.get(field) for field in immutable_fields)
|
||||
)
|
||||
|
||||
attempt_revision = str(attempt.get("revision") or "")
|
||||
expected_facts = {
|
||||
"recorded_to_live": 1,
|
||||
"canonical_recorded_to_live": 1,
|
||||
"live_to_recorded": 0,
|
||||
"canonical_live_to_recorded": 0,
|
||||
"mounted_recorded_main": 0,
|
||||
"canonical_recorded_main": 0,
|
||||
"canonical_live_main": 0,
|
||||
"recorded_to_attempt": 0,
|
||||
"canonical_recorded_to_attempt": 0,
|
||||
"attempt_to_main": 0,
|
||||
"canonical_attempt_to_main": 0,
|
||||
}
|
||||
return (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == repository
|
||||
and state.get("revision") == recorded_live
|
||||
and state.get("live_revision") == recorded_live
|
||||
and state.get("status") == "failed"
|
||||
and state.get("live_status") == "unknown"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{40}", live_revision) is not None
|
||||
and recorded_live != live_revision
|
||||
and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None
|
||||
and attempt_revision != recorded_live
|
||||
and attempt.get("status") == "failed"
|
||||
and attempt.get("message") == "Existing production containers require a valid healthy live deployment state"
|
||||
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
||||
and set(metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
||||
and metadata.get("id") == attempt_revision[:12]
|
||||
and metadata.get("repository") == repository
|
||||
and metadata.get("revision") == attempt_revision
|
||||
and re.fullmatch(r"[0-9a-f]{40}", str(metadata.get("tree") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("treeDigest") or "")) is not None
|
||||
and isinstance(metadata.get("builtAt"), str)
|
||||
and bool(metadata.get("builtAt"))
|
||||
and set(artifacts) == {"compose_sha256", "containers"}
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None
|
||||
and set(artifact_containers) == {expected_container}
|
||||
and _same_immutable_artifact_after_container_recreate(stored_artifact, container_artifact)
|
||||
and set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"}
|
||||
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
||||
and receipt.get("repository") == repository
|
||||
and receipt.get("revision") == recorded_live
|
||||
and receipt.get("live") is None
|
||||
and re.fullmatch(r"[0-9a-f]{40}", str(receipt.get("tree") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("treeDigest") or "")) is not None
|
||||
and set(receipt_containers) == {expected_container}
|
||||
and set(build) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
||||
and build.get("id") == recorded_live[:12]
|
||||
and build.get("repository") == repository
|
||||
and build.get("revision") == recorded_live
|
||||
and build.get("tree") == receipt.get("tree")
|
||||
and build.get("treeDigest") == receipt.get("treeDigest")
|
||||
and isinstance(build.get("builtAt"), str)
|
||||
and bool(build.get("builtAt"))
|
||||
and container_artifact.get("container_revision") == live_revision
|
||||
and container_artifact.get("image_revision") == live_revision
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None
|
||||
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(container_artifact.get("image_id") or "")) is not None
|
||||
and git_facts == expected_facts
|
||||
)
|
||||
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
main_revision = main.stdout.strip()
|
||||
if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None:
|
||||
raise SystemExit(26)
|
||||
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
||||
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
with project_lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
with global_lock:
|
||||
try:
|
||||
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
try:
|
||||
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
if current_projects.get(repository) != project:
|
||||
raise SystemExit(37)
|
||||
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(42)
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(42)
|
||||
if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository:
|
||||
raise SystemExit(43)
|
||||
|
||||
inspected = subprocess.run(["docker", "inspect", expected_container], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(44)
|
||||
try:
|
||||
container = json.loads(inspected.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(44)
|
||||
runtime_state = dict(container.get("State") or {})
|
||||
labels = dict(dict(container.get("Config") or {}).get("Labels") or {})
|
||||
image_ref = str(dict(container.get("Config") or {}).get("Image") or "")
|
||||
image_id = str(container.get("Image") or "")
|
||||
container_revision = str(labels.get("org.opencontainers.image.revision") or "")
|
||||
if not runtime_state.get("Running") or str(dict(runtime_state.get("Health") or {}).get("Status") or "") != "healthy":
|
||||
raise SystemExit(45)
|
||||
if labels.get("com.docker.compose.project") != "media-codec-dashboard":
|
||||
raise SystemExit(45)
|
||||
if not image_id.startswith("sha256:") or re.fullmatch(r"[0-9a-f]{40}", container_revision) is None:
|
||||
raise SystemExit(46)
|
||||
|
||||
project_names = subprocess.run(["docker", "ps", "--all", "--filter", "label=com.docker.compose.project=media-codec-dashboard", "--format", "{{.Names}}"], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if project_names.returncode != 0 or {line.strip() for line in project_names.stdout.splitlines() if line.strip()} != {expected_container}:
|
||||
raise SystemExit(47)
|
||||
|
||||
mounts = {str(item.get("Destination") or ""): str(item.get("Source") or "") for item in list(container.get("Mounts") or []) if isinstance(item, dict)}
|
||||
required_mounts = {
|
||||
"/app/data": "/srv/example/appdata/media-codec-dashboard/data",
|
||||
"/app/data/quality-runtime": "/srv/example/appdata/tdarr/logs/media-codec-dashboard-quality-audit",
|
||||
"/run/secrets": "/srv/example/appdata/media-codec-dashboard/secrets",
|
||||
"/tdarr-output": "/srv/example/Media",
|
||||
}
|
||||
if any(mounts.get(destination) != source for destination, source in required_mounts.items()):
|
||||
raise SystemExit(48)
|
||||
|
||||
bindings = dict(dict(container.get("HostConfig") or {}).get("PortBindings") or {})
|
||||
host_ports = {str(item.get("HostPort") or "") for item in list(bindings.get("1211/tcp") or []) if isinstance(item, dict)}
|
||||
if "1211" not in host_ports:
|
||||
raise SystemExit(49)
|
||||
|
||||
image_result = subprocess.run(["docker", "image", "inspect", image_id], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if image_result.returncode != 0:
|
||||
raise SystemExit(50)
|
||||
try:
|
||||
image = json.loads(image_result.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(50)
|
||||
image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {})
|
||||
image_revision = str(image_labels.get("org.opencontainers.image.revision") or "")
|
||||
live_revision = image_revision
|
||||
if re.fullmatch(r"[0-9a-f]{40}", live_revision) is None or container_revision != image_revision:
|
||||
raise SystemExit(51)
|
||||
container_artifact = {
|
||||
"container_id": str(container.get("Id") or ""),
|
||||
"image_id": image_id,
|
||||
"image_ref": image_ref,
|
||||
"container_revision": container_revision,
|
||||
"image_revision": image_revision,
|
||||
"image_source": str(image_labels.get("org.opencontainers.image.source") or ""),
|
||||
"image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""),
|
||||
"image_version": str(image_labels.get("org.opencontainers.image.version") or ""),
|
||||
"image_created": str(image_labels.get("org.opencontainers.image.created") or ""),
|
||||
}
|
||||
resolved = git_run("rev-parse", f"{live_revision}^{{commit}}")
|
||||
if resolved.returncode != 0 or resolved.stdout.strip() != live_revision:
|
||||
raise SystemExit(25)
|
||||
if git_run("merge-base", "--is-ancestor", live_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(27)
|
||||
|
||||
recorded_live = str(state.get("live_revision") or state.get("revision") or "")
|
||||
recorded_status = str(state.get("status") or "")
|
||||
recorded_live_status = str(state.get("live_status") or "")
|
||||
last_attempt = dict(state.get("last_attempt") or {})
|
||||
attempt_revision = str(last_attempt.get("revision") or "")
|
||||
attempt_status = str(last_attempt.get("status") or "")
|
||||
reconciled_from = recorded_live
|
||||
if recorded_live == live_revision:
|
||||
if recorded_status == "success" and state.get("live_legacy_provenance") is True:
|
||||
print("already-reconciled", flush=True)
|
||||
raise SystemExit(0)
|
||||
|
||||
current_artifacts = state.get("live_artifacts")
|
||||
current_receipt = state.get("live_receipt")
|
||||
current_artifact_containers = (
|
||||
current_artifacts.get("containers")
|
||||
if isinstance(current_artifacts, dict)
|
||||
else None
|
||||
)
|
||||
current_stored_artifact = (
|
||||
current_artifact_containers.get(expected_container)
|
||||
if isinstance(current_artifact_containers, dict)
|
||||
else None
|
||||
)
|
||||
current_attempt_metadata = (
|
||||
last_attempt.get("metadata")
|
||||
if isinstance(last_attempt, dict)
|
||||
else None
|
||||
)
|
||||
current_artifact_fields = {
|
||||
"container_id",
|
||||
"image_id",
|
||||
"image_ref",
|
||||
"container_revision",
|
||||
"image_revision",
|
||||
"image_source",
|
||||
"image_tree",
|
||||
"image_version",
|
||||
"image_created",
|
||||
}
|
||||
current_immutable_fields = current_artifact_fields - {"container_id"}
|
||||
current_live_unknown_snapshot_baseline = (
|
||||
recorded_status == "failed"
|
||||
and recorded_live_status == "unknown"
|
||||
and state.get("live_legacy_provenance") is True
|
||||
and isinstance(current_artifacts, dict)
|
||||
and set(current_artifacts) == {"compose_sha256", "containers"}
|
||||
and current_artifacts.get("compose_sha256") == ""
|
||||
and isinstance(current_artifact_containers, dict)
|
||||
and set(current_artifact_containers) == {expected_container}
|
||||
and isinstance(current_stored_artifact, dict)
|
||||
and set(current_stored_artifact) == current_artifact_fields
|
||||
and set(container_artifact) == current_artifact_fields
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(current_stored_artifact.get("container_id") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None
|
||||
and all(current_stored_artifact.get(field) == container_artifact.get(field) for field in current_immutable_fields)
|
||||
and isinstance(current_receipt, dict)
|
||||
and current_receipt == {}
|
||||
and attempt_status == "failed"
|
||||
# Do not bind recovery to one historical error string. The proof
|
||||
# is the preserved immutable artifact plus the currently healthy
|
||||
# runtime, exact candidate metadata/timestamp, and Git ancestry.
|
||||
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
||||
and isinstance(current_attempt_metadata, dict)
|
||||
and set(current_attempt_metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
||||
and current_attempt_metadata.get("id") == attempt_revision[:12]
|
||||
and current_attempt_metadata.get("repository") == repository
|
||||
and current_attempt_metadata.get("revision") == attempt_revision
|
||||
and re.fullmatch(r"[0-9a-f]{40}", str(current_attempt_metadata.get("tree") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(current_attempt_metadata.get("treeDigest") or "")) is not None
|
||||
and isinstance(current_attempt_metadata.get("builtAt"), str)
|
||||
and bool(current_attempt_metadata.get("builtAt"))
|
||||
)
|
||||
current_verified_receipt_snapshot_baseline = _is_current_verified_receipt_snapshot_baseline(
|
||||
state, repository, expected_container, live_revision, attempt_revision
|
||||
)
|
||||
current_live_failed_baseline = (
|
||||
recorded_status == "failed"
|
||||
and (recorded_live_status == "healthy" or current_live_unknown_snapshot_baseline or current_verified_receipt_snapshot_baseline)
|
||||
and attempt_status == "failed"
|
||||
and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None
|
||||
and attempt_revision != live_revision
|
||||
)
|
||||
if not current_live_failed_baseline:
|
||||
raise SystemExit(55)
|
||||
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision:
|
||||
raise SystemExit(25)
|
||||
if git_run("merge-base", "--is-ancestor", live_revision, attempt_revision).returncode != 0:
|
||||
raise SystemExit(56)
|
||||
if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(57)
|
||||
if current_verified_receipt_snapshot_baseline:
|
||||
if controller_path.is_symlink() or not controller_path.is_file():
|
||||
raise SystemExit(62)
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("dashboard_deploy_controller", controller_path)
|
||||
controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None
|
||||
if controller is None or spec is None or spec.loader is None:
|
||||
raise RuntimeError("controller import unavailable")
|
||||
spec.loader.exec_module(controller)
|
||||
verified_project = dict(project)
|
||||
verified_project["_repository_name"] = repository
|
||||
verified_project["_repository_path"] = str(bare_repo)
|
||||
build = controller.validate_recorded_receipt(
|
||||
verified_project, current_receipt, repository, live_revision
|
||||
)
|
||||
observed_receipt = controller.verify_deployment(
|
||||
verified_project,
|
||||
build,
|
||||
expected_image_ids={
|
||||
expected_container: current_receipt["containers"][expected_container]["image_id"]
|
||||
},
|
||||
)
|
||||
observed_artifacts = controller.deployment_artifacts(verified_project)
|
||||
except Exception:
|
||||
raise SystemExit(63)
|
||||
if observed_receipt != current_receipt or observed_artifacts != current_artifacts:
|
||||
raise SystemExit(64)
|
||||
reconciled_from = attempt_revision
|
||||
else:
|
||||
if re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None:
|
||||
raise SystemExit(58)
|
||||
stale_success_baseline = (
|
||||
recorded_status == "success" and recorded_live_status == "healthy"
|
||||
and attempt_status == "success" and attempt_revision == recorded_live
|
||||
)
|
||||
stale_failed_attempt = recorded_status == "failed"
|
||||
if not (stale_success_baseline or stale_failed_attempt):
|
||||
raise SystemExit(59)
|
||||
resolved_recorded = git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
||||
if resolved_recorded.returncode != 0 or resolved_recorded.stdout.strip() != recorded_live:
|
||||
raise SystemExit(25)
|
||||
recorded_to_live = git_run("merge-base", "--is-ancestor", recorded_live, live_revision)
|
||||
if recorded_to_live.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
if recorded_to_live.returncode != 0:
|
||||
canonical_recorded = canonical_git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
||||
canonical_live = canonical_git_run("rev-parse", f"{live_revision}^{{commit}}")
|
||||
canonical_main = canonical_git_run("rev-parse", "refs/heads/main")
|
||||
canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)
|
||||
canonical_recorded_main = canonical_git_run("merge-base", "--is-ancestor", recorded_live, main_revision)
|
||||
canonical_live_main = canonical_git_run("merge-base", "--is-ancestor", live_revision, main_revision)
|
||||
canonical_revisions_available = (
|
||||
canonical_bare_repo.is_dir()
|
||||
and not canonical_bare_repo.is_symlink()
|
||||
and canonical_recorded.returncode == 0
|
||||
and canonical_recorded.stdout.strip() == recorded_live
|
||||
and canonical_live.returncode == 0
|
||||
and canonical_live.stdout.strip() == live_revision
|
||||
and canonical_main.returncode == 0
|
||||
and canonical_main.stdout.strip() == main_revision
|
||||
)
|
||||
canonical_ancestry_fallback = (
|
||||
canonical_revisions_available
|
||||
and canonical_recorded_to_live.returncode == 0
|
||||
and canonical_recorded_main.returncode == 0
|
||||
and canonical_live_main.returncode == 0
|
||||
)
|
||||
if not canonical_ancestry_fallback:
|
||||
if not canonical_revisions_available:
|
||||
raise SystemExit(61)
|
||||
if canonical_recorded_to_live.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
if canonical_recorded_main.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
if canonical_live_main.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
|
||||
live_to_recorded = git_run("merge-base", "--is-ancestor", live_revision, recorded_live)
|
||||
canonical_live_to_recorded = canonical_git_run("merge-base", "--is-ancestor", live_revision, recorded_live)
|
||||
mounted_recorded_main = git_run("merge-base", "--is-ancestor", recorded_live, main_revision)
|
||||
mounted_merge_base = git_run("merge-base", recorded_live, live_revision)
|
||||
canonical_merge_base = canonical_git_run("merge-base", recorded_live, live_revision)
|
||||
mounted_recorded_time = git_run("show", "-s", "--format=%ct", recorded_live)
|
||||
mounted_live_time = git_run("show", "-s", "--format=%ct", live_revision)
|
||||
canonical_recorded_time = canonical_git_run("show", "-s", "--format=%ct", recorded_live)
|
||||
canonical_live_time = canonical_git_run("show", "-s", "--format=%ct", live_revision)
|
||||
mounted_recorded_distance = git_run("rev-list", "--count", f"{recorded_live}..{main_revision}")
|
||||
mounted_live_distance = git_run("rev-list", "--count", f"{live_revision}..{main_revision}")
|
||||
canonical_recorded_distance = canonical_git_run("rev-list", "--count", f"{recorded_live}..{main_revision}")
|
||||
canonical_live_distance = canonical_git_run("rev-list", "--count", f"{live_revision}..{main_revision}")
|
||||
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
canonical_resolved_attempt = canonical_git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
recorded_to_attempt = git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)
|
||||
canonical_recorded_to_attempt = canonical_git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)
|
||||
attempt_to_main = git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)
|
||||
canonical_attempt_to_main = canonical_git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)
|
||||
|
||||
def git_nonnegative_int(result):
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
value = result.stdout.strip()
|
||||
if not value.isdigit():
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
recorded_commit_epoch = git_nonnegative_int(mounted_recorded_time)
|
||||
live_commit_epoch = git_nonnegative_int(mounted_live_time)
|
||||
canonical_recorded_commit_epoch = git_nonnegative_int(canonical_recorded_time)
|
||||
canonical_live_commit_epoch = git_nonnegative_int(canonical_live_time)
|
||||
recorded_distance = git_nonnegative_int(mounted_recorded_distance)
|
||||
live_distance = git_nonnegative_int(mounted_live_distance)
|
||||
canonical_recorded_distance_value = git_nonnegative_int(canonical_recorded_distance)
|
||||
canonical_live_distance_value = git_nonnegative_int(canonical_live_distance)
|
||||
try:
|
||||
recorded_updated_at = int(state.get("updated_at_epoch") or 0)
|
||||
except (TypeError, ValueError):
|
||||
recorded_updated_at = 0
|
||||
|
||||
mounted_merge_base_value = mounted_merge_base.stdout.strip()
|
||||
canonical_merge_base_value = canonical_merge_base.stdout.strip()
|
||||
history_transition_baseline = (
|
||||
stale_success_baseline
|
||||
and recorded_to_live.returncode == 1
|
||||
and canonical_recorded_to_live.returncode == 1
|
||||
and live_to_recorded.returncode == 1
|
||||
and canonical_live_to_recorded.returncode == 1
|
||||
and mounted_recorded_main.returncode == 0
|
||||
and canonical_recorded_main.returncode == 0
|
||||
and canonical_live_main.returncode == 0
|
||||
and mounted_merge_base.returncode == 0
|
||||
and canonical_merge_base.returncode == 0
|
||||
and re.fullmatch(r"[0-9a-f]{40}", mounted_merge_base_value) is not None
|
||||
and canonical_merge_base_value == mounted_merge_base_value
|
||||
and mounted_merge_base_value not in {recorded_live, live_revision}
|
||||
and recorded_commit_epoch is not None
|
||||
and live_commit_epoch is not None
|
||||
and canonical_recorded_commit_epoch == recorded_commit_epoch
|
||||
and canonical_live_commit_epoch == live_commit_epoch
|
||||
and 0 < recorded_updated_at
|
||||
and recorded_commit_epoch <= recorded_updated_at
|
||||
and recorded_updated_at < live_commit_epoch
|
||||
and recorded_commit_epoch < live_commit_epoch
|
||||
and recorded_distance is not None
|
||||
and live_distance is not None
|
||||
and canonical_recorded_distance_value == recorded_distance
|
||||
and canonical_live_distance_value == live_distance
|
||||
and 0 <= live_distance < recorded_distance
|
||||
)
|
||||
exact_legacy_state_ahead_baseline = (
|
||||
resolved_attempt.returncode == 0
|
||||
and resolved_attempt.stdout.strip() == attempt_revision
|
||||
and canonical_resolved_attempt.returncode == 0
|
||||
and canonical_resolved_attempt.stdout.strip() == attempt_revision
|
||||
and _is_exact_legacy_state_ahead_baseline(
|
||||
state,
|
||||
repository,
|
||||
expected_container,
|
||||
recorded_live,
|
||||
live_revision,
|
||||
container_artifact,
|
||||
{
|
||||
"recorded_to_live": recorded_to_live.returncode,
|
||||
"canonical_recorded_to_live": canonical_recorded_to_live.returncode,
|
||||
"live_to_recorded": live_to_recorded.returncode,
|
||||
"canonical_live_to_recorded": canonical_live_to_recorded.returncode,
|
||||
"mounted_recorded_main": mounted_recorded_main.returncode,
|
||||
"canonical_recorded_main": canonical_recorded_main.returncode,
|
||||
"canonical_live_main": canonical_live_main.returncode,
|
||||
"recorded_to_attempt": recorded_to_attempt.returncode,
|
||||
"canonical_recorded_to_attempt": canonical_recorded_to_attempt.returncode,
|
||||
"attempt_to_main": attempt_to_main.returncode,
|
||||
"canonical_attempt_to_main": canonical_attempt_to_main.returncode,
|
||||
},
|
||||
)
|
||||
)
|
||||
if not (history_transition_baseline or exact_legacy_state_ahead_baseline):
|
||||
if canonical_recorded_to_live.returncode == 1:
|
||||
raise SystemExit(60)
|
||||
raise SystemExit(61)
|
||||
|
||||
artifacts = {"containers": {expected_container: container_artifact}, "compose_sha256": ""}
|
||||
now = int(time.time())
|
||||
payload = {
|
||||
"schema_version": 3,
|
||||
"repository": repository,
|
||||
"slug": "media-codec-dashboard",
|
||||
"compose_project": "media-codec-dashboard",
|
||||
"revision": live_revision,
|
||||
"live_revision": live_revision,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_artifacts": artifacts,
|
||||
"live_receipt": {},
|
||||
"status": "success",
|
||||
"message": "Verified existing Media Codec Dashboard runtime reconciled to actual immutable live image",
|
||||
"updated_at_epoch": now,
|
||||
"artifacts": artifacts,
|
||||
"last_attempt": {
|
||||
"revision": live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified runtime state reconciliation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {"reconciled_from": reconciled_from},
|
||||
},
|
||||
"previous_revision": recorded_live,
|
||||
"rollback_restored_revision": "",
|
||||
}
|
||||
if recorded_live == live_revision and current_verified_receipt_snapshot_baseline:
|
||||
payload = dict(state)
|
||||
payload.update({
|
||||
"live_status": "healthy",
|
||||
"status": "success",
|
||||
"message": "Verified Media Codec Dashboard receipt re-attested after failed forward deployment",
|
||||
"updated_at_epoch": now,
|
||||
"artifacts": current_artifacts,
|
||||
"last_attempt": {
|
||||
"revision": live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified existing runtime receipt re-attestation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {"reconciled_failed_revision": reconciled_from},
|
||||
},
|
||||
})
|
||||
|
||||
backup = state_path.with_name(f"{state_path.name}.reconcile-backup-{now}-{os.getpid()}")
|
||||
try:
|
||||
shutil.copy2(state_path, backup)
|
||||
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
|
||||
with temporary.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.S_IMODE(state_path.stat().st_mode))
|
||||
os.replace(temporary, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
raise SystemExit(53)
|
||||
|
||||
verify = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
expected_legacy = not (recorded_live == live_revision and current_verified_receipt_snapshot_baseline)
|
||||
if not (verify.get("live_revision") == live_revision and verify.get("live_status") == "healthy" and verify.get("live_legacy_provenance") is expected_legacy and dict(dict(verify.get("live_artifacts") or {}).get("containers") or {}).get(expected_container, {}).get("image_id") == image_id):
|
||||
raise SystemExit(54)
|
||||
if not expected_legacy and (verify.get("live_receipt") != current_receipt or verify.get("live_artifacts") != current_artifacts):
|
||||
raise SystemExit(64)
|
||||
|
||||
print("reconciled", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class MediaCodecDashboardDeploymentReconciliation:
|
||||
"""Repair only stale controller state proven against the live immutable dashboard runtime."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return {"status": "controller_unavailable", "reason": redact_message(str(status.get("reason") or status.get("status") or "unavailable"))}
|
||||
controller = dict(status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {"status": "controller_unavailable", "reason": "deployment controller is not running"}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not container_id:
|
||||
return {"status": "connector_unavailable", "reason": "Portainer deployment control is unavailable"}
|
||||
|
||||
command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _CONTAINER]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command}).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "dashboard state reconciliation could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False})
|
||||
deadline = time.monotonic() + 45
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request("GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json").json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "dashboard state reconciliation timed out"}
|
||||
reasons = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
22: "Media Codec Dashboard deployment target is missing",
|
||||
23: "Media Codec Dashboard deployment target topology differs from the bounded contract",
|
||||
24: "mounted Media Codec Dashboard bare repository is unavailable",
|
||||
25: "deployment revisions cannot be resolved",
|
||||
26: "Media Codec Dashboard main cannot be resolved",
|
||||
27: "live Media Codec Dashboard revision is not an ancestor of main",
|
||||
29: "deployment lock files cannot be opened",
|
||||
30: "deployment locks cannot be acquired",
|
||||
31: "deployment projects registry cannot be read",
|
||||
36: "deployment projects registry host alias is unavailable or mismatched",
|
||||
37: "deployment target changed while reconciliation was waiting",
|
||||
40: "fixed Media Codec Dashboard reconciliation contract was altered",
|
||||
42: "deployment state file is unavailable or invalid",
|
||||
43: "deployment state is not a modern Media Codec Dashboard state",
|
||||
44: "live Media Codec Dashboard container cannot be inspected",
|
||||
45: "live Media Codec Dashboard container is not healthy in the expected Compose project",
|
||||
46: "live Media Codec Dashboard immutable image identity is invalid",
|
||||
47: "Media Codec Dashboard Compose project contains unexpected containers",
|
||||
48: "live Media Codec Dashboard mount topology differs from the verified runtime",
|
||||
49: "live Media Codec Dashboard port mapping differs from the verified runtime",
|
||||
50: "live Media Codec Dashboard image cannot be inspected",
|
||||
51: "live Media Codec Dashboard image revision label does not match the verified runtime",
|
||||
52: "controller state drift is not the exact known recoverable state",
|
||||
53: "reconciled deployment state could not be written durably",
|
||||
54: "reconciled deployment state did not verify after write",
|
||||
55: "controller state for the current live revision is internally inconsistent",
|
||||
56: "failed deployment attempt is not a descendant of the live revision",
|
||||
57: "failed deployment attempt is not an ancestor of main",
|
||||
58: "recorded live deployment revision is invalid",
|
||||
59: "stale controller state metadata is not a supported recoverable baseline",
|
||||
60: "recorded live deployment revision is not an ancestor of the verified runtime",
|
||||
61: "mounted deployment repository ancestry differs from canonical Gitea repository",
|
||||
62: "deployment controller verifier is unavailable or unsafe",
|
||||
63: "recorded dashboard receipt cannot be reverified against the live runtime",
|
||||
64: "verified dashboard receipt or artifacts differ from the recorded state",
|
||||
}
|
||||
if exit_code != 0:
|
||||
return {"status": "validation_failed", "reason": reasons.get(exit_code, "Media Codec Dashboard deployment reconciliation failed closed")}
|
||||
return {
|
||||
"status": "reconciled",
|
||||
"repository": _REPOSITORY,
|
||||
"container": _CONTAINER,
|
||||
}
|
||||
@@ -0,0 +1,637 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from threading import Lock
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..connectors.deluge import DelugeConnector
|
||||
from ..connectors.plex import PlexConnector
|
||||
from ..connectors.prowlarr import ProwlarrConnector
|
||||
from ..connectors.sonarr import SonarrConnector
|
||||
from ..connectors.tautulli import TautulliConnector
|
||||
from ..connectors.widefrog import WidefrogConnector
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import App, AppSetting, Connector, Secret
|
||||
from .audit_logger import log_event
|
||||
from .job_runner import complete_job, create_job, set_job_progress, start_job
|
||||
from .policy_engine import evaluate_policy
|
||||
|
||||
|
||||
CONNECTOR_IDS = ('sonarr', 'prowlarr', 'deluge', 'plex', 'tautulli', 'widefrog')
|
||||
BLOCKED_ACTION_TOKENS = ('delete', 'remove', 'kill_session', 'terminate', 'empty_trash', 'download_database', 'download_config')
|
||||
SNAPSHOT_CACHE_TTL_SECONDS = 15.0
|
||||
SNAPSHOT_MAX_WORKERS = 8
|
||||
SUMMARY_CACHE_SCHEMA_VERSION = 1
|
||||
SUMMARY_CACHE_SETTING_KEY = 'media_pipeline:summary_cache'
|
||||
SUMMARY_CACHE_TTL_SECONDS = 60
|
||||
SUMMARY_CACHE_MAX_STALE_SECONDS = 300
|
||||
_SNAPSHOT_CACHE: dict[tuple[int, ...], tuple[float, dict[str, Any]]] = {}
|
||||
_SNAPSHOT_CACHE_LOCK = Lock()
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return utc_now().isoformat()
|
||||
|
||||
|
||||
def _parse_utc(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
|
||||
|
||||
def _summary_cache_ttl_seconds() -> int:
|
||||
raw = os.getenv('MEDIA_PIPELINE_SUMMARY_CACHE_TTL_SECONDS', str(SUMMARY_CACHE_TTL_SECONDS))
|
||||
try:
|
||||
return max(1, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return SUMMARY_CACHE_TTL_SECONDS
|
||||
|
||||
|
||||
def _summary_cache_max_stale_seconds() -> int:
|
||||
raw = os.getenv('MEDIA_PIPELINE_SUMMARY_CACHE_MAX_STALE_SECONDS', str(SUMMARY_CACHE_MAX_STALE_SECONDS))
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return SUMMARY_CACHE_MAX_STALE_SECONDS
|
||||
|
||||
|
||||
def _redact(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return redact_message(value)
|
||||
if isinstance(value, list):
|
||||
return [_redact(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
str(key): _redact(item)
|
||||
for key, item in value.items()
|
||||
if 'secret' not in str(key).lower() and 'token' not in str(key).lower() and 'password' not in str(key).lower()
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def _int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _safe_call(call: Callable[[], dict[str, Any]], fallback_status: str = 'error') -> dict[str, Any]:
|
||||
try:
|
||||
payload = call()
|
||||
return payload if isinstance(payload, dict) else {'success': False, 'status': fallback_status, 'message': 'unexpected_payload'}
|
||||
except Exception as exc: # noqa: BLE001 - connector failures must not crash the dashboard.
|
||||
return {'success': False, 'status': fallback_status, 'message': str(exc)}
|
||||
|
||||
|
||||
def _snapshot_cache_key() -> tuple[int, ...]:
|
||||
return (
|
||||
id(SonarrConnector.health),
|
||||
id(SonarrConnector.wanted_missing),
|
||||
id(SonarrConnector.queue_status),
|
||||
id(ProwlarrConnector.health),
|
||||
id(ProwlarrConnector.indexer_status),
|
||||
id(ProwlarrConnector.application_status),
|
||||
id(ProwlarrConnector.sync_status),
|
||||
id(DelugeConnector.health),
|
||||
id(DelugeConnector.session_status),
|
||||
id(DelugeConnector.active_downloads),
|
||||
id(DelugeConnector.stalled_downloads),
|
||||
id(DelugeConnector.paused_downloads),
|
||||
id(DelugeConnector.speed_status),
|
||||
id(PlexConnector.health),
|
||||
id(PlexConnector.library_status),
|
||||
id(PlexConnector.active_sessions),
|
||||
id(PlexConnector.recent_media),
|
||||
id(PlexConnector.stream_summary),
|
||||
id(TautulliConnector.health),
|
||||
id(TautulliConnector.activity),
|
||||
id(TautulliConnector.history),
|
||||
id(TautulliConnector.watch_stats),
|
||||
id(TautulliConnector.stream_summary),
|
||||
id(TautulliConnector.libraries),
|
||||
id(WidefrogConnector.health),
|
||||
id(WidefrogConnector.queue_status),
|
||||
id(WidefrogConnector.failed_jobs),
|
||||
id(WidefrogConnector.activity),
|
||||
id(WidefrogConnector.download_status),
|
||||
)
|
||||
|
||||
|
||||
def _summary_signature(session: Session) -> str:
|
||||
connectors = [
|
||||
{
|
||||
'id': item.id,
|
||||
'status': item.status,
|
||||
'lastTest': item.last_test,
|
||||
'secretRef': item.secret_ref,
|
||||
'capabilities': item.capabilities,
|
||||
}
|
||||
for item in session.exec(select(Connector)).all()
|
||||
if item.id in CONNECTOR_IDS
|
||||
]
|
||||
secret_rows = [
|
||||
{
|
||||
'id': item.id,
|
||||
'name': item.name,
|
||||
'updatedAt': item.updated_at.isoformat() if item.updated_at else None,
|
||||
'usedBy': item.used_by_json,
|
||||
}
|
||||
for item in session.exec(select(Secret)).all()
|
||||
if any(connector_id in item.used_by_json for connector_id in CONNECTOR_IDS)
|
||||
]
|
||||
media_app_count = session.exec(select(func.count(App.id)).where(App.category.in_(['media', 'downloads']))).one()
|
||||
signature_payload = {
|
||||
'schemaVersion': SUMMARY_CACHE_SCHEMA_VERSION,
|
||||
'connectors': sorted(connectors, key=lambda item: item['id']),
|
||||
'secrets': sorted(secret_rows, key=lambda item: item['id']),
|
||||
'mediaAppCount': media_app_count,
|
||||
}
|
||||
serialized = json.dumps(signature_payload, sort_keys=True, default=str, separators=(',', ':'))
|
||||
return hashlib.sha256(serialized.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def _persistent_summary(session: Session, signature: str) -> dict[str, Any] | None:
|
||||
row = session.get(AppSetting, SUMMARY_CACHE_SETTING_KEY)
|
||||
if not row or not isinstance(row.value_json, dict):
|
||||
return None
|
||||
value = row.value_json
|
||||
if value.get('schemaVersion') != SUMMARY_CACHE_SCHEMA_VERSION or value.get('signature') != signature:
|
||||
return None
|
||||
payload = value.get('payload')
|
||||
expires_at_raw = value.get('expiresAt')
|
||||
if not isinstance(payload, dict) or not isinstance(expires_at_raw, str):
|
||||
return None
|
||||
expires_at = _parse_utc(expires_at_raw)
|
||||
now_dt = datetime.now(UTC)
|
||||
stale_until = expires_at.timestamp() + _summary_cache_max_stale_seconds()
|
||||
if now_dt >= expires_at and now_dt.timestamp() >= stale_until:
|
||||
return None
|
||||
return dict(payload)
|
||||
|
||||
|
||||
def _store_persistent_summary(session: Session, signature: str, payload: dict[str, Any]) -> None:
|
||||
ttl_seconds = _summary_cache_ttl_seconds()
|
||||
expires_at = datetime.fromtimestamp(datetime.now(UTC).replace(microsecond=0).timestamp() + ttl_seconds, UTC).isoformat().replace('+00:00', 'Z')
|
||||
row = session.get(AppSetting, SUMMARY_CACHE_SETTING_KEY)
|
||||
if not row:
|
||||
row = AppSetting(key=SUMMARY_CACHE_SETTING_KEY, is_secret=False)
|
||||
row.value_json = {
|
||||
'schemaVersion': SUMMARY_CACHE_SCHEMA_VERSION,
|
||||
'signature': signature,
|
||||
'payload': _redact(payload),
|
||||
'generatedAt': payload.get('lastUpdated'),
|
||||
'expiresAt': expires_at,
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
|
||||
def _snapshot_cached() -> dict[str, Any]:
|
||||
key = _snapshot_cache_key()
|
||||
now = monotonic()
|
||||
cached = _SNAPSHOT_CACHE.get(key)
|
||||
if cached and now - cached[0] <= SNAPSHOT_CACHE_TTL_SECONDS:
|
||||
return cached[1]
|
||||
with _SNAPSHOT_CACHE_LOCK:
|
||||
now = monotonic()
|
||||
cached = _SNAPSHOT_CACHE.get(key)
|
||||
if cached and now - cached[0] <= SNAPSHOT_CACHE_TTL_SECONDS:
|
||||
return cached[1]
|
||||
data = _build_snapshot()
|
||||
_SNAPSHOT_CACHE.clear()
|
||||
_SNAPSHOT_CACHE[key] = (now, data)
|
||||
return data
|
||||
|
||||
|
||||
def _connector_status(payload: dict[str, Any]) -> str:
|
||||
status = str(payload.get('status') or ('connected' if payload.get('success') else 'error')).lower()
|
||||
if status in {'ok', 'healthy'}:
|
||||
return 'connected'
|
||||
if status in {'connected', 'missing_config', 'auth_failed', 'unreachable', 'unsupported', 'unavailable', 'error'}:
|
||||
return status
|
||||
return 'error'
|
||||
|
||||
|
||||
def _ui_status(connector_status: str, warning: bool = False, critical: bool = False) -> str:
|
||||
if connector_status == 'missing_config':
|
||||
return 'missing_config'
|
||||
if connector_status in {'auth_failed', 'unreachable', 'error'} or critical:
|
||||
return 'critical'
|
||||
if connector_status in {'unsupported', 'unavailable'} or warning:
|
||||
return 'warning'
|
||||
return 'healthy'
|
||||
|
||||
|
||||
def _policy_action(session: Session, *, label: str, action_id: str, target: str, source: str = 'manual') -> dict[str, Any]:
|
||||
policy = evaluate_policy(session=session, action=action_id, target=target, source=source)
|
||||
return {
|
||||
'label': label,
|
||||
'actionId': action_id,
|
||||
'target': target,
|
||||
'policyDecision': policy['decision'],
|
||||
'risk': policy.get('risk', 'medium'),
|
||||
'reason': policy.get('reason'),
|
||||
'link': _action_link(action_id, target),
|
||||
}
|
||||
|
||||
|
||||
def _action_link(action_id: str, target: str) -> str | None:
|
||||
if action_id == 'media_pipeline.diagnose':
|
||||
return '/media-pipeline'
|
||||
if target in CONNECTOR_IDS:
|
||||
return f'/apps/{target}'
|
||||
return None
|
||||
|
||||
|
||||
def _filter_safe_actions(actions: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [action for action in actions if not any(token in action.get('actionId', '') for token in BLOCKED_ACTION_TOKENS)]
|
||||
|
||||
|
||||
def _build_snapshot() -> dict[str, Any]:
|
||||
calls: dict[str, Callable[[], dict[str, Any]]] = {
|
||||
'sonarrHealth': lambda: SonarrConnector().health(),
|
||||
'sonarrWanted': lambda: SonarrConnector().wanted_missing(),
|
||||
'sonarrQueue': lambda: SonarrConnector().queue_status(),
|
||||
'prowlarrHealth': lambda: ProwlarrConnector().health(),
|
||||
'prowlarrIndexers': lambda: ProwlarrConnector().indexer_status(),
|
||||
'prowlarrApplications': lambda: ProwlarrConnector().application_status(),
|
||||
'prowlarrSync': lambda: ProwlarrConnector().sync_status(),
|
||||
'delugeHealth': lambda: DelugeConnector().health(),
|
||||
'delugeSession': lambda: DelugeConnector().session_status(),
|
||||
'delugeActive': lambda: DelugeConnector().active_downloads(),
|
||||
'delugeStalled': lambda: DelugeConnector().stalled_downloads(),
|
||||
'delugePaused': lambda: DelugeConnector().paused_downloads(),
|
||||
'delugeSpeeds': lambda: DelugeConnector().speed_status(),
|
||||
'plexHealth': lambda: PlexConnector().health(),
|
||||
'plexLibraries': lambda: PlexConnector().library_status(),
|
||||
'plexSessions': lambda: PlexConnector().active_sessions(),
|
||||
'plexRecent': lambda: PlexConnector().recent_media(),
|
||||
'plexStreams': lambda: PlexConnector().stream_summary(),
|
||||
'tautulliHealth': lambda: TautulliConnector().health(),
|
||||
'tautulliActivity': lambda: TautulliConnector().activity(),
|
||||
'tautulliHistory': lambda: TautulliConnector().history(),
|
||||
'tautulliStats': lambda: TautulliConnector().watch_stats(),
|
||||
'tautulliStreams': lambda: TautulliConnector().stream_summary(),
|
||||
'tautulliLibraries': lambda: TautulliConnector().libraries(),
|
||||
'widefrogHealth': lambda: WidefrogConnector().health(),
|
||||
'widefrogQueue': lambda: WidefrogConnector().queue_status(),
|
||||
'widefrogFailed': lambda: WidefrogConnector().failed_jobs(),
|
||||
'widefrogActivity': lambda: WidefrogConnector().activity(),
|
||||
'widefrogDownloads': lambda: WidefrogConnector().download_status(),
|
||||
}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(SNAPSHOT_MAX_WORKERS, len(calls))) as executor:
|
||||
futures = {key: executor.submit(_safe_call, call) for key, call in calls.items()}
|
||||
return {key: future.result() for key, future in futures.items()}
|
||||
|
||||
|
||||
def _connector_map(data: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
return {
|
||||
'sonarr': {'status': _connector_status(data['sonarrHealth']), 'enabled': True},
|
||||
'prowlarr': {'status': _connector_status(data['prowlarrHealth']), 'enabled': True},
|
||||
'deluge': {'status': _connector_status(data['delugeHealth']), 'enabled': True},
|
||||
'plex': {'status': _connector_status(data['plexHealth']), 'enabled': True},
|
||||
'tautulli': {'status': _connector_status(data['tautulliHealth']), 'enabled': True},
|
||||
'widefrog': {'status': _connector_status(data['widefrogHealth']), 'enabled': True},
|
||||
}
|
||||
|
||||
|
||||
def _counts(data: dict[str, Any]) -> dict[str, int]:
|
||||
return {
|
||||
'wantedSeries': _int(data['sonarrWanted'].get('total')),
|
||||
'missingEpisodes': _int(data['sonarrWanted'].get('total')),
|
||||
'activeDownloads': _int(data['delugeActive'].get('total', data['delugeSession'].get('active'))),
|
||||
'stalledDownloads': _int(data['delugeStalled'].get('total', data['delugeSession'].get('stalled'))),
|
||||
'failedWidefrogJobs': _int(data['widefrogFailed'].get('failed')),
|
||||
'plexLibraries': _int(data['plexLibraries'].get('total')),
|
||||
'activeStreams': max(_int(data['plexStreams'].get('total')), _int(data['tautulliStreams'].get('activeStreams')), _int(data['tautulliActivity'].get('streamCount'))),
|
||||
'transcodes': max(_int(data['plexStreams'].get('transcoding')), _int(data['tautulliStreams'].get('transcoding'))),
|
||||
'indexers': _int(data['prowlarrIndexers'].get('total')),
|
||||
'failingIndexers': _int(data['prowlarrIndexers'].get('failing'), _int(data['prowlarrSync'].get('indexersFailing'))),
|
||||
}
|
||||
|
||||
|
||||
def _summary_from_snapshot(data: dict[str, Any]) -> dict[str, Any]:
|
||||
connectors = _connector_map(data)
|
||||
counts = _counts(data)
|
||||
missing = [key for key, value in connectors.items() if value['status'] == 'missing_config']
|
||||
hard_failures = [key for key, value in connectors.items() if value['status'] in {'auth_failed', 'unreachable', 'error'}]
|
||||
warnings: list[str] = []
|
||||
warnings.extend(f'{name} missing_config' for name in missing)
|
||||
warnings.extend(f'{name} {connectors[name]["status"]}' for name in hard_failures)
|
||||
if counts['failingIndexers']:
|
||||
warnings.append('Prowlarr has failing indexers')
|
||||
if counts['stalledDownloads']:
|
||||
warnings.append('Deluge has stalled downloads')
|
||||
if counts['failedWidefrogJobs']:
|
||||
warnings.append('Widefrog has failed jobs')
|
||||
if counts['transcodes'] >= 3:
|
||||
warnings.append('Plex has high active transcodes')
|
||||
|
||||
if hard_failures:
|
||||
status = 'critical'
|
||||
risk_level = 'high'
|
||||
elif missing:
|
||||
status = 'partial' if len(missing) < len(CONNECTOR_IDS) else 'missing_config'
|
||||
risk_level = 'medium'
|
||||
elif warnings:
|
||||
status = 'warning'
|
||||
risk_level = 'medium'
|
||||
else:
|
||||
status = 'healthy'
|
||||
risk_level = 'low'
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'status': status,
|
||||
'message': 'Media pipeline ready.' if status == 'healthy' else 'Media pipeline has items requiring attention.',
|
||||
'connectors': connectors,
|
||||
'counts': counts,
|
||||
'risk': {'level': risk_level, 'reasons': warnings},
|
||||
'lastUpdated': _now(),
|
||||
}
|
||||
|
||||
|
||||
def summary(session: Session | None = None, *, force_refresh: bool = False) -> dict[str, Any]:
|
||||
signature = _summary_signature(session) if session else None
|
||||
if session and signature and not force_refresh:
|
||||
persisted = _persistent_summary(session, signature)
|
||||
if persisted:
|
||||
return persisted
|
||||
|
||||
data = _snapshot_cached()
|
||||
payload = _summary_from_snapshot(data)
|
||||
if session and signature:
|
||||
_store_persistent_summary(session, signature, payload)
|
||||
return payload
|
||||
|
||||
|
||||
def flow(session: Session) -> dict[str, Any]:
|
||||
data = _snapshot_cached()
|
||||
connectors = _connector_map(data)
|
||||
counts = _counts(data)
|
||||
nodes = [
|
||||
{'id': 'sonarr', 'label': 'Sonarr', 'type': 'wanted', 'status': _ui_status(connectors['sonarr']['status'], counts['wantedSeries'] > 0), 'metrics': {'wanted': counts['wantedSeries'], 'queue': _int(data['sonarrQueue'].get('total'))}, 'actions': []},
|
||||
{'id': 'prowlarr', 'label': 'Prowlarr', 'type': 'indexers', 'status': _ui_status(connectors['prowlarr']['status'], counts['failingIndexers'] > 0), 'metrics': {'indexers': counts['indexers'], 'failing': counts['failingIndexers'], 'apps': _int(data['prowlarrApplications'].get('total'))}, 'actions': [_policy_action(session, label='Run search test', action_id='prowlarr.search_test', target='prowlarr')]},
|
||||
{'id': 'deluge', 'label': 'Deluge', 'type': 'downloads', 'status': _ui_status(connectors['deluge']['status'], counts['stalledDownloads'] > 0), 'metrics': {'active': counts['activeDownloads'], 'stalled': counts['stalledDownloads'], 'paused': _int(data['delugePaused'].get('total'))}, 'actions': [_policy_action(session, label='Pause all', action_id='deluge.pause_all', target='deluge')]},
|
||||
{'id': 'widefrog', 'label': 'Widefrog', 'type': 'processing', 'status': _ui_status(connectors['widefrog']['status'], counts['failedWidefrogJobs'] > 0), 'metrics': {'queue': _int(data['widefrogQueue'].get('total')), 'failed': counts['failedWidefrogJobs']}, 'actions': [_policy_action(session, label='Retry failed', action_id='widefrog.retry_failed', target='widefrog')]},
|
||||
{'id': 'plex', 'label': 'Plex', 'type': 'library', 'status': _ui_status(connectors['plex']['status'], counts['transcodes'] >= 3), 'metrics': {'libraries': counts['plexLibraries'], 'streams': counts['activeStreams'], 'transcodes': counts['transcodes']}, 'actions': [_policy_action(session, label='Scan all libraries', action_id='plex.scan_all_libraries', target='plex')]},
|
||||
{'id': 'tautulli', 'label': 'Tautulli', 'type': 'analytics', 'status': _ui_status(connectors['tautulli']['status']), 'metrics': {'streams': _int(data['tautulliStreams'].get('activeStreams')), 'plays': _int(data['tautulliStats'].get('totalPlays'))}, 'actions': []},
|
||||
]
|
||||
edges = [
|
||||
{'from': 'sonarr', 'to': 'prowlarr', 'status': 'broken' if connectors['prowlarr']['status'] in {'missing_config', 'error'} else 'healthy', 'label': 'indexer sync'},
|
||||
{'from': 'prowlarr', 'to': 'deluge', 'status': 'warning' if counts['failingIndexers'] else 'healthy', 'label': 'release search'},
|
||||
{'from': 'widefrog', 'to': 'plex', 'status': 'warning' if counts['failedWidefrogJobs'] else 'healthy', 'label': 'media processing'},
|
||||
{'from': 'plex', 'to': 'tautulli', 'status': 'broken' if connectors['tautulli']['status'] == 'missing_config' else 'healthy', 'label': 'playback analytics'},
|
||||
]
|
||||
return {'success': True, 'nodes': nodes, 'edges': edges, 'lastUpdated': _now()}
|
||||
|
||||
|
||||
def issues(session: Session) -> dict[str, Any]:
|
||||
data = _snapshot_cached()
|
||||
connectors = _connector_map(data)
|
||||
counts = _counts(data)
|
||||
result: list[dict[str, Any]] = []
|
||||
observed_at = _now()
|
||||
|
||||
for name, connector in connectors.items():
|
||||
if connector['status'] == 'missing_config':
|
||||
result.append(_issue('connector-missing-' + name, 'warning', f'{name.title()} missing_config', f'{name} connector configuration is incomplete.', [name], [connector['status']], [_configure_action(name)]))
|
||||
elif connector['status'] in {'auth_failed', 'unreachable', 'error'}:
|
||||
result.append(_issue('connector-error-' + name, 'critical', f'{name.title()} connector unavailable', f'{name} returned {connector["status"]}.', [name], [connector['status']], [_configure_action(name)]))
|
||||
|
||||
if connectors['sonarr']['status'] == 'connected' and counts['failingIndexers']:
|
||||
result.append(_issue('prowlarr-failing-indexers', 'warning', 'Prowlarr failing indexers', 'Sonarr is connected but one or more indexers are failing.', ['sonarr', 'prowlarr'], [f'{counts["failingIndexers"]} failing indexers'], [_policy_action(session, label='Run search test', action_id='prowlarr.search_test', target='prowlarr')]))
|
||||
if counts['wantedSeries'] and counts['activeDownloads'] == 0:
|
||||
result.append(_issue('wanted-no-downloads', 'info', 'Wanted items without active downloads', 'Sonarr has wanted items but Deluge has no active downloads.', ['sonarr', 'deluge'], [f'{counts["wantedSeries"]} wanted', '0 active downloads'], [_policy_action(session, label='Run Prowlarr search test', action_id='prowlarr.search_test', target='prowlarr')]))
|
||||
if counts['stalledDownloads']:
|
||||
source = 'deluge.stalled_downloads'
|
||||
result.append(_issue(
|
||||
'deluge-stalled',
|
||||
'warning',
|
||||
'Deluge stalled downloads',
|
||||
'One or more Deluge downloads are stalled.',
|
||||
['deluge'],
|
||||
[f'{counts["stalledDownloads"]} stalled downloads', f'source={source}', f'observedAt={observed_at}'],
|
||||
[_policy_action(session, label='Resume stalled torrent', action_id='deluge.resume_torrent', target='deluge')],
|
||||
source=source,
|
||||
observed_at=observed_at,
|
||||
))
|
||||
if connectors['plex']['status'] == 'connected' and connectors['tautulli']['status'] == 'missing_config':
|
||||
result.append(_issue('tautulli-missing-with-plex', 'warning', 'Tautulli analytics incomplete', 'Plex is connected but Tautulli is missing configuration.', ['plex', 'tautulli'], ['Plex connected', 'Tautulli missing_config'], [_configure_action('tautulli')]))
|
||||
if counts['transcodes'] >= 3:
|
||||
result.append(_issue('plex-high-transcodes', 'warning', 'Plex transcodes active', 'Multiple active transcodes may indicate client or server load.', ['plex', 'tautulli'], [f'{counts["transcodes"]} active transcodes'], []))
|
||||
if counts['failedWidefrogJobs']:
|
||||
result.append(_issue('widefrog-failed-jobs', 'warning', 'Widefrog failed jobs', 'Widefrog reports failed processing jobs.', ['widefrog'], [f'{counts["failedWidefrogJobs"]} failed jobs'], [_policy_action(session, label='Retry Widefrog failed jobs', action_id='widefrog.retry_failed', target='widefrog')]))
|
||||
|
||||
result.extend(_needs_review_issues(session))
|
||||
for issue in result:
|
||||
issue['suggestedActions'] = _filter_safe_actions(issue['suggestedActions'])
|
||||
return {'success': True, 'issues': result, 'lastUpdated': _now()}
|
||||
|
||||
|
||||
def _issue(issue_id: str, severity: str, title: str, description: str, affected: list[str], evidence: list[str], actions: list[dict[str, Any]], *, source: str = 'media_pipeline', observed_at: str | None = None) -> dict[str, Any]:
|
||||
return {'id': issue_id, 'severity': severity, 'title': title, 'description': description, 'affectedConnectors': affected, 'evidence': evidence, 'suggestedActions': actions, 'source': source, 'observedAt': observed_at or _now()}
|
||||
|
||||
|
||||
def _configure_action(connector: str) -> dict[str, Any]:
|
||||
link = f'/setup?step=connectors&connector={connector}' if connector in CONNECTOR_IDS else f'/apps/{connector}'
|
||||
return {'label': f'Configure {connector}', 'actionId': 'connector.configure', 'target': connector, 'policyDecision': 'allowed', 'risk': 'low', 'link': link}
|
||||
|
||||
|
||||
def _needs_review_issues(session: Session) -> list[dict[str, Any]]:
|
||||
apps = session.exec(select(App).where(App.needs_review == True)).all() # noqa: E712
|
||||
result: list[dict[str, Any]] = []
|
||||
for app in apps:
|
||||
if (app.category or '').lower() not in {'media', 'downloads'}:
|
||||
continue
|
||||
target = str(getattr(app, 'connector_type', None) or app.id)
|
||||
result.append(
|
||||
_issue(
|
||||
f'app-needs-review-{app.id}',
|
||||
'info',
|
||||
f'{app.name} needs review',
|
||||
'Imported media app needs operator review before enabling broader actions.',
|
||||
[target],
|
||||
['needsReview=true'],
|
||||
[_configure_action(target)],
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def downloads(session: Session) -> dict[str, Any]:
|
||||
data = _snapshot_cached()
|
||||
actions = [
|
||||
{'label': 'Open Deluge detail', 'actionId': 'app.open', 'target': 'deluge', 'policyDecision': 'allowed', 'risk': 'low', 'link': '/apps/deluge'},
|
||||
_policy_action(session, label='Resume stalled torrent', action_id='deluge.resume_torrent', target='deluge'),
|
||||
_policy_action(session, label='Retry Widefrog failed', action_id='widefrog.retry_failed', target='widefrog'),
|
||||
_policy_action(session, label='Pause all Deluge', action_id='deluge.pause_all', target='deluge'),
|
||||
]
|
||||
warnings = []
|
||||
if _int(data['delugeStalled'].get('total')):
|
||||
warnings.append('Deluge has stalled downloads')
|
||||
if _int(data['widefrogFailed'].get('failed')):
|
||||
warnings.append('Widefrog has failed jobs')
|
||||
return {
|
||||
'success': True,
|
||||
'active': data['delugeActive'],
|
||||
'stalled': data['delugeStalled'],
|
||||
'paused': data['delugePaused'],
|
||||
'failedWidefrogJobs': data['widefrogFailed'],
|
||||
'speedSummary': data['delugeSpeeds'],
|
||||
'queueSummary': data['widefrogQueue'],
|
||||
'warnings': warnings,
|
||||
'suggestedActions': _filter_safe_actions(actions),
|
||||
'lastUpdated': _now(),
|
||||
}
|
||||
|
||||
|
||||
def library(session: Session) -> dict[str, Any]:
|
||||
data = _snapshot_cached()
|
||||
actions = [
|
||||
_policy_action(session, label='Scan Plex library', action_id='plex.scan_library', target='plex'),
|
||||
_policy_action(session, label='Scan all Plex libraries', action_id='plex.scan_all_libraries', target='plex'),
|
||||
_policy_action(session, label='Scan Widefrog library', action_id='widefrog.scan_library', target='widefrog'),
|
||||
]
|
||||
return {
|
||||
'success': True,
|
||||
'plexLibraries': data['plexLibraries'],
|
||||
'recentMedia': data['plexRecent'],
|
||||
'activeSessions': data['plexSessions'],
|
||||
'transcodeSummary': data['plexStreams'],
|
||||
'tautulliStats': data['tautulliStats'],
|
||||
'tautulliLibraries': data['tautulliLibraries'],
|
||||
'widefrogQueue': data['widefrogQueue'],
|
||||
'scanSuggestions': _filter_safe_actions(actions),
|
||||
'suggestedActions': _filter_safe_actions(actions),
|
||||
'lastUpdated': _now(),
|
||||
}
|
||||
|
||||
|
||||
def activity() -> dict[str, Any]:
|
||||
data = _snapshot_cached()
|
||||
return {
|
||||
'success': True,
|
||||
'tautulliActivity': data['tautulliActivity'],
|
||||
'plexSessions': data['plexSessions'],
|
||||
'tautulliHistory': data['tautulliHistory'],
|
||||
'plexRecent': data['plexRecent'],
|
||||
'streamSummary': data['tautulliStreams'],
|
||||
'transcodeSummary': data['plexStreams'],
|
||||
'privacy': {'sensitive': True, 'actions': ['tautulli.history', 'tautulli.users', 'tautulli.watch_stats'], 'voicePolicy': 'approval_required'},
|
||||
'lastUpdated': _now(),
|
||||
}
|
||||
|
||||
|
||||
def recommendations(session: Session) -> dict[str, Any]:
|
||||
issue_payload = issues(session)
|
||||
items = [
|
||||
{
|
||||
'id': f'rec-{issue["id"]}',
|
||||
'severity': issue['severity'],
|
||||
'confidence': 0.86 if issue['severity'] == 'warning' else 0.72,
|
||||
'title': issue['title'],
|
||||
'evidence': issue['evidence'],
|
||||
'affectedConnectors': issue['affectedConnectors'],
|
||||
'action': issue['suggestedActions'][0] if issue['suggestedActions'] else {'label': 'Review', 'actionId': 'dashboard.review', 'target': 'media-pipeline', 'policyDecision': 'allowed', 'risk': 'low'},
|
||||
}
|
||||
for issue in issue_payload['issues']
|
||||
]
|
||||
if not items:
|
||||
items.append({'id': 'rec-healthy', 'severity': 'info', 'confidence': 0.7, 'title': 'Media pipeline healthy', 'evidence': ['No active cross-connector issues detected'], 'affectedConnectors': list(CONNECTOR_IDS), 'action': {'label': 'Review dashboard', 'actionId': 'dashboard.review', 'target': 'media-pipeline', 'policyDecision': 'allowed', 'risk': 'low'}})
|
||||
return {'success': True, 'recommendations': items, 'lastUpdated': _now()}
|
||||
|
||||
|
||||
def check(session: Session) -> dict[str, Any]:
|
||||
steps = ['Check connector statuses', 'Pull summaries', 'Detect issues', 'Write audit event', 'Return media pipeline summary']
|
||||
job = create_job(session, action_id='media_pipeline.check', target='media-pipeline', source='manual', risk='low', policy_decision='allowed', steps=steps)
|
||||
start_job(session, job)
|
||||
set_job_progress(session, job, 45, 'Collected connector summaries.')
|
||||
summary_payload = summary(session, force_refresh=True)
|
||||
issue_payload = issues(session)
|
||||
completed = complete_job(session, job, status='succeeded', result_summary=summary_payload['status'])
|
||||
audit = log_event(
|
||||
session,
|
||||
source='action',
|
||||
identity='manual',
|
||||
action_id='media_pipeline.check',
|
||||
target='media-pipeline',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Read-only media pipeline check.',
|
||||
result='success',
|
||||
execution_plan={'plan': steps, 'jobId': completed.id},
|
||||
redacted_metadata={'connectors': summary_payload['connectors'], 'issueCount': len(issue_payload['issues'])},
|
||||
)
|
||||
return {'success': True, 'summary': summary_payload, 'issues': issue_payload['issues'], 'jobId': completed.id, 'auditId': audit.id}
|
||||
|
||||
|
||||
def diagnose(session: Session, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
query = str(payload.get('query') or '').strip()
|
||||
scope = str(payload.get('scope') or 'all').strip() or 'all'
|
||||
data = _snapshot_cached()
|
||||
counts = _counts(data)
|
||||
likely_causes: list[str] = []
|
||||
evidence: list[str] = []
|
||||
actions: list[dict[str, Any]] = []
|
||||
|
||||
if counts['wantedSeries'] and counts['activeDownloads'] == 0:
|
||||
likely_causes.append('Wanted media exists but no download is currently active.')
|
||||
evidence.extend([f'{counts["wantedSeries"]} wanted items', '0 active Deluge downloads'])
|
||||
actions.append(_policy_action(session, label='Run Prowlarr search test', action_id='prowlarr.search_test', target='prowlarr'))
|
||||
if counts['failingIndexers']:
|
||||
likely_causes.append('One or more Prowlarr indexers are failing.')
|
||||
evidence.append(f'{counts["failingIndexers"]} failing indexers')
|
||||
actions.append(_policy_action(session, label='Run Prowlarr search test', action_id='prowlarr.search_test', target='prowlarr'))
|
||||
if counts['stalledDownloads']:
|
||||
likely_causes.append('Deluge has stalled downloads.')
|
||||
evidence.append(f'{counts["stalledDownloads"]} stalled downloads')
|
||||
actions.append(_policy_action(session, label='Resume stalled torrent', action_id='deluge.resume_torrent', target='deluge'))
|
||||
if counts['failedWidefrogJobs']:
|
||||
likely_causes.append('Widefrog has failed processing jobs.')
|
||||
evidence.append(f'{counts["failedWidefrogJobs"]} failed Widefrog jobs')
|
||||
actions.append(_policy_action(session, label='Retry Widefrog failed', action_id='widefrog.retry_failed', target='widefrog'))
|
||||
if counts['transcodes'] >= 3:
|
||||
likely_causes.append('Plex has several active transcodes.')
|
||||
evidence.append(f'{counts["transcodes"]} transcodes')
|
||||
|
||||
if not likely_causes:
|
||||
likely_causes.append('No obvious rule-based issue found.')
|
||||
evidence.append('Connector summaries did not cross configured thresholds')
|
||||
|
||||
audit = log_event(
|
||||
session,
|
||||
source='action',
|
||||
identity='manual',
|
||||
action_id='media_pipeline.diagnose',
|
||||
target='media-pipeline',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Read-only media pipeline diagnosis.',
|
||||
result='success',
|
||||
execution_plan={'query': query, 'scope': scope},
|
||||
redacted_metadata={'scope': scope},
|
||||
)
|
||||
return {
|
||||
'success': True,
|
||||
'diagnosis': {
|
||||
'query': query,
|
||||
'scope': scope,
|
||||
'likelyCauses': likely_causes,
|
||||
'evidence': evidence,
|
||||
'suggestedActions': _filter_safe_actions(actions),
|
||||
'executedActions': [],
|
||||
'auditId': audit.id,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import and_, or_
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.redaction import redact_value
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting, AuditEvent, JobRun, NotificationEvent
|
||||
from .action_registry import ACTIONS, ACTION_RISK
|
||||
from .approval_service import create_approval, get_approval
|
||||
from .audit_logger import log_event
|
||||
from .policy_engine import evaluate_policy
|
||||
|
||||
|
||||
_IDEMPOTENCY_LOCK = Lock()
|
||||
_IDEMPOTENCY_PREFIX = "n8n:idempotency:"
|
||||
_IDEMPOTENCY_MAX_RECORDS = 5_000
|
||||
_HEARTBEAT_KEY = "n8n:workflow_heartbeat"
|
||||
|
||||
|
||||
def _redact(value: Any) -> Any:
|
||||
return redact_value(value)
|
||||
|
||||
|
||||
def _encode_cursor(timestamp: datetime, event_id: str) -> str:
|
||||
raw = json.dumps({"at": timestamp.astimezone(UTC).isoformat(), "id": event_id}, separators=(",", ":"))
|
||||
return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
def _decode_cursor(value: str | None) -> tuple[datetime, str]:
|
||||
if not value:
|
||||
return datetime.fromtimestamp(0, UTC), ""
|
||||
try:
|
||||
padded = value + "=" * (-len(value) % 4)
|
||||
payload = json.loads(base64.urlsafe_b64decode(padded).decode())
|
||||
timestamp = datetime.fromisoformat(str(payload["at"]).replace("Z", "+00:00"))
|
||||
return timestamp.astimezone(UTC), str(payload["id"])
|
||||
except (ValueError, KeyError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError("invalid_cursor") from exc
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||
|
||||
|
||||
def event_feed(session: Session, *, cursor: str | None, limit: int) -> dict[str, Any]:
|
||||
after_at, after_id = _decode_cursor(cursor)
|
||||
after_db = after_at.replace(tzinfo=None)
|
||||
events: list[dict[str, Any]] = []
|
||||
audit_query = (
|
||||
select(AuditEvent)
|
||||
.where(or_(
|
||||
AuditEvent.timestamp > after_db,
|
||||
and_(AuditEvent.timestamp == after_db, AuditEvent.id > after_id),
|
||||
))
|
||||
.order_by(AuditEvent.timestamp, AuditEvent.id)
|
||||
.limit(limit + 1)
|
||||
)
|
||||
for row in session.exec(audit_query).all():
|
||||
events.append(_redact({
|
||||
"id": row.id,
|
||||
"occurredAt": _as_utc(row.timestamp),
|
||||
"type": "audit.action",
|
||||
"severity": row.risk,
|
||||
"source": row.source,
|
||||
"target": row.target,
|
||||
"payload": {
|
||||
"actionId": row.action_id,
|
||||
"policyDecision": row.policy_decision,
|
||||
"result": row.result,
|
||||
"reason": row.reason,
|
||||
"metadata": row.redacted_metadata or {},
|
||||
},
|
||||
}))
|
||||
notification_query = (
|
||||
select(NotificationEvent)
|
||||
.where(or_(
|
||||
NotificationEvent.created_at > after_db,
|
||||
and_(NotificationEvent.created_at == after_db, NotificationEvent.id > after_id),
|
||||
))
|
||||
.order_by(NotificationEvent.created_at, NotificationEvent.id)
|
||||
.limit(limit + 1)
|
||||
)
|
||||
for row in session.exec(notification_query).all():
|
||||
events.append(_redact({
|
||||
"id": row.id,
|
||||
"occurredAt": _as_utc(row.created_at),
|
||||
"type": "notification.created",
|
||||
"severity": row.severity,
|
||||
"source": row.source,
|
||||
"target": row.target,
|
||||
"payload": {
|
||||
"title": row.title,
|
||||
"message": row.message,
|
||||
"category": row.category,
|
||||
"status": row.status,
|
||||
},
|
||||
}))
|
||||
events.sort(key=lambda item: (item["occurredAt"], item["id"]))
|
||||
page = events[:limit]
|
||||
for item in page:
|
||||
item["occurredAt"] = _as_utc(item["occurredAt"]).isoformat()
|
||||
next_cursor = cursor
|
||||
if page:
|
||||
last = page[-1]
|
||||
next_cursor = _encode_cursor(datetime.fromisoformat(last["occurredAt"]), last["id"])
|
||||
return {"items": page, "nextCursor": next_cursor, "hasMore": len(events) > len(page)}
|
||||
|
||||
|
||||
def record_heartbeat(session: Session, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
now = utc_now()
|
||||
setting = session.get(AppSetting, _HEARTBEAT_KEY)
|
||||
if not setting:
|
||||
setting = AppSetting(key=_HEARTBEAT_KEY, is_secret=False)
|
||||
previous = setting.value_json if isinstance(setting.value_json, dict) else {}
|
||||
branch = str(payload.get("branch", "unknown"))[:40]
|
||||
branch_status = {
|
||||
"branch": branch,
|
||||
"processedEvents": max(0, min(int(payload.get("processedEvents", 0) or 0), 100_000)),
|
||||
"backlogRemaining": bool(payload.get("backlogRemaining", False)),
|
||||
"confidence": str(payload.get("confidence", ""))[:20],
|
||||
"receivedAt": now.isoformat(),
|
||||
}
|
||||
branches = previous.get("branches") if isinstance(previous.get("branches"), dict) else {}
|
||||
branches = {**branches, branch: branch_status}
|
||||
event_status = branch_status if branch == "events" else branches.get("events", {})
|
||||
setting.value_json = {
|
||||
"workflowId": str(payload.get("workflowId", ""))[:120],
|
||||
"workflowVersion": str(payload.get("workflowVersion", ""))[:40],
|
||||
"branch": branch,
|
||||
"processedEvents": int(event_status.get("processedEvents", 0) or 0),
|
||||
"backlogRemaining": bool(event_status.get("backlogRemaining", False)),
|
||||
"confidence": branch_status["confidence"],
|
||||
"receivedAt": now.isoformat(),
|
||||
"branches": branches,
|
||||
}
|
||||
setting.updated_at = now
|
||||
session.add(setting)
|
||||
session.commit()
|
||||
return {"accepted": True, "receivedAt": now.isoformat(), "canExecute": False}
|
||||
|
||||
|
||||
def heartbeat_status(session: Session) -> dict[str, Any]:
|
||||
setting = session.get(AppSetting, _HEARTBEAT_KEY)
|
||||
payload = setting.value_json if setting and isinstance(setting.value_json, dict) else {}
|
||||
received_at = payload.get("receivedAt")
|
||||
fresh = False
|
||||
if received_at:
|
||||
try:
|
||||
observed = _as_utc(datetime.fromisoformat(str(received_at).replace("Z", "+00:00")))
|
||||
fresh = (datetime.now(UTC) - observed).total_seconds() <= 30 * 60
|
||||
except ValueError:
|
||||
fresh = False
|
||||
return {
|
||||
"state": "reachable" if fresh else "configured",
|
||||
"lastVerifiedAt": received_at,
|
||||
"workflow": payload or None,
|
||||
}
|
||||
|
||||
|
||||
def _trim_idempotency_records(session: Session) -> None:
|
||||
stale = session.exec(
|
||||
select(AppSetting)
|
||||
.where(AppSetting.key.startswith(_IDEMPOTENCY_PREFIX))
|
||||
.order_by(AppSetting.updated_at.desc())
|
||||
.offset(_IDEMPOTENCY_MAX_RECORDS - 1)
|
||||
).all()
|
||||
for row in stale:
|
||||
session.delete(row)
|
||||
|
||||
|
||||
def preflight(session: Session, action: str, target: str) -> dict[str, Any]:
|
||||
decision = evaluate_policy(session, action=action, target=target, source="automation")
|
||||
return {
|
||||
"action": action,
|
||||
"target": target,
|
||||
"source": "automation",
|
||||
"risk": decision.get("risk", ACTION_RISK.get(action, "critical")),
|
||||
"policyDecision": decision["decision"],
|
||||
"reason": decision["reason"],
|
||||
"knownAction": action in ACTIONS,
|
||||
"executionAvailable": False,
|
||||
"nextStep": "create_approval" if decision["decision"] == "approval_required" else "manual_review" if decision["decision"] == "blocked" else "use_typed_appops_surface",
|
||||
}
|
||||
|
||||
|
||||
def request_approval(
|
||||
session: Session,
|
||||
*,
|
||||
action: str,
|
||||
target: str,
|
||||
reason: str,
|
||||
plan: list[str],
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
with _IDEMPOTENCY_LOCK:
|
||||
return _request_approval_locked(
|
||||
session,
|
||||
action=action,
|
||||
target=target,
|
||||
reason=reason,
|
||||
plan=plan,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
|
||||
def _request_approval_locked(
|
||||
session: Session,
|
||||
*,
|
||||
action: str,
|
||||
target: str,
|
||||
reason: str,
|
||||
plan: list[str],
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
request_digest = hashlib.sha256(json.dumps({
|
||||
"action": action, "target": target, "reason": reason, "plan": plan,
|
||||
}, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
setting_key = f"{_IDEMPOTENCY_PREFIX}{hashlib.sha256(idempotency_key.encode()).hexdigest()}"
|
||||
existing = session.get(AppSetting, setting_key)
|
||||
if existing:
|
||||
if existing.value_json.get("requestDigest") != request_digest:
|
||||
raise ValueError("idempotency_key_reused_with_different_request")
|
||||
return {**existing.value_json.get("response", {}), "idempotentReplay": True}
|
||||
|
||||
preview = preflight(session, action, target)
|
||||
if preview["policyDecision"] == "blocked":
|
||||
log_event(
|
||||
session, source="automation", identity="n8n", action_id=action, target=target,
|
||||
risk=preview["risk"], policy_decision="blocked", reason=preview["reason"], result="blocked",
|
||||
redacted_metadata={"integration": "n8n", "request": "approval"},
|
||||
)
|
||||
raise PermissionError("action_blocked_by_policy")
|
||||
if preview["policyDecision"] != "approval_required":
|
||||
response = {**preview, "created": False, "message": "Deze actie vereist volgens de huidige policy geen approval; n8n voert haar niet uit."}
|
||||
else:
|
||||
safe_reason = str(_redact(reason))[:1000]
|
||||
approval = create_approval(
|
||||
session,
|
||||
action=action,
|
||||
target=target,
|
||||
source="automation",
|
||||
risk=preview["risk"],
|
||||
reason=safe_reason,
|
||||
requester="n8n",
|
||||
context={"plan": [_redact(item)[:300] for item in plan[:30]], "integration": "n8n"},
|
||||
)
|
||||
audit = log_event(
|
||||
session, source="automation", identity="n8n", action_id="approval.request", target=target,
|
||||
target_type="approval", risk=preview["risk"], policy_decision="approval_required",
|
||||
reason=safe_reason, result="approval_requested",
|
||||
redacted_metadata={"approvalId": approval.id, "requestedAction": action, "integration": "n8n"},
|
||||
)
|
||||
response = {
|
||||
**preview,
|
||||
"created": True,
|
||||
"approvalId": approval.id,
|
||||
"approvalStatus": approval.status,
|
||||
"auditId": audit.id,
|
||||
"expiresAt": approval.expires_at.isoformat(),
|
||||
}
|
||||
_trim_idempotency_records(session)
|
||||
session.add(AppSetting(key=setting_key, value_json={"requestDigest": request_digest, "response": response}))
|
||||
session.commit()
|
||||
return {**response, "idempotentReplay": False}
|
||||
|
||||
|
||||
def approval_status(session: Session, approval_id: str) -> dict[str, Any] | None:
|
||||
approval = get_approval(session, approval_id)
|
||||
if not approval:
|
||||
return None
|
||||
return {
|
||||
"id": approval.id,
|
||||
"action": approval.action,
|
||||
"target": approval.target,
|
||||
"risk": approval.risk,
|
||||
"status": approval.status,
|
||||
"expiresAt": approval.expires_at,
|
||||
"jobId": approval.job_id,
|
||||
"auditEventId": approval.audit_event_id,
|
||||
}
|
||||
|
||||
|
||||
def job_status(session: Session, job_id: str) -> dict[str, Any] | None:
|
||||
job = session.get(JobRun, job_id)
|
||||
if not job:
|
||||
return None
|
||||
return _redact(job.model_dump())
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Notification channel delivery helpers."""
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from email.message import EmailMessage
|
||||
import json
|
||||
import smtplib
|
||||
import ssl
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib import error, request
|
||||
|
||||
from ...core.redaction import redact_message
|
||||
from ...db.models import NotificationChannel
|
||||
from ..secret_store import get_secret
|
||||
|
||||
|
||||
SECRET_REFS = {
|
||||
"discord": "NOTIFICATION_DISCORD_WEBHOOK",
|
||||
"telegram": "NOTIFICATION_TELEGRAM_BOT_TOKEN",
|
||||
"email": "NOTIFICATION_SMTP_PASSWORD",
|
||||
}
|
||||
|
||||
|
||||
def safe_config(channel_type: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
allowed = {
|
||||
"discord": {"username", "timeoutSeconds", "maxAttempts", "retryBackoffSeconds"},
|
||||
"telegram": {"chatId", "timeoutSeconds", "maxAttempts", "retryBackoffSeconds"},
|
||||
"email": {"host", "port", "username", "fromAddress", "toAddress", "useTls", "smtpMode", "timeoutSeconds", "maxAttempts", "retryBackoffSeconds"},
|
||||
}.get(channel_type, set())
|
||||
return {key: value for key, value in config.items() if key in allowed}
|
||||
|
||||
|
||||
def config_status(session, channel: NotificationChannel) -> dict[str, Any]:
|
||||
config = safe_config(channel.type, channel.config_json or {})
|
||||
secret_ref = SECRET_REFS.get(channel.type)
|
||||
has_secret = bool(secret_ref and get_secret(session, secret_ref, touch=False))
|
||||
missing: list[str] = []
|
||||
if channel.type == "telegram" and not str(config.get("chatId") or "").strip():
|
||||
missing.append("chatId")
|
||||
if channel.type == "email":
|
||||
for field in ("host", "fromAddress", "toAddress"):
|
||||
if not str(config.get(field) or "").strip():
|
||||
missing.append(field)
|
||||
if not has_secret:
|
||||
missing.append("credential")
|
||||
elif channel.type == "discord":
|
||||
webhook = get_secret(session, secret_ref, touch=False) or ""
|
||||
if not webhook.startswith(("https://discord.com/api/webhooks/", "https://discordapp.com/api/webhooks/")):
|
||||
missing.append("validDiscordWebhook")
|
||||
return {
|
||||
"status": "disabled" if not channel.enabled else ("configured" if not missing else "missing_config"),
|
||||
"hasSecret": has_secret,
|
||||
"missing": missing,
|
||||
"safeConfig": config,
|
||||
}
|
||||
|
||||
|
||||
def _http_json(url: str, payload: dict[str, Any], timeout: int) -> dict[str, Any]:
|
||||
try:
|
||||
body = json.dumps(payload).encode()
|
||||
req = request.Request(url, data=body, headers={"Content-Type": "application/json"}, method="POST")
|
||||
with request.urlopen(req, timeout=timeout) as response:
|
||||
status = int(response.status)
|
||||
return {"success": 200 <= status < 300, "status": "sent" if 200 <= status < 300 else "failed", "httpStatus": status}
|
||||
except error.HTTPError as exc:
|
||||
return {"success": False, "status": "failed", "httpStatus": int(exc.code), "reason": "remote_http_error"}
|
||||
except (error.URLError, TimeoutError, OSError, ValueError):
|
||||
return {"success": False, "status": "failed", "reason": "connection_failed"}
|
||||
|
||||
|
||||
def _send_once(session, channel: NotificationChannel, event: dict[str, Any]) -> dict[str, Any]:
|
||||
status = config_status(session, channel)
|
||||
if not channel.enabled:
|
||||
return {"success": True, "status": "disabled"}
|
||||
if status["missing"]:
|
||||
return {"success": False, "status": "missing_config", "missing": status["missing"]}
|
||||
config = status["safeConfig"]
|
||||
timeout = max(2, min(int(config.get("timeoutSeconds") or 10), 30))
|
||||
title = redact_message(str(event.get("title") or "AppOps notification"))[:200]
|
||||
message = redact_message(str(event.get("message") or ""))[:1500]
|
||||
secret = get_secret(session, SECRET_REFS[channel.type], touch=True) or ""
|
||||
|
||||
if channel.type == "discord":
|
||||
return _http_json(secret, {"username": str(config.get("username") or "AppOps"), "content": f"**{title}**\n{message}"}, timeout)
|
||||
if channel.type == "telegram":
|
||||
url = f"https://api.telegram.org/bot{secret}/sendMessage"
|
||||
return _http_json(url, {"chat_id": str(config["chatId"]), "text": f"{title}\n{message}"}, timeout)
|
||||
if channel.type == "email":
|
||||
email = EmailMessage()
|
||||
email["Subject"] = title
|
||||
email["From"] = str(config["fromAddress"])
|
||||
email["To"] = str(config["toAddress"])
|
||||
email.set_content(message)
|
||||
try:
|
||||
smtp_mode = str(config.get("smtpMode") or ("implicit_tls" if config.get("useTls", True) else "plain"))
|
||||
if smtp_mode not in {"implicit_tls", "starttls", "plain"}:
|
||||
return {"success": False, "status": "missing_config", "missing": ["validSmtpMode"]}
|
||||
port = int(config.get("port") or (465 if smtp_mode == "implicit_tls" else 587 if smtp_mode == "starttls" else 25))
|
||||
if smtp_mode == "implicit_tls":
|
||||
with smtplib.SMTP_SSL(str(config["host"]), port, timeout=timeout, context=ssl.create_default_context()) as smtp:
|
||||
if config.get("username"):
|
||||
smtp.login(str(config["username"]), secret)
|
||||
smtp.send_message(email)
|
||||
else:
|
||||
with smtplib.SMTP(str(config["host"]), port, timeout=timeout) as smtp:
|
||||
if smtp_mode == "starttls":
|
||||
smtp.ehlo()
|
||||
smtp.starttls(context=ssl.create_default_context())
|
||||
smtp.ehlo()
|
||||
if config.get("username"):
|
||||
smtp.login(str(config["username"]), secret)
|
||||
smtp.send_message(email)
|
||||
return {"success": True, "status": "sent"}
|
||||
except (OSError, smtplib.SMTPException, ValueError):
|
||||
return {"success": False, "status": "failed", "reason": "smtp_delivery_failed"}
|
||||
return {"success": False, "status": "unsupported_channel"}
|
||||
|
||||
|
||||
def send_notification(session, channel: NotificationChannel, event: dict[str, Any]) -> dict[str, Any]:
|
||||
config = safe_config(channel.type, channel.config_json or {})
|
||||
max_attempts = max(1, min(int(config.get("maxAttempts") or 3), 3))
|
||||
backoff = max(0.0, min(float(config.get("retryBackoffSeconds") or 0.25), 2.0))
|
||||
result: dict[str, Any] = {"success": False, "status": "failed", "reason": "delivery_failed"}
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
result = _send_once(session, channel, event)
|
||||
result["attempts"] = attempt
|
||||
if result.get("success") or result.get("status") in {"disabled", "missing_config", "unsupported_channel"}:
|
||||
return result
|
||||
retryable = result.get("reason") in {"connection_failed", "smtp_delivery_failed"} or int(result.get("httpStatus") or 0) >= 500 or int(result.get("httpStatus") or 0) == 429
|
||||
if not retryable:
|
||||
return result
|
||||
if attempt < max_attempts and backoff:
|
||||
time.sleep(backoff * (2 ** (attempt - 1)))
|
||||
result["deadLetter"] = True
|
||||
return result
|
||||
@@ -0,0 +1,211 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ...core.config import settings
|
||||
from ...db.models import NotificationChannel
|
||||
from ..secret_store import get_secret
|
||||
|
||||
SECRET_REF = "HOME_ASSISTANT_TOKEN"
|
||||
|
||||
|
||||
def _api_value(key: str, default: Any = None) -> Any:
|
||||
value = settings.apis.get(key)
|
||||
return default if value is None else value
|
||||
|
||||
|
||||
def _bool(value: Any, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _int(value: Any, default: int) -> int:
|
||||
try:
|
||||
return max(1, int(value))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _clean_url(value: Any) -> str | None:
|
||||
text = str(value or "").strip().rstrip("/")
|
||||
return text or None
|
||||
|
||||
|
||||
def normalize_config(raw: dict[str, Any] | None) -> dict[str, Any]:
|
||||
config = dict(raw or {})
|
||||
return {
|
||||
"url": _clean_url(config.get("url") or _api_value("HOME_ASSISTANT_URL")),
|
||||
"notifyDomain": str(config.get("notifyDomain") or _api_value("HOME_ASSISTANT_NOTIFY_DOMAIN", "notify") or "notify").strip() or "notify",
|
||||
"notifyService": str(config.get("notifyService") or _api_value("HOME_ASSISTANT_NOTIFY_SERVICE", "notify") or "notify").strip() or "notify",
|
||||
"target": str(config.get("target") or _api_value("HOME_ASSISTANT_NOTIFY_TARGET") or "").strip() or None,
|
||||
"verifyTls": _bool(config.get("verifyTls"), _bool(_api_value("HOME_ASSISTANT_VERIFY_TLS"), False)),
|
||||
"timeoutSeconds": _int(config.get("timeoutSeconds") or _api_value("HOME_ASSISTANT_TIMEOUT_SECONDS"), 10),
|
||||
"titlePrefix": str(config.get("titlePrefix") or _api_value("HOME_ASSISTANT_NOTIFICATION_TITLE_PREFIX", "AppOps") or "AppOps").strip() or "AppOps",
|
||||
"dryRun": _bool(config.get("dryRun"), _bool(_api_value("HOME_ASSISTANT_DRY_RUN"), False)),
|
||||
}
|
||||
|
||||
|
||||
def safe_config(config: dict[str, Any] | None) -> dict[str, Any]:
|
||||
normalized = normalize_config(config)
|
||||
return {
|
||||
"url": normalized["url"],
|
||||
"notifyDomain": normalized["notifyDomain"],
|
||||
"notifyService": normalized["notifyService"],
|
||||
"target": normalized["target"],
|
||||
"verifyTls": normalized["verifyTls"],
|
||||
"timeoutSeconds": normalized["timeoutSeconds"],
|
||||
"titlePrefix": normalized["titlePrefix"],
|
||||
"dryRun": normalized["dryRun"],
|
||||
}
|
||||
|
||||
|
||||
def _token(session: Session) -> str | None:
|
||||
return get_secret(session, SECRET_REF) or (str(_api_value(SECRET_REF) or "").strip() or None)
|
||||
|
||||
|
||||
def config_status(session: Session, channel: NotificationChannel) -> dict[str, Any]:
|
||||
config = normalize_config(channel.config_json)
|
||||
missing: list[str] = []
|
||||
token = _token(session)
|
||||
if not config["url"]:
|
||||
missing.append("HOME_ASSISTANT_URL")
|
||||
if not token:
|
||||
missing.append("HOME_ASSISTANT_TOKEN")
|
||||
if missing:
|
||||
status = "missing_config"
|
||||
elif not channel.enabled:
|
||||
status = "disabled"
|
||||
else:
|
||||
status = "connected"
|
||||
return {
|
||||
"status": status,
|
||||
"missing": missing,
|
||||
"hasSecret": bool(token),
|
||||
"safeConfig": safe_config(channel.config_json),
|
||||
}
|
||||
|
||||
|
||||
def format_message(event: dict[str, Any]) -> str:
|
||||
severity = str(event.get("severity") or "info").upper()
|
||||
source = str(event.get("source") or "appops")
|
||||
target = str(event.get("target") or "global")
|
||||
message = str(event.get("message") or event.get("title") or "Notification")
|
||||
return f"{severity} - {source} - {target}\n{message}"
|
||||
|
||||
|
||||
def _title(config: dict[str, Any], event: dict[str, Any]) -> str:
|
||||
prefix = str(config.get("titlePrefix") or "AppOps")
|
||||
title = str(event.get("title") or "Notification")
|
||||
return f"{prefix}: {title}" if prefix else title
|
||||
|
||||
|
||||
def _request_context(config: dict[str, Any]) -> ssl.SSLContext | None:
|
||||
if config["verifyTls"]:
|
||||
return None
|
||||
return ssl._create_unverified_context()
|
||||
|
||||
|
||||
def _call_home_assistant(config: dict[str, Any], token: str, method: str, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
body = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
request = urllib.request.Request(
|
||||
f"{config['url']}{path}",
|
||||
data=body,
|
||||
method=method,
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=config["timeoutSeconds"], context=_request_context(config)) as response:
|
||||
return {"ok": True, "statusCode": response.status}
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in {401, 403}:
|
||||
return {"ok": False, "status": "auth_failed", "statusCode": exc.code}
|
||||
return {"ok": False, "status": "error", "statusCode": exc.code}
|
||||
except urllib.error.URLError:
|
||||
return {"ok": False, "status": "unreachable"}
|
||||
except OSError:
|
||||
return {"ok": False, "status": "unreachable"}
|
||||
|
||||
|
||||
def test_connection(session: Session, channel: NotificationChannel) -> dict[str, Any]:
|
||||
status = config_status(session, channel)
|
||||
if status["missing"]:
|
||||
return {"success": False, "status": "missing_config", "missing": status["missing"]}
|
||||
config = normalize_config(channel.config_json)
|
||||
token = _token(session) or ""
|
||||
if config["dryRun"]:
|
||||
return {"success": True, "status": "connected", "dryRun": True}
|
||||
result = _call_home_assistant(config, token, "GET", "/api/")
|
||||
return {
|
||||
"success": bool(result["ok"]),
|
||||
"status": "connected" if result["ok"] else result.get("status", "error"),
|
||||
"statusCode": result.get("statusCode"),
|
||||
}
|
||||
|
||||
|
||||
def _notify_payload(config: dict[str, Any], event: dict[str, Any]) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"title": _title(config, event),
|
||||
"message": format_message(event),
|
||||
"data": {
|
||||
"source": event.get("source"),
|
||||
"severity": event.get("severity"),
|
||||
"category": event.get("category"),
|
||||
"target": event.get("target"),
|
||||
"ruleId": event.get("ruleId"),
|
||||
"eventId": event.get("id"),
|
||||
},
|
||||
}
|
||||
if config["target"]:
|
||||
payload["target"] = config["target"]
|
||||
return payload
|
||||
|
||||
|
||||
def send_notification(session: Session, channel: NotificationChannel, event: dict[str, Any]) -> dict[str, Any]:
|
||||
if not channel.enabled:
|
||||
return {"status": "suppressed", "reason": "disabled"}
|
||||
status = config_status(session, channel)
|
||||
if status["missing"]:
|
||||
return {"status": "failed", "reason": "missing_config", "missing": status["missing"]}
|
||||
config = normalize_config(channel.config_json)
|
||||
if config["dryRun"]:
|
||||
return {"status": "sent", "reason": "dry_run"}
|
||||
token = _token(session) or ""
|
||||
path = f"/api/services/{config['notifyDomain']}/{config['notifyService']}"
|
||||
result = _call_home_assistant(config, token, "POST", path, _notify_payload(config, event))
|
||||
if result["ok"]:
|
||||
return {"status": "sent"}
|
||||
return {"status": "failed", "reason": result.get("status", "error"), "statusCode": result.get("statusCode")}
|
||||
|
||||
|
||||
def send_test_notification(session: Session, channel: NotificationChannel) -> dict[str, Any]:
|
||||
event = {
|
||||
"id": "test",
|
||||
"ruleId": "notification.channel.test",
|
||||
"source": "notification",
|
||||
"event_type": "notification.channel.test",
|
||||
"severity": "info",
|
||||
"title": "Home Assistant test notification",
|
||||
"message": "AppOps test notification.",
|
||||
"category": "test",
|
||||
"target": "home_assistant",
|
||||
}
|
||||
result = send_notification(session, channel, event)
|
||||
return {
|
||||
"success": result["status"] == "sent",
|
||||
"status": result["status"],
|
||||
"reason": result.get("reason"),
|
||||
"missing": result.get("missing"),
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..db.models import Policy
|
||||
from .action_registry import ACTION_RISK, ACTIONS
|
||||
|
||||
_DEFAULT_DECISIONS = {
|
||||
'privatemuse.comfyui.install_artifact': 'approval_required',
|
||||
'deployment.target.deploy_workmesh': 'approval_required',
|
||||
'deployment.target.promote_workmesh': 'approval_required',
|
||||
'container.clear_logs': 'blocked',
|
||||
'container.stop': 'blocked',
|
||||
'gateway.clear_cache': 'blocked',
|
||||
'jellyfin.scan_library': 'blocked',
|
||||
'volume.delete': 'blocked',
|
||||
'docker.prune': 'blocked',
|
||||
'secrets.show': 'blocked',
|
||||
'unraid.disk.format': 'blocked',
|
||||
'unraid.vm.stop': 'blocked',
|
||||
'unraid.vm.reboot': 'blocked',
|
||||
'unraid.vm.force_stop': 'blocked',
|
||||
'unraid.vm.reset': 'blocked',
|
||||
'sonarr.delete_series': 'blocked',
|
||||
'sonarr.mass_edit': 'blocked',
|
||||
'radarr.delete_movie': 'blocked',
|
||||
'radarr.delete_files': 'blocked',
|
||||
'radarr.mass_edit': 'blocked',
|
||||
'widefrog.delete_job': 'blocked',
|
||||
'widefrog.delete_media': 'blocked',
|
||||
'widefrog.clear_library': 'blocked',
|
||||
'widefrog.clear_queue': 'blocked',
|
||||
'widefrog.reset_database': 'blocked',
|
||||
'widefrog.delete_downloads': 'blocked',
|
||||
'widefrog.prune': 'blocked',
|
||||
'prowlarr.delete_indexer': 'blocked',
|
||||
'prowlarr.delete_application': 'blocked',
|
||||
'prowlarr.update_indexer': 'blocked',
|
||||
'prowlarr.update_application': 'blocked',
|
||||
'prowlarr.grab_release': 'blocked',
|
||||
'prowlarr.push_release': 'blocked',
|
||||
'prowlarr.mass_edit': 'blocked',
|
||||
'prowlarr.test_all_indexers': 'blocked',
|
||||
'deluge.remove_torrent': 'blocked',
|
||||
'deluge.remove_torrent_data': 'blocked',
|
||||
'deluge.delete_data': 'blocked',
|
||||
'deluge.force_recheck': 'blocked',
|
||||
'deluge.move_storage': 'blocked',
|
||||
'deluge.add_torrent': 'blocked',
|
||||
'deluge.add_magnet': 'blocked',
|
||||
'deluge.mass_edit': 'blocked',
|
||||
'deluge.set_options': 'blocked',
|
||||
'deluge.unknown_mutation': 'blocked',
|
||||
'plex.delete_media': 'blocked',
|
||||
'plex.delete_library': 'blocked',
|
||||
'plex.empty_trash': 'blocked',
|
||||
'plex.clean_bundles': 'blocked',
|
||||
'plex.optimize_database': 'blocked',
|
||||
'plex.kill_session': 'blocked',
|
||||
'plex.user_management': 'blocked',
|
||||
'plex.force_refresh_all_metadata': 'blocked',
|
||||
'plex.mass_edit': 'blocked',
|
||||
'plex.unknown_mutation': 'blocked',
|
||||
'tautulli.terminate_session': 'blocked',
|
||||
'tautulli.delete_newsletter': 'blocked',
|
||||
'tautulli.delete_notification_log': 'blocked',
|
||||
'tautulli.delete_notifier': 'blocked',
|
||||
'tautulli.delete_recently_added': 'blocked',
|
||||
'tautulli.download_config': 'blocked',
|
||||
'tautulli.download_database': 'blocked',
|
||||
'tautulli.download_export': 'blocked',
|
||||
'tautulli.download_log': 'blocked',
|
||||
'tautulli.download_plex_log': 'blocked',
|
||||
'tautulli.notify': 'blocked',
|
||||
'tautulli.user_management': 'blocked',
|
||||
'tautulli.mass_edit': 'blocked',
|
||||
'tautulli.unknown_mutation': 'blocked',
|
||||
'npm.create_proxy_host': 'blocked',
|
||||
'npm.update_proxy_host': 'blocked',
|
||||
'npm.delete_proxy_host': 'blocked',
|
||||
'npm.enable_proxy_host': 'blocked',
|
||||
'npm.disable_proxy_host': 'blocked',
|
||||
'npm.renew_certificate': 'blocked',
|
||||
'npm.delete_certificate': 'blocked',
|
||||
'npm.modify_access_list': 'blocked',
|
||||
'npm.create_user': 'blocked',
|
||||
'npm.update_user': 'blocked',
|
||||
'npm.delete_user': 'blocked',
|
||||
'npm.mass_edit': 'blocked',
|
||||
'npm.unknown_mutation': 'blocked',
|
||||
'authentik.create_application': 'blocked',
|
||||
'authentik.update_application': 'blocked',
|
||||
'authentik.delete_application': 'blocked',
|
||||
'authentik.create_provider': 'blocked',
|
||||
'authentik.update_provider': 'blocked',
|
||||
'authentik.delete_provider': 'blocked',
|
||||
'authentik.create_outpost': 'blocked',
|
||||
'authentik.update_outpost': 'blocked',
|
||||
'authentik.delete_outpost': 'blocked',
|
||||
'authentik.create_policy': 'blocked',
|
||||
'authentik.update_policy': 'blocked',
|
||||
'authentik.delete_policy': 'blocked',
|
||||
'authentik.create_user': 'blocked',
|
||||
'authentik.update_user': 'blocked',
|
||||
'authentik.delete_user': 'blocked',
|
||||
'authentik.create_group': 'blocked',
|
||||
'authentik.update_group': 'blocked',
|
||||
'authentik.delete_group': 'blocked',
|
||||
'authentik.rotate_token': 'blocked',
|
||||
'authentik.force_sync': 'blocked',
|
||||
'authentik.mass_edit': 'blocked',
|
||||
'authentik.unknown_mutation': 'blocked',
|
||||
}
|
||||
|
||||
_DEFAULT_REASONS = {
|
||||
'privatemuse.comfyui.install_artifact': 'Installing a model artifact mutates the production ComfyUI model store and requires an explicit approved operator flow with pinned SHA-256 provenance.',
|
||||
'deployment.target.deploy_workmesh': 'WorkMesh first deployment is a high-impact production mutation and requires an explicit approved operator flow outside typed MCP automation.',
|
||||
'deployment.target.promote_workmesh': 'WorkMesh product promotion requires exact-revision controller preflight and a human approval bound to its digest.',
|
||||
'container.clear_logs': 'Clearing container logs is outside AppOps Gateway V1; log viewing and local download remain read-only.',
|
||||
'container.stop': 'Container stop is outside AppOps Gateway V1; use Portainer or Unraid directly after manual review.',
|
||||
'gateway.clear_cache': 'Clearing gateway cache is outside AppOps Gateway V1; use explicit refresh or restart procedures after review.',
|
||||
'jellyfin.scan_library': 'No live Jellyfin connector or executor is registered. Use the Jellyfin UI until a typed, audited connector is implemented.',
|
||||
'unraid.vm.stop': 'Unraid stop may force-stop after its graceful timeout; AppOps blocks that destructive fallback.',
|
||||
'unraid.vm.reboot': 'VM reboot remains blocked until guest-shutdown and recovery behavior is safely verifiable.',
|
||||
'unraid.vm.force_stop': 'Force-stop is destructive and blocked by default.',
|
||||
'unraid.vm.reset': 'VM reset is destructive and blocked by default.',
|
||||
}
|
||||
|
||||
|
||||
def evaluate_policy(
|
||||
session: Session,
|
||||
action: str,
|
||||
target: str,
|
||||
source: str,
|
||||
) -> dict:
|
||||
if action not in ACTIONS:
|
||||
return {
|
||||
'decision': 'blocked',
|
||||
'reason': 'Unknown action id.',
|
||||
'risk': ACTION_RISK.get(action, 'critical'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
if action in _DEFAULT_DECISIONS:
|
||||
reason = _DEFAULT_REASONS.get(action, 'Action uses default governance policy.')
|
||||
return {
|
||||
'decision': _DEFAULT_DECISIONS[action],
|
||||
'reason': reason,
|
||||
'risk': ACTION_RISK.get(action, 'critical'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
if action == 'container.restart' and source == 'voice' and target == 'widefrog':
|
||||
return {
|
||||
'decision': 'allowed',
|
||||
'reason': 'Widefrog restart is explicitly allow-listed for voice actions.',
|
||||
'risk': ACTION_RISK.get(action, 'medium'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
records = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == action) & (
|
||||
(Policy.target == target) | (Policy.target == '*')
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
# Prefer exact target over wildcard.
|
||||
exact_match = next((r for r in records if r.target == target), None)
|
||||
wildcard_match = next((r for r in records if r.target == '*'), None)
|
||||
policy = exact_match or wildcard_match
|
||||
|
||||
if not policy:
|
||||
# Safety-first defaults by scope.
|
||||
if action in {'deployment.apply', 'deployment.rollback'}:
|
||||
return {
|
||||
'decision': 'blocked',
|
||||
'reason': 'Production deployment requires an explicit exact-target deployment policy.',
|
||||
'risk': ACTION_RISK.get(action, 'high'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
if source == 'voice' and action.startswith('container.') and target in {'authentik', 'portainer'}:
|
||||
return {
|
||||
'decision': 'approval_required',
|
||||
'reason': 'High-risk service action via voice requires explicit approval by policy.',
|
||||
'risk': ACTION_RISK.get(action, 'high'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
if source == 'voice' and action.startswith('container.') and target in {'postgres', 'redis', 'database'}:
|
||||
return {
|
||||
'decision': 'blocked',
|
||||
'reason': 'Datastore/container restarts via voice are blocked by default.',
|
||||
'risk': ACTION_RISK.get(action, 'high'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
if source == 'voice' and action in {'unraid.mover.start'}:
|
||||
return {
|
||||
'decision': 'blocked',
|
||||
'reason': 'High-impact maintenance action is blocked for voice unless explicitly approved.',
|
||||
'risk': ACTION_RISK.get(action, 'high'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
if action == 'stack.redeploy':
|
||||
return {
|
||||
'decision': 'approval_required',
|
||||
'reason': 'Default safe policy for stack redeploy requires approval.',
|
||||
'risk': ACTION_RISK.get(action, 'high'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
return {
|
||||
'decision': 'approval_required' if source == 'voice' else 'allowed',
|
||||
'reason': 'No explicit policy found; using default safe policy.',
|
||||
'risk': ACTION_RISK.get(action, 'medium'),
|
||||
'policy_id': None,
|
||||
'scope_decision': None,
|
||||
}
|
||||
|
||||
decision = policy.voice if source == 'voice' else policy.manual
|
||||
return {
|
||||
'decision': decision,
|
||||
'reason': policy.reason or 'Policy match from database.',
|
||||
'risk': policy.risk,
|
||||
'policy_id': policy.id,
|
||||
'scope_decision': decision,
|
||||
}
|
||||
@@ -0,0 +1,815 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/portfolio"
|
||||
_LEGACY_REVISION = "e59bfb3484531883e5486ee9e10b663ed0f6764f"
|
||||
_RECOVERY_LIVE_REVISION = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
_RECOVERY_FAILED_REVISION = "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
_RECOVERY_FAILURE_MESSAGE = "Existing production containers require a valid healthy live deployment state"
|
||||
|
||||
_PROJECT: dict[str, object] = {
|
||||
"branch": "main",
|
||||
"slug": "portfolio",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/portfolio",
|
||||
"exact_source": True,
|
||||
"env_source": "/srv/example/appdata/Portfolio/.env",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"compose_project": "portfolio",
|
||||
"containers": ["Portfolio", "Portfolio-api"],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
"environment": {
|
||||
"GIT_COMMIT": "{revision}",
|
||||
"BUILD_ID": "build-{short_revision}",
|
||||
},
|
||||
}
|
||||
|
||||
_LEGACY_SINGLE_PROJECT: dict[str, object] = {
|
||||
"branch": "main",
|
||||
"slug": "portfolio",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/portfolio",
|
||||
"exact_source": True,
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"compose_project": "portfolio",
|
||||
"containers": ["portfolio"],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
}
|
||||
|
||||
_INSPECT_SCRIPT = r'''
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
repository = sys.argv[1]
|
||||
path = Path("/opt/gitea-deploy/projects.json")
|
||||
try:
|
||||
projects = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(20)
|
||||
current = projects.get(repository)
|
||||
if current is None:
|
||||
print(json.dumps({"present": False}, sort_keys=True, separators=(",", ":")))
|
||||
raise SystemExit(0)
|
||||
if not isinstance(current, dict):
|
||||
print(json.dumps({"present": True, "type": type(current).__name__}, sort_keys=True, separators=(",", ":")))
|
||||
raise SystemExit(0)
|
||||
|
||||
safe = {"present": True}
|
||||
for key in (
|
||||
"branch",
|
||||
"slug",
|
||||
"source_dir",
|
||||
"exact_source",
|
||||
"env_source",
|
||||
"compose_files",
|
||||
"compose_project",
|
||||
"containers",
|
||||
"verify_oci_revision",
|
||||
"health_timeout",
|
||||
"deploy_script",
|
||||
"archive_deploy_script",
|
||||
"manual",
|
||||
):
|
||||
if key in current:
|
||||
safe[key] = current[key]
|
||||
environment = current.get("environment")
|
||||
if isinstance(environment, dict):
|
||||
safe["environment_keys"] = sorted(str(key) for key in environment.keys())
|
||||
live_probe = current.get("live_probe")
|
||||
if isinstance(live_probe, dict):
|
||||
safe["live_probe"] = {
|
||||
key: live_probe[key]
|
||||
for key in ("type", "container", "port", "path")
|
||||
if key in live_probe
|
||||
}
|
||||
|
||||
state_path = Path("/mnt/cache/appdata/gitea-deploy-runners/state/portfolio.json")
|
||||
if state_path.is_file() and not state_path.is_symlink():
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
state = None
|
||||
if isinstance(state, dict):
|
||||
safe_state = {
|
||||
key: state[key]
|
||||
for key in (
|
||||
"schema_version",
|
||||
"repository",
|
||||
"revision",
|
||||
"live_revision",
|
||||
"live_status",
|
||||
"live_legacy_provenance",
|
||||
"status",
|
||||
"updated_at_epoch",
|
||||
)
|
||||
if key in state
|
||||
}
|
||||
last_attempt = state.get("last_attempt")
|
||||
if isinstance(last_attempt, dict):
|
||||
safe_attempt = {
|
||||
key: last_attempt[key]
|
||||
for key in ("revision", "status", "updated_at_epoch")
|
||||
if key in last_attempt
|
||||
}
|
||||
message = last_attempt.get("message")
|
||||
if isinstance(message, str):
|
||||
safe_attempt["message"] = message.replace("\r", " ").replace("\n", " ")[:768]
|
||||
safe_state["last_attempt"] = safe_attempt
|
||||
live_artifacts = state.get("live_artifacts")
|
||||
if isinstance(live_artifacts, dict):
|
||||
containers = live_artifacts.get("containers")
|
||||
if isinstance(containers, dict):
|
||||
safe_state["live_artifact_containers"] = sorted(str(key) for key in containers.keys())
|
||||
safe["state"] = safe_state
|
||||
print(json.dumps(safe, sort_keys=True, separators=(",", ":")))
|
||||
'''
|
||||
|
||||
_ENROLL_SCRIPT = r'''
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
state_path = state_root / "portfolio.json"
|
||||
project_lock_path = state_root / "locks" / "portfolio.lock"
|
||||
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
(
|
||||
repository,
|
||||
legacy_revision,
|
||||
recovery_live_revision,
|
||||
recovery_failed_revision,
|
||||
recovery_failure_message,
|
||||
raw_project,
|
||||
raw_legacy_project,
|
||||
) = sys.argv[1:8]
|
||||
project = json.loads(raw_project)
|
||||
legacy_project = json.loads(raw_legacy_project)
|
||||
|
||||
expected_project = {
|
||||
"branch": "main",
|
||||
"slug": "portfolio",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/portfolio",
|
||||
"exact_source": True,
|
||||
"env_source": "/srv/example/appdata/Portfolio/.env",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"compose_project": "portfolio",
|
||||
"containers": ["Portfolio", "Portfolio-api"],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
"environment": {
|
||||
"GIT_COMMIT": "{revision}",
|
||||
"BUILD_ID": "build-{short_revision}",
|
||||
},
|
||||
}
|
||||
expected_legacy_project = {
|
||||
"branch": "main",
|
||||
"slug": "portfolio",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/portfolio",
|
||||
"exact_source": True,
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"compose_project": "portfolio",
|
||||
"containers": ["portfolio"],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
}
|
||||
expected_legacy_revision = "e59bfb3484531883e5486ee9e10b663ed0f6764f"
|
||||
expected_recovery_live_revision = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
expected_recovery_failed_revision = "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
expected_recovery_failure_message = "Existing production containers require a valid healthy live deployment state"
|
||||
|
||||
if (
|
||||
repository != "jens/portfolio"
|
||||
or legacy_revision != expected_legacy_revision
|
||||
or recovery_live_revision != expected_recovery_live_revision
|
||||
or recovery_failed_revision != expected_recovery_failed_revision
|
||||
or recovery_failure_message != expected_recovery_failure_message
|
||||
or project != expected_project
|
||||
or legacy_project != expected_legacy_project
|
||||
):
|
||||
raise SystemExit(40)
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(20)
|
||||
registry_stat = registry_path.stat()
|
||||
if not stat.S_ISREG(registry_stat.st_mode):
|
||||
raise SystemExit(21)
|
||||
try:
|
||||
if (
|
||||
host_registry_path.is_symlink()
|
||||
or not host_registry_path.is_file()
|
||||
or not os.path.samefile(registry_path, host_registry_path)
|
||||
):
|
||||
raise SystemExit(36)
|
||||
except OSError:
|
||||
raise SystemExit(36)
|
||||
|
||||
env_source = Path(str(project["env_source"]))
|
||||
if env_source.is_symlink() or not env_source.is_file():
|
||||
raise SystemExit(41)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(24)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
for revision in (legacy_revision, recovery_live_revision, recovery_failed_revision):
|
||||
resolved = git_run("rev-parse", f"{revision}^{{commit}}")
|
||||
if resolved.returncode != 0 or resolved.stdout.strip() != revision:
|
||||
raise SystemExit(25)
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
main_revision = main.stdout.strip()
|
||||
if main.returncode != 0 or len(main_revision) != 40:
|
||||
raise SystemExit(26)
|
||||
if git_run("merge-base", "--is-ancestor", legacy_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(27)
|
||||
if git_run("merge-base", "--is-ancestor", recovery_live_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(27)
|
||||
if main_revision != recovery_failed_revision:
|
||||
raise SystemExit(58)
|
||||
for compose_file in project["compose_files"]:
|
||||
if git_run("cat-file", "-e", f"{main_revision}:{compose_file}").returncode != 0:
|
||||
raise SystemExit(28)
|
||||
|
||||
try:
|
||||
if state_root.is_symlink() or not state_root.is_dir():
|
||||
raise SystemExit(50)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if project_lock_path.parent.is_symlink() or project_lock_path.is_symlink():
|
||||
raise SystemExit(29)
|
||||
project_lock_handle = project_lock_path.open("a+", encoding="utf-8")
|
||||
lock_handle = lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
with project_lock_handle as project_lock, lock_handle as lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
try:
|
||||
projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
if not isinstance(projects, dict):
|
||||
raise SystemExit(22)
|
||||
|
||||
current = projects.get(repository)
|
||||
|
||||
def legacy_script_target_matches(value):
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
expected_keys = {
|
||||
"branch",
|
||||
"slug",
|
||||
"source_dir",
|
||||
"env_source",
|
||||
"compose_files",
|
||||
"compose_project",
|
||||
"containers",
|
||||
"deploy_script",
|
||||
"environment",
|
||||
}
|
||||
if set(value) != expected_keys:
|
||||
return False
|
||||
environment = value.get("environment")
|
||||
return (
|
||||
value.get("branch") == "main"
|
||||
and value.get("slug") == "portfolio"
|
||||
and value.get("source_dir") == "/srv/example/appdata/Portfolio/source"
|
||||
and value.get("env_source") == "/srv/example/appdata/Portfolio/.env"
|
||||
and value.get("compose_files") == ["docker-compose.yml"]
|
||||
and value.get("compose_project") == "portfolio"
|
||||
and value.get("containers") == ["Portfolio", "Portfolio-api"]
|
||||
and value.get("deploy_script") == "scripts/deploy-unraid.sh"
|
||||
and isinstance(environment, dict)
|
||||
and set(environment) == {"PORTFOLIO_APP_DIR"}
|
||||
and environment.get("PORTFOLIO_APP_DIR")
|
||||
in {"/srv/example/appdata/Portfolio", "/srv/example/appdata/portfolio"}
|
||||
)
|
||||
|
||||
if current not in (None, project, legacy_project) and not legacy_script_target_matches(current):
|
||||
raise SystemExit(23)
|
||||
|
||||
state_requires_reconcile = False
|
||||
runtime_state_requires_reconcile = False
|
||||
existing_state = None
|
||||
if state_path.exists():
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(42)
|
||||
try:
|
||||
existing_state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(42)
|
||||
if not isinstance(existing_state, dict):
|
||||
raise SystemExit(42)
|
||||
if existing_state.get("schema_version") == 3:
|
||||
if existing_state.get("repository") != repository:
|
||||
raise SystemExit(43)
|
||||
if (
|
||||
current == project
|
||||
and existing_state.get("live_status") == "healthy"
|
||||
and isinstance(existing_state.get("live_revision"), str)
|
||||
and len(existing_state.get("live_revision")) == 40
|
||||
and (
|
||||
existing_state.get("live_legacy_provenance") is True
|
||||
or isinstance(existing_state.get("live_receipt"), dict)
|
||||
)
|
||||
):
|
||||
print("already-enrolled", flush=True)
|
||||
raise SystemExit(0)
|
||||
|
||||
last_attempt = existing_state.get("last_attempt")
|
||||
live_artifacts = existing_state.get("live_artifacts")
|
||||
live_receipt = existing_state.get("live_receipt")
|
||||
artifact_containers = (
|
||||
live_artifacts.get("containers")
|
||||
if isinstance(live_artifacts, dict)
|
||||
else None
|
||||
)
|
||||
if (
|
||||
current == project
|
||||
and existing_state.get("revision") == recovery_live_revision
|
||||
and existing_state.get("live_revision") == recovery_live_revision
|
||||
and existing_state.get("live_status") == "unknown"
|
||||
and existing_state.get("live_legacy_provenance") is False
|
||||
and existing_state.get("status") == "failed"
|
||||
and isinstance(last_attempt, dict)
|
||||
and last_attempt.get("revision") == recovery_failed_revision
|
||||
and last_attempt.get("status") == "failed"
|
||||
and last_attempt.get("message") == recovery_failure_message
|
||||
and isinstance(live_receipt, dict)
|
||||
and bool(live_receipt)
|
||||
and isinstance(artifact_containers, dict)
|
||||
and set(artifact_containers) == set(project["containers"])
|
||||
):
|
||||
runtime_state_requires_reconcile = True
|
||||
elif legacy_script_target_matches(current):
|
||||
state_requires_reconcile = True
|
||||
else:
|
||||
raise SystemExit(43)
|
||||
elif not (
|
||||
existing_state.get("repository") == repository
|
||||
and existing_state.get("status") == "success"
|
||||
and existing_state.get("revision") == legacy_revision
|
||||
and "schema_version" not in existing_state
|
||||
and "live_receipt" not in existing_state
|
||||
and "live_legacy_provenance" not in existing_state
|
||||
):
|
||||
if legacy_script_target_matches(current):
|
||||
state_requires_reconcile = True
|
||||
else:
|
||||
raise SystemExit(43)
|
||||
|
||||
expected_names = set(project["containers"])
|
||||
observed_result = subprocess.run(
|
||||
[
|
||||
"docker", "ps", "--all", "--filter",
|
||||
f"label=com.docker.compose.project={project['compose_project']}",
|
||||
"--format", "{{.Names}}",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
if observed_result.returncode != 0:
|
||||
raise SystemExit(44)
|
||||
observed_names = {line.strip() for line in observed_result.stdout.splitlines() if line.strip()}
|
||||
if observed_names != expected_names:
|
||||
raise SystemExit(45)
|
||||
|
||||
for required in sorted(expected_names):
|
||||
inspected = subprocess.run(
|
||||
["docker", "inspect", required],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(46)
|
||||
try:
|
||||
container = json.loads(inspected.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(46)
|
||||
runtime = dict(container.get("State") or {})
|
||||
health = str(dict(runtime.get("Health") or {}).get("Status") or "none")
|
||||
config = dict(container.get("Config") or {})
|
||||
labels = dict(config.get("Labels") or {})
|
||||
revision_label = str(labels.get("tech.itworx.commit") or "")
|
||||
build_label = str(labels.get("tech.itworx.build-id") or "")
|
||||
if (
|
||||
not runtime.get("Running")
|
||||
or health in {"starting", "unhealthy"}
|
||||
or labels.get("com.docker.compose.project") != "portfolio"
|
||||
):
|
||||
raise SystemExit(46)
|
||||
if legacy_script_target_matches(current) and revision_label != legacy_revision:
|
||||
raise SystemExit(47)
|
||||
if runtime_state_requires_reconcile:
|
||||
if (
|
||||
revision_label != recovery_live_revision
|
||||
or build_label != f"build-{recovery_live_revision[:12]}"
|
||||
):
|
||||
raise SystemExit(55)
|
||||
artifacts = dict(dict(existing_state.get("live_artifacts") or {}).get("containers") or {})
|
||||
artifact = artifacts.get(required)
|
||||
if not isinstance(artifact, dict):
|
||||
raise SystemExit(55)
|
||||
if (
|
||||
str(artifact.get("image_id") or "") != str(container.get("Image") or "")
|
||||
or str(artifact.get("container_id") or "") != str(container.get("Id") or "")
|
||||
):
|
||||
raise SystemExit(55)
|
||||
|
||||
if runtime_state_requires_reconcile:
|
||||
if state_root.is_symlink() or not state_root.is_dir():
|
||||
raise SystemExit(50)
|
||||
now = int(time.time())
|
||||
state_backup = state_path.with_name(
|
||||
f"portfolio.json.runtime-reconcile-backup-{now}-{os.getpid()}"
|
||||
)
|
||||
temporary_state = state_path.with_name(
|
||||
f".{state_path.name}.{os.getpid()}.runtime-reconcile.tmp"
|
||||
)
|
||||
reconciled_state = dict(existing_state)
|
||||
reconciled_state["live_status"] = "healthy"
|
||||
reconciled_state["status"] = "success"
|
||||
reconciled_state["message"] = (
|
||||
"Verified Portfolio runtime matches recorded live artifacts; controller state reconciled"
|
||||
)
|
||||
reconciled_state["updated_at_epoch"] = now
|
||||
reconciled_state["last_attempt"] = {
|
||||
"revision": recovery_live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified live Portfolio runtime state reconciliation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {"reconciled_failed_revision": recovery_failed_revision},
|
||||
}
|
||||
try:
|
||||
mode = stat.S_IMODE(state_path.stat().st_mode)
|
||||
shutil.copy2(state_path, state_backup)
|
||||
with temporary_state.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(reconciled_state, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary_state, mode)
|
||||
os.replace(temporary_state, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary_state.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(56)
|
||||
try:
|
||||
verify = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(57)
|
||||
if not (
|
||||
isinstance(verify, dict)
|
||||
and verify.get("live_revision") == recovery_live_revision
|
||||
and verify.get("live_status") == "healthy"
|
||||
and verify.get("live_legacy_provenance") is False
|
||||
and isinstance(verify.get("live_receipt"), dict)
|
||||
and bool(verify.get("live_receipt"))
|
||||
and verify.get("status") == "success"
|
||||
):
|
||||
raise SystemExit(57)
|
||||
print("runtime-state-reconciled", flush=True)
|
||||
raise SystemExit(0)
|
||||
|
||||
if state_requires_reconcile:
|
||||
if state_root.is_symlink() or not state_root.is_dir():
|
||||
raise SystemExit(50)
|
||||
state_backup = state_path.with_name(
|
||||
f"portfolio.json.verified-live-backup-{int(time.time())}-{os.getpid()}"
|
||||
)
|
||||
temporary_state = state_path.with_name(f".{state_path.name}.{os.getpid()}.reconcile.tmp")
|
||||
try:
|
||||
shutil.copy2(state_path, state_backup)
|
||||
reconciled_state = {
|
||||
"repository": repository,
|
||||
"revision": legacy_revision,
|
||||
"status": "success",
|
||||
}
|
||||
with temporary_state.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(reconciled_state, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary_state, 0o600)
|
||||
os.replace(temporary_state, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
existing_state = reconciled_state
|
||||
except Exception:
|
||||
try:
|
||||
temporary_state.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(54)
|
||||
|
||||
if current != project:
|
||||
backup = host_registry_path.with_name(
|
||||
f"projects.json.portfolio-backup-{int(time.time())}-{os.getpid()}"
|
||||
)
|
||||
try:
|
||||
shutil.copy2(host_registry_path, backup)
|
||||
except Exception:
|
||||
raise SystemExit(32)
|
||||
updated = dict(projects)
|
||||
updated[repository] = project
|
||||
temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
|
||||
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
|
||||
try:
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.S_IMODE(registry_stat.st_mode))
|
||||
os.replace(temporary, host_registry_path)
|
||||
directory_fd = os.open(str(host_registry_path.parent), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(34)
|
||||
|
||||
if existing_state is None:
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
if state_root.is_symlink() or not state_root.is_dir():
|
||||
raise SystemExit(50)
|
||||
temporary_state = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
|
||||
payload = {"repository": repository, "revision": legacy_revision, "status": "success"}
|
||||
try:
|
||||
with temporary_state.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary_state, 0o600)
|
||||
os.replace(temporary_state, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary_state.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(51)
|
||||
|
||||
try:
|
||||
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
||||
if controller_path.is_symlink() or not controller_path.is_file():
|
||||
raise RuntimeError("controller module is unavailable")
|
||||
controller_spec = importlib.util.spec_from_file_location(
|
||||
"appops_locked_portfolio_bootstrap", controller_path
|
||||
)
|
||||
if controller_spec is None or controller_spec.loader is None:
|
||||
raise RuntimeError("controller module cannot be loaded")
|
||||
controller = importlib.util.module_from_spec(controller_spec)
|
||||
controller_spec.loader.exec_module(controller)
|
||||
controller.bootstrap_legacy_state(repository, project, legacy_revision)
|
||||
except Exception:
|
||||
raise SystemExit(52)
|
||||
|
||||
try:
|
||||
final_state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(53)
|
||||
if not (
|
||||
isinstance(final_state, dict)
|
||||
and final_state.get("schema_version") == 3
|
||||
and final_state.get("repository") == repository
|
||||
and final_state.get("live_revision") == legacy_revision
|
||||
and final_state.get("live_status") == "healthy"
|
||||
and final_state.get("live_legacy_provenance") is True
|
||||
):
|
||||
raise SystemExit(53)
|
||||
|
||||
print("enrolled", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class PortfolioDeploymentEnrollment:
|
||||
"""Enroll or reconcile the fixed paired Portfolio exact-source controller target."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def _inspect_current_target(self, *, endpoint: str, container_id: str) -> dict[str, Any] | None:
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": True,
|
||||
"AttachStderr": False,
|
||||
"Tty": True,
|
||||
"Cmd": ["python3", "-c", _INSPECT_SCRIPT, _REPOSITORY],
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return None
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
response = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": False, "Tty": True},
|
||||
)
|
||||
raw = (response.text or "").strip()
|
||||
if not raw or len(raw) > 4096:
|
||||
return None
|
||||
decoded = json.loads(raw)
|
||||
return decoded if isinstance(decoded, dict) else None
|
||||
except (RuntimeError, ValueError, TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": redact_message(
|
||||
str(status.get("reason") or status.get("status") or "unavailable")
|
||||
),
|
||||
}
|
||||
controller = dict(status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": "deployment controller is not running",
|
||||
}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not container_id:
|
||||
return {
|
||||
"status": "connector_unavailable",
|
||||
"reason": "Portainer deployment control is unavailable",
|
||||
}
|
||||
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
current_target = self._inspect_current_target(endpoint=endpoint, container_id=container_id)
|
||||
if current_target is not None:
|
||||
container_revisions: dict[str, str | None] = {}
|
||||
for required in ("Portfolio", "Portfolio-api"):
|
||||
try:
|
||||
inspected_container = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{quote(required, safe='')}/json",
|
||||
).json()
|
||||
config_payload = inspected_container.get("Config") if isinstance(inspected_container, dict) else None
|
||||
labels = config_payload.get("Labels") if isinstance(config_payload, dict) else None
|
||||
revision = labels.get("tech.itworx.commit") if isinstance(labels, dict) else None
|
||||
container_revisions[required] = str(revision) if revision else None
|
||||
except (RuntimeError, ValueError, TypeError):
|
||||
container_revisions[required] = None
|
||||
current_target["container_revisions"] = container_revisions
|
||||
command = [
|
||||
"python3",
|
||||
"-c",
|
||||
_ENROLL_SCRIPT,
|
||||
_REPOSITORY,
|
||||
_LEGACY_REVISION,
|
||||
_RECOVERY_LIVE_REVISION,
|
||||
_RECOVERY_FAILED_REVISION,
|
||||
_RECOVERY_FAILURE_MESSAGE,
|
||||
json.dumps(_PROJECT, separators=(",", ":"), sort_keys=True),
|
||||
json.dumps(_LEGACY_SINGLE_PROJECT, separators=(",", ":"), sort_keys=True),
|
||||
]
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": command,
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "Portfolio enrollment could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 120
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "Portfolio enrollment timed out"}
|
||||
reasons = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
21: "deployment projects registry is not a regular file",
|
||||
22: "deployment projects registry does not contain an object",
|
||||
23: "Portfolio already has an unexpected deployment target",
|
||||
24: "mounted Portfolio bare repository is unavailable",
|
||||
25: "known Portfolio deployment revisions cannot be resolved",
|
||||
26: "Portfolio main cannot be resolved",
|
||||
27: "known Portfolio live revision is not an ancestor of main",
|
||||
28: "required Portfolio docker-compose.yml is absent from main",
|
||||
29: "deployment registry lock file cannot be opened",
|
||||
30: "deployment registry lock cannot be acquired",
|
||||
31: "deployment projects registry cannot be read as JSON",
|
||||
32: "deployment projects registry backup cannot be created",
|
||||
34: "deployment projects registry replacement failed",
|
||||
36: "deployment projects registry host alias is unavailable or mismatched",
|
||||
40: "Portfolio fixed enrollment contract was altered",
|
||||
41: "Portfolio production environment file is unavailable or unsafe",
|
||||
42: "Portfolio deployment state file is unavailable or invalid",
|
||||
43: "Portfolio deployment state conflicts with the verified live baseline",
|
||||
44: "Portfolio Compose project inventory could not be read",
|
||||
45: "Portfolio Compose project does not contain exactly the expected web/API containers",
|
||||
46: "required Portfolio production container is missing or unhealthy",
|
||||
47: "live Portfolio container revision does not match the verified legacy baseline",
|
||||
50: "deployment state directory is unavailable or unsafe",
|
||||
51: "verified Portfolio legacy state could not be written durably",
|
||||
52: "deployment controller rejected Portfolio legacy-state bootstrap",
|
||||
53: "Portfolio legacy-state bootstrap did not produce the expected managed state",
|
||||
54: "verified stale Portfolio controller state could not be reconciled safely",
|
||||
55: "live Portfolio runtime no longer matches the controller-recorded modern artifacts",
|
||||
56: "verified Portfolio runtime state could not be reconciled durably",
|
||||
57: "reconciled Portfolio runtime state did not verify after write",
|
||||
58: "Portfolio main changed after the bounded recovery contract was recorded",
|
||||
}
|
||||
if exit_code != 0:
|
||||
reason = reasons.get(
|
||||
exit_code,
|
||||
"Portfolio enrollment failed closed before any application deployment",
|
||||
)
|
||||
if exit_code in {23, 43, 47, 55, 58} and current_target is not None:
|
||||
reason = redact_message(
|
||||
f"{reason}: {json.dumps(current_target, sort_keys=True, separators=(',', ':'))}"
|
||||
)
|
||||
return {"status": "validation_failed", "reason": reason}
|
||||
return {
|
||||
"status": "enrolled",
|
||||
"repository": _REPOSITORY,
|
||||
"legacy_revision": _LEGACY_REVISION,
|
||||
"compose_project": _PROJECT["compose_project"],
|
||||
"containers": list(_PROJECT["containers"]),
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
from .portfolio_deployment_enrollment import PortfolioDeploymentEnrollment
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/portfolio"
|
||||
_LIVE_REVISION = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
_FAILED_REVISION = "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
_FAILURE_MESSAGE = "Existing production containers require a valid healthy live deployment state"
|
||||
|
||||
# One-time, fail-closed re-attestation for the exact observed Portfolio state.
|
||||
# The recovery never builds, restarts, recreates or stops application containers.
|
||||
# It uses the controller's own receipt/runtime verifier and accepts only a change
|
||||
# in ephemeral container IDs while immutable image/provenance evidence remains
|
||||
# exactly equal to the previously recorded verified receipt.
|
||||
_REATTEST_SCRIPT = r'''
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
repository, live_revision, failed_revision, failure_message = sys.argv[1:5]
|
||||
expected_repository = "jens/portfolio"
|
||||
expected_live_revision = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
expected_failed_revision = "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
expected_failure_message = "Existing production containers require a valid healthy live deployment state"
|
||||
if (
|
||||
repository != expected_repository
|
||||
or live_revision != expected_live_revision
|
||||
or failed_revision != expected_failed_revision
|
||||
or failure_message != expected_failure_message
|
||||
):
|
||||
raise SystemExit(40)
|
||||
|
||||
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
state_path = state_root / "portfolio.json"
|
||||
project_lock_path = state_root / "locks" / "portfolio.lock"
|
||||
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
|
||||
if controller_path.is_symlink() or not controller_path.is_file():
|
||||
raise SystemExit(20)
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(21)
|
||||
try:
|
||||
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(22)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
if not isinstance(project, dict):
|
||||
raise SystemExit(23)
|
||||
if (
|
||||
project.get("branch") != "main"
|
||||
or project.get("slug") != "portfolio"
|
||||
or project.get("compose_project") != "portfolio"
|
||||
or project.get("containers") != ["Portfolio", "Portfolio-api"]
|
||||
or project.get("compose_files") != ["docker-compose.yml"]
|
||||
or project.get("exact_source") is not True
|
||||
or project.get("verify_oci_revision") is not False
|
||||
):
|
||||
raise SystemExit(24)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(25)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
for revision in (live_revision, failed_revision):
|
||||
resolved = git_run("rev-parse", f"{revision}^{{commit}}")
|
||||
if resolved.returncode != 0 or resolved.stdout.strip() != revision:
|
||||
raise SystemExit(26)
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
if main.returncode != 0 or main.stdout.strip() != failed_revision:
|
||||
raise SystemExit(27)
|
||||
if git_run("merge-base", "--is-ancestor", live_revision, failed_revision).returncode != 0:
|
||||
raise SystemExit(28)
|
||||
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("itworx_deploy_controller", controller_path)
|
||||
controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None
|
||||
if controller is None or spec is None or spec.loader is None:
|
||||
raise RuntimeError("controller import unavailable")
|
||||
spec.loader.exec_module(controller)
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
||||
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
with project_lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
with global_lock:
|
||||
try:
|
||||
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
|
||||
try:
|
||||
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(32)
|
||||
if current_projects.get(repository) != project:
|
||||
raise SystemExit(33)
|
||||
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(34)
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(34)
|
||||
if not isinstance(state, dict):
|
||||
raise SystemExit(34)
|
||||
attempt = state.get("last_attempt")
|
||||
receipt = state.get("live_receipt")
|
||||
artifacts = state.get("live_artifacts")
|
||||
if not (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == repository
|
||||
and state.get("revision") == live_revision
|
||||
and state.get("live_revision") == live_revision
|
||||
and state.get("live_status") == "unknown"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
and state.get("status") == "failed"
|
||||
and isinstance(attempt, dict)
|
||||
and attempt.get("revision") == failed_revision
|
||||
and attempt.get("status") == "failed"
|
||||
and attempt.get("message") == failure_message
|
||||
and isinstance(receipt, dict)
|
||||
and bool(receipt)
|
||||
and isinstance(artifacts, dict)
|
||||
):
|
||||
raise SystemExit(35)
|
||||
|
||||
project = dict(project)
|
||||
project["_repository_name"] = repository
|
||||
project["_repository_path"] = str(bare_repo)
|
||||
|
||||
try:
|
||||
build = controller.validate_recorded_receipt(project, receipt, repository, live_revision)
|
||||
recorded_receipt_containers = receipt.get("containers")
|
||||
if not isinstance(recorded_receipt_containers, dict):
|
||||
raise RuntimeError("receipt containers missing")
|
||||
expected_image_ids = {
|
||||
str(container_name): str(dict(raw).get("image_id") or "")
|
||||
for container_name, raw in recorded_receipt_containers.items()
|
||||
if isinstance(raw, dict)
|
||||
}
|
||||
if set(expected_image_ids) != {"Portfolio", "Portfolio-api"} or not all(expected_image_ids.values()):
|
||||
raise RuntimeError("receipt image IDs missing")
|
||||
observed_receipt = controller.verify_deployment(
|
||||
project,
|
||||
build,
|
||||
expected_image_ids=expected_image_ids,
|
||||
)
|
||||
observed_artifacts = controller.deployment_artifacts(project)
|
||||
except Exception:
|
||||
raise SystemExit(36)
|
||||
|
||||
immutable_receipt_keys = ("schema", "repository", "revision", "tree", "treeDigest", "build", "live")
|
||||
if any(observed_receipt.get(key) != receipt.get(key) for key in immutable_receipt_keys):
|
||||
raise SystemExit(37)
|
||||
observed_receipt_containers = observed_receipt.get("containers")
|
||||
if not isinstance(observed_receipt_containers, dict) or set(observed_receipt_containers) != set(recorded_receipt_containers):
|
||||
raise SystemExit(37)
|
||||
|
||||
container_id_changed = False
|
||||
for container_name in sorted(recorded_receipt_containers):
|
||||
old = dict(recorded_receipt_containers[container_name])
|
||||
new = dict(observed_receipt_containers[container_name])
|
||||
if old.get("container_id") != new.get("container_id"):
|
||||
container_id_changed = True
|
||||
old_without_id = {key: value for key, value in old.items() if key != "container_id"}
|
||||
new_without_id = {key: value for key, value in new.items() if key != "container_id"}
|
||||
if old_without_id != new_without_id:
|
||||
raise SystemExit(38)
|
||||
if not container_id_changed:
|
||||
raise SystemExit(39)
|
||||
|
||||
recorded_artifact_containers = artifacts.get("containers")
|
||||
observed_artifact_containers = observed_artifacts.get("containers")
|
||||
if (
|
||||
not isinstance(recorded_artifact_containers, dict)
|
||||
or not isinstance(observed_artifact_containers, dict)
|
||||
or set(recorded_artifact_containers) != set(observed_artifact_containers)
|
||||
or artifacts.get("compose_sha256") != observed_artifacts.get("compose_sha256")
|
||||
):
|
||||
raise SystemExit(41)
|
||||
for container_name in sorted(recorded_artifact_containers):
|
||||
old = dict(recorded_artifact_containers[container_name])
|
||||
new = dict(observed_artifact_containers[container_name])
|
||||
old_without_id = {key: value for key, value in old.items() if key != "container_id"}
|
||||
new_without_id = {key: value for key, value in new.items() if key != "container_id"}
|
||||
if old_without_id != new_without_id:
|
||||
raise SystemExit(41)
|
||||
|
||||
now = int(time.time())
|
||||
repaired = dict(state)
|
||||
repaired["live_status"] = "healthy"
|
||||
repaired["live_artifacts"] = observed_artifacts
|
||||
repaired["artifacts"] = observed_artifacts
|
||||
repaired["live_receipt"] = observed_receipt
|
||||
repaired["status"] = "success"
|
||||
repaired["message"] = "Verified Portfolio runtime re-attested after container recreation"
|
||||
repaired["updated_at_epoch"] = now
|
||||
repaired["last_attempt"] = {
|
||||
"revision": live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified live Portfolio runtime receipt re-attestation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {
|
||||
"reconciled_failed_revision": failed_revision,
|
||||
"container_identity_only": True,
|
||||
},
|
||||
}
|
||||
|
||||
backup = state_path.with_name(f"portfolio.json.reattest-backup-{now}-{os.getpid()}")
|
||||
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.reattest.tmp")
|
||||
try:
|
||||
mode = stat.S_IMODE(state_path.stat().st_mode)
|
||||
shutil.copy2(state_path, backup)
|
||||
with temporary.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(repaired, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, mode)
|
||||
os.replace(temporary, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(42)
|
||||
|
||||
try:
|
||||
verified = controller.verified_current_receipt(
|
||||
repository,
|
||||
project,
|
||||
live_revision,
|
||||
str(receipt.get("tree") or ""),
|
||||
)
|
||||
except Exception:
|
||||
verified = None
|
||||
if not isinstance(verified, dict) or verified != observed_receipt:
|
||||
raise SystemExit(43)
|
||||
|
||||
print("re-attested", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class PortfolioDeploymentRecovery:
|
||||
"""Recover the exact known Portfolio state drift, otherwise use normal enrollment."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
controller = dict(status.get("container") or {}) if status.get("status") == "ok" else {}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running" or not container_id or not config:
|
||||
return PortfolioDeploymentEnrollment(portainer=self.portainer).run()
|
||||
|
||||
command = [
|
||||
"python3",
|
||||
"-c",
|
||||
_REATTEST_SCRIPT,
|
||||
_REPOSITORY,
|
||||
_LIVE_REVISION,
|
||||
_FAILED_REVISION,
|
||||
_FAILURE_MESSAGE,
|
||||
]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "Portfolio runtime re-attestation could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
deadline = time.monotonic() + 45
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "Portfolio runtime re-attestation timed out"}
|
||||
if exit_code == 0:
|
||||
return {
|
||||
"status": "reconciled",
|
||||
"repository": _REPOSITORY,
|
||||
"live_revision": _LIVE_REVISION,
|
||||
"failed_revision": _FAILED_REVISION,
|
||||
"containers": ["Portfolio", "Portfolio-api"],
|
||||
"compose_project": "portfolio",
|
||||
}
|
||||
|
||||
# Exit 35 means the exact recoverable state is no longer present. In
|
||||
# normal/healthy states the established enrollment routine remains the
|
||||
# authoritative path. Every other re-attestation failure is fail-closed.
|
||||
if exit_code == 35:
|
||||
return PortfolioDeploymentEnrollment(portainer=self.portainer).run()
|
||||
|
||||
reasons = {
|
||||
20: "deployment controller executable is unavailable or unsafe",
|
||||
21: "deployment projects registry is unavailable or unsafe",
|
||||
22: "deployment projects registry cannot be read",
|
||||
23: "Portfolio deployment target is missing",
|
||||
24: "Portfolio deployment target differs from the bounded paired contract",
|
||||
25: "mounted Portfolio bare repository is unavailable",
|
||||
26: "known Portfolio recovery revisions cannot be resolved",
|
||||
27: "Portfolio main changed after the bounded recovery contract was recorded",
|
||||
28: "known Portfolio live revision is not an ancestor of the failed revision",
|
||||
29: "deployment controller verifier cannot be loaded",
|
||||
30: "deployment lock files cannot be opened",
|
||||
31: "deployment locks cannot be acquired",
|
||||
32: "deployment projects registry changed or became unreadable while waiting",
|
||||
33: "Portfolio deployment target changed while recovery was waiting",
|
||||
34: "Portfolio deployment state is unavailable or invalid",
|
||||
36: "controller could not re-attest the current Portfolio runtime against recorded immutable image IDs",
|
||||
37: "Portfolio receipt identity changed during re-attestation",
|
||||
38: "Portfolio immutable container/image provenance differs from the recorded receipt",
|
||||
39: "Portfolio container identities did not change; the known recovery cause is absent",
|
||||
40: "fixed Portfolio re-attestation contract was altered",
|
||||
41: "Portfolio recorded live artifacts differ beyond ephemeral container IDs",
|
||||
42: "re-attested Portfolio deployment state could not be written durably",
|
||||
43: "controller rejected the repaired Portfolio receipt after write",
|
||||
}
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": reasons.get(exit_code, "Portfolio runtime re-attestation failed closed"),
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
NO_ROLLBACK_ACTIONS = {
|
||||
'container.clear_logs',
|
||||
'container.stop',
|
||||
'gateway.clear_cache',
|
||||
'unraid.mover.start',
|
||||
'volume.delete',
|
||||
}
|
||||
|
||||
|
||||
def build_preflight(action: str, target: str, policy_decision: str, risk: str):
|
||||
dependencies = ['connector']
|
||||
if action.startswith('container.'):
|
||||
dependencies.append('container runtime')
|
||||
if action.startswith('sonarr.') or action.startswith('radarr.'):
|
||||
dependencies.append('media indexer api')
|
||||
|
||||
return {
|
||||
'action': action,
|
||||
'target': target,
|
||||
'risk': risk,
|
||||
'policyDecision': policy_decision,
|
||||
'affectedApps': [target],
|
||||
'dependencies': dependencies,
|
||||
'preChecks': ['policy evaluate', 'connector config', 'maintenance status'],
|
||||
'postChecks': ['connector state', 'service availability', 'policy audit'],
|
||||
'rollbackAvailable': action not in NO_ROLLBACK_ACTIONS,
|
||||
'auditPreview': f'Action {action} on {target} is prepared for execution review.',
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import ssl
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_ALLOWED_KINDS = {
|
||||
"lora": "loras",
|
||||
"checkpoint": "checkpoints",
|
||||
"unet": "diffusion_models",
|
||||
"vae": "vae",
|
||||
"clip": "text_encoders",
|
||||
"gguf": "diffusion_models",
|
||||
}
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_FILENAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+ -]{0,180}\.(?:safetensors|gguf)$")
|
||||
_MAX_BYTES = 24 * 1024 * 1024 * 1024
|
||||
|
||||
|
||||
class ArtifactInstallError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def validate_request(payload: dict[str, Any]) -> dict[str, str]:
|
||||
kind = str(payload.get("artifact_type") or "").strip().lower()
|
||||
url = str(payload.get("source_url") or "").strip()
|
||||
sha256 = str(payload.get("sha256") or "").strip().lower()
|
||||
filename = str(payload.get("filename") or "").strip()
|
||||
license_name = str(payload.get("license") or "").strip()
|
||||
source_id = str(payload.get("source_id") or "").strip()
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
if kind not in _ALLOWED_KINDS:
|
||||
raise ArtifactInstallError("artifact_type is not allow-listed")
|
||||
if parsed.scheme != "https" or not parsed.hostname:
|
||||
raise ArtifactInstallError("source_url must be an absolute HTTPS URL")
|
||||
if parsed.username or parsed.password or parsed.fragment:
|
||||
raise ArtifactInstallError("source_url credentials/fragments are forbidden")
|
||||
if not _SHA256.fullmatch(sha256):
|
||||
raise ArtifactInstallError("sha256 must be a pinned lowercase SHA-256 digest")
|
||||
expected_extension = ".gguf" if kind == "gguf" else ".safetensors"
|
||||
if (
|
||||
not _FILENAME.fullmatch(filename)
|
||||
or "/" in filename
|
||||
or "\\" in filename
|
||||
or not filename.casefold().endswith(expected_extension)
|
||||
):
|
||||
raise ArtifactInstallError(
|
||||
f"filename must be a bounded {expected_extension} basename for {kind}"
|
||||
)
|
||||
if not license_name or len(license_name) > 240:
|
||||
raise ArtifactInstallError("license provenance is required")
|
||||
if not source_id or len(source_id) > 240:
|
||||
raise ArtifactInstallError("source_id/version provenance is required")
|
||||
return {
|
||||
"artifact_type": kind,
|
||||
"source_url": url,
|
||||
"sha256": sha256,
|
||||
"filename": filename,
|
||||
"license": license_name,
|
||||
"source_id": source_id,
|
||||
}
|
||||
|
||||
|
||||
def preflight(payload: dict[str, Any], *, model_root: Path) -> dict[str, Any]:
|
||||
request = validate_request(payload)
|
||||
root = model_root.resolve(strict=True)
|
||||
if not root.is_dir() or not os.path.ismount(root):
|
||||
raise ArtifactInstallError("model root must be an existing persistent mount")
|
||||
destination_dir = (root / _ALLOWED_KINDS[request["artifact_type"]]).resolve(strict=True)
|
||||
if root not in destination_dir.parents:
|
||||
raise ArtifactInstallError("resolved destination escaped the model root")
|
||||
if not os.access(destination_dir, os.W_OK | os.X_OK):
|
||||
raise ArtifactInstallError("artifact destination is not writable by the AppOps service")
|
||||
destination = destination_dir / request["filename"]
|
||||
existing_digest = None
|
||||
if destination.exists():
|
||||
if not destination.is_file() or destination.is_symlink():
|
||||
raise ArtifactInstallError("destination is not a regular artifact")
|
||||
existing_digest = _hash_file(destination)
|
||||
if existing_digest != request["sha256"]:
|
||||
raise ArtifactInstallError("destination already exists with a different digest")
|
||||
evidence = {"request": request, "model_root": str(root), "existing_digest": existing_digest}
|
||||
fingerprint = hashlib.sha256(json.dumps(evidence, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
return {"ready": True, "fingerprint": fingerprint, "artifact_type": request["artifact_type"],
|
||||
"filename": request["filename"], "sha256": request["sha256"],
|
||||
"model_root": str(root), "already_installed": existing_digest is not None}
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _write_receipt(request: dict[str, str], destination: Path, size: int, receipt_root: Path) -> Path:
|
||||
receipt_root.mkdir(parents=True, exist_ok=True)
|
||||
receipt = {
|
||||
"schema": "privatemuse-comfyui-artifact.v1",
|
||||
"artifact_type": request["artifact_type"],
|
||||
"filename": request["filename"],
|
||||
"sha256": request["sha256"],
|
||||
"bytes": size,
|
||||
"source_id": request["source_id"],
|
||||
"source_host": urllib.parse.urlsplit(request["source_url"]).hostname,
|
||||
"license": request["license"],
|
||||
"destination": str(destination),
|
||||
}
|
||||
receipt_path = receipt_root / f'{request["sha256"]}.json'
|
||||
if receipt_path.exists():
|
||||
return receipt_path
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", prefix=".receipt-", suffix=".json", dir=receipt_root,
|
||||
delete=False, encoding="utf-8",
|
||||
) as handle:
|
||||
json.dump(receipt, handle, sort_keys=True, separators=(",", ":"))
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
temporary = Path(handle.name)
|
||||
try:
|
||||
os.link(temporary, receipt_path)
|
||||
except FileExistsError:
|
||||
pass
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return receipt_path
|
||||
|
||||
|
||||
def install(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
model_root: Path,
|
||||
receipt_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
request = validate_request(payload)
|
||||
root = model_root.resolve()
|
||||
destination_dir = (root / _ALLOWED_KINDS[request["artifact_type"]]).resolve()
|
||||
if root not in destination_dir.parents:
|
||||
raise ArtifactInstallError("resolved destination escaped the model root")
|
||||
destination_dir.mkdir(parents=True, exist_ok=True)
|
||||
destination = destination_dir / request["filename"]
|
||||
if destination.is_symlink():
|
||||
raise ArtifactInstallError("destination symlinks are forbidden")
|
||||
if destination.exists():
|
||||
digest = _hash_file(destination)
|
||||
if digest != request["sha256"]:
|
||||
raise ArtifactInstallError(
|
||||
"destination already exists with a different digest"
|
||||
)
|
||||
receipt_path = _write_receipt(request, destination, destination.stat().st_size, receipt_root)
|
||||
return {
|
||||
"status": "already_installed",
|
||||
"path": str(destination),
|
||||
"sha256": digest,
|
||||
"bytes": destination.stat().st_size,
|
||||
"receipt": str(receipt_path),
|
||||
}
|
||||
|
||||
request_obj = urllib.request.Request(
|
||||
request["source_url"],
|
||||
headers={"User-Agent": "ITWorx-AppOps/PrivateMuse-artifact-installer"},
|
||||
)
|
||||
hasher = hashlib.sha256()
|
||||
total = 0
|
||||
tmp_path: Path | None = None
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request_obj,
|
||||
timeout=30,
|
||||
context=ssl.create_default_context(),
|
||||
) as response:
|
||||
final = urllib.parse.urlsplit(response.geturl())
|
||||
if final.scheme != "https" or not final.hostname:
|
||||
raise ArtifactInstallError("download redirected outside HTTPS")
|
||||
declared = response.headers.get("Content-Length")
|
||||
if declared and int(declared) > _MAX_BYTES:
|
||||
raise ArtifactInstallError(
|
||||
"artifact exceeds the 24 GiB bounded installer limit"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
prefix=".appops-",
|
||||
suffix=".part",
|
||||
dir=destination_dir,
|
||||
delete=False,
|
||||
) as handle:
|
||||
tmp_path = Path(handle.name)
|
||||
while True:
|
||||
chunk = response.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > _MAX_BYTES:
|
||||
raise ArtifactInstallError(
|
||||
"artifact exceeded the 24 GiB bounded installer limit"
|
||||
)
|
||||
hasher.update(chunk)
|
||||
handle.write(chunk)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
digest = hasher.hexdigest()
|
||||
if digest != request["sha256"]:
|
||||
raise ArtifactInstallError(
|
||||
"downloaded artifact SHA-256 did not match the pinned digest"
|
||||
)
|
||||
# Link is atomic and fails if another installer created the final name.
|
||||
# os.replace would silently overwrite a different digest in that race.
|
||||
try:
|
||||
os.link(tmp_path, destination)
|
||||
except FileExistsError as exc:
|
||||
if destination.is_symlink() or not destination.is_file() or _hash_file(destination) != digest:
|
||||
raise ArtifactInstallError("destination appeared with a different digest") from exc
|
||||
tmp_path.unlink()
|
||||
tmp_path = None
|
||||
receipt_path = _write_receipt(request, destination, total, receipt_root)
|
||||
return {
|
||||
"status": "installed",
|
||||
"path": str(destination),
|
||||
"sha256": digest,
|
||||
"bytes": total,
|
||||
"receipt": str(receipt_path),
|
||||
}
|
||||
finally:
|
||||
if tmp_path is not None:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,546 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/privatemuse"
|
||||
_LEGACY_REVISION = "715f7d152d35745521a2e014e334a1509bdce6a2"
|
||||
|
||||
_PROJECT: dict[str, object] = {
|
||||
"branch": "main",
|
||||
"slug": "privatemuse",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
|
||||
"exact_source": True,
|
||||
"env_source": "/srv/example/appdata/privatemuse/.env",
|
||||
"compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
|
||||
"compose_project": "privatemuse",
|
||||
"containers": [
|
||||
"privatemuse-backend-1",
|
||||
"privatemuse-frontend-1",
|
||||
"privatemuse-audio-enhancement-1",
|
||||
],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
"environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
|
||||
}
|
||||
|
||||
# This is a fixed, application-owned one-time migration. Callers cannot supply
|
||||
# commands, paths, container names, Compose files or deployment JSON. It uses
|
||||
# the same controller-owned registry, global production lock and official
|
||||
# deploy.py bootstrap-legacy path as the deployment broker itself.
|
||||
_ENROLL_SCRIPT = r'''
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
state_path = state_root / "privatemuse.json"
|
||||
project_lock_path = state_root / "locks" / "privatemuse.lock"
|
||||
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
repository, legacy_revision, raw_project = sys.argv[1:4]
|
||||
project = json.loads(raw_project)
|
||||
|
||||
expected_project = {
|
||||
"branch": "main",
|
||||
"slug": "privatemuse",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
|
||||
"exact_source": True,
|
||||
"env_source": "/srv/example/appdata/privatemuse/.env",
|
||||
"compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
|
||||
"compose_project": "privatemuse",
|
||||
"containers": [
|
||||
"privatemuse-backend-1",
|
||||
"privatemuse-frontend-1",
|
||||
"privatemuse-audio-enhancement-1",
|
||||
],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
"environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
|
||||
}
|
||||
expected_legacy = "715f7d152d35745521a2e014e334a1509bdce6a2"
|
||||
stale_audio = {
|
||||
"privatemuse-audio-runtime-base-1",
|
||||
"privatemuse-audio-speech-1",
|
||||
"privatemuse-audio-music-1",
|
||||
"privatemuse-audio-sound-1",
|
||||
"privatemuse-audio-transcription-1",
|
||||
"privatemuse-audio-lipsync-1",
|
||||
}
|
||||
|
||||
if repository != "jens/privatemuse" or legacy_revision != expected_legacy or project != expected_project:
|
||||
raise SystemExit(40)
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(20)
|
||||
registry_stat = registry_path.stat()
|
||||
if not stat.S_ISREG(registry_stat.st_mode):
|
||||
raise SystemExit(21)
|
||||
try:
|
||||
if (
|
||||
host_registry_path.is_symlink()
|
||||
or not host_registry_path.is_file()
|
||||
or not os.path.samefile(registry_path, host_registry_path)
|
||||
):
|
||||
raise SystemExit(36)
|
||||
except OSError:
|
||||
raise SystemExit(36)
|
||||
|
||||
env_source = Path(str(project["env_source"]))
|
||||
if env_source.is_symlink() or not env_source.is_file():
|
||||
raise SystemExit(41)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(24)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
legacy = git_run("rev-parse", f"{legacy_revision}^{{commit}}")
|
||||
if legacy.returncode != 0 or legacy.stdout.strip() != legacy_revision:
|
||||
raise SystemExit(25)
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
main_revision = main.stdout.strip()
|
||||
if main.returncode != 0 or len(main_revision) != 40:
|
||||
raise SystemExit(26)
|
||||
if git_run("merge-base", "--is-ancestor", legacy_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(27)
|
||||
for compose_file in project["compose_files"]:
|
||||
if git_run("cat-file", "-e", f"{main_revision}:{compose_file}").returncode != 0:
|
||||
raise SystemExit(28)
|
||||
|
||||
try:
|
||||
if state_root.is_symlink() or not state_root.is_dir():
|
||||
raise SystemExit(50)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if project_lock_path.parent.is_symlink() or project_lock_path.is_symlink():
|
||||
raise SystemExit(29)
|
||||
project_lock_handle = project_lock_path.open("a+", encoding="utf-8")
|
||||
lock_handle = lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
with project_lock_handle as project_lock, lock_handle as lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
try:
|
||||
projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
if not isinstance(projects, dict):
|
||||
raise SystemExit(22)
|
||||
|
||||
current = projects.get(repository)
|
||||
canonical_target_match = current == project
|
||||
if current is not None and current != project:
|
||||
if not isinstance(current, dict):
|
||||
raise SystemExit(54)
|
||||
if current.get("appops_managed_target") is True:
|
||||
raise SystemExit(55)
|
||||
target_fields = [
|
||||
"branch",
|
||||
"slug",
|
||||
"source_dir",
|
||||
"exact_source",
|
||||
"env_source",
|
||||
"compose_files",
|
||||
"compose_project",
|
||||
"containers",
|
||||
"verify_oci_revision",
|
||||
"health_timeout",
|
||||
"environment",
|
||||
]
|
||||
for index, key in enumerate(target_fields, start=60):
|
||||
if key not in current or current.get(key) != project[key]:
|
||||
raise SystemExit(index)
|
||||
extra_keys = set(current) - set(project)
|
||||
allowed_metadata_keys = {
|
||||
"gitea_repository_id",
|
||||
"gitea_webhook_secret_file",
|
||||
"validation",
|
||||
}
|
||||
expected_validation = {
|
||||
"validation_gate_version": 1,
|
||||
"required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
repository_id = current.get("gitea_repository_id")
|
||||
if (
|
||||
extra_keys != allowed_metadata_keys
|
||||
or not isinstance(repository_id, int)
|
||||
or isinstance(repository_id, bool)
|
||||
or repository_id <= 0
|
||||
or current.get("gitea_webhook_secret_file") != "jens__privatemuse"
|
||||
or current.get("validation") != expected_validation
|
||||
):
|
||||
raise SystemExit(71)
|
||||
canonical_target_match = True
|
||||
|
||||
existing_state = None
|
||||
if state_path.exists():
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(42)
|
||||
try:
|
||||
existing_state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(42)
|
||||
if not isinstance(existing_state, dict):
|
||||
raise SystemExit(42)
|
||||
if existing_state.get("schema_version") == 3:
|
||||
if existing_state.get("repository") != repository:
|
||||
raise SystemExit(43)
|
||||
if not canonical_target_match:
|
||||
raise SystemExit(23)
|
||||
print("already-enrolled", flush=True)
|
||||
raise SystemExit(0)
|
||||
if not (
|
||||
existing_state.get("repository") == repository
|
||||
and existing_state.get("status") == "success"
|
||||
and existing_state.get("revision") == legacy_revision
|
||||
and "schema_version" not in existing_state
|
||||
and "live_receipt" not in existing_state
|
||||
and "live_legacy_provenance" not in existing_state
|
||||
):
|
||||
raise SystemExit(43)
|
||||
|
||||
expected_names = set(project["containers"])
|
||||
observed_result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"--all",
|
||||
"--filter",
|
||||
f"label=com.docker.compose.project={project['compose_project']}",
|
||||
"--format",
|
||||
"{{.Names}}",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
if observed_result.returncode != 0:
|
||||
raise SystemExit(44)
|
||||
observed_names = {line.strip() for line in observed_result.stdout.splitlines() if line.strip()}
|
||||
unexpected = observed_names - expected_names - stale_audio
|
||||
if unexpected:
|
||||
raise SystemExit(45)
|
||||
|
||||
for required in sorted(expected_names):
|
||||
inspected = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{index .Config.Labels \"com.docker.compose.project\"}}",
|
||||
required,
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(46)
|
||||
try:
|
||||
running, health, compose_project = inspected.stdout.strip().split("|", 2)
|
||||
except ValueError:
|
||||
raise SystemExit(46)
|
||||
if running != "true" or health in {"starting", "unhealthy"} or compose_project != "privatemuse":
|
||||
raise SystemExit(46)
|
||||
|
||||
# Remove only known, stopped provider containers. Images, named volumes,
|
||||
# model directories and persistent media are deliberately left untouched.
|
||||
for stale in sorted(observed_names & stale_audio):
|
||||
inspected = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{.State.Running}}|{{index .Config.Labels \"com.docker.compose.project\"}}",
|
||||
stale,
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(47)
|
||||
try:
|
||||
running, compose_project = inspected.stdout.strip().split("|", 1)
|
||||
except ValueError:
|
||||
raise SystemExit(47)
|
||||
if running == "true" or compose_project != "privatemuse":
|
||||
raise SystemExit(48)
|
||||
removed = subprocess.run(
|
||||
["docker", "rm", stale],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
if removed.returncode != 0:
|
||||
raise SystemExit(49)
|
||||
|
||||
if current is None:
|
||||
backup = host_registry_path.with_name(
|
||||
f"projects.json.privatemuse-backup-{int(time.time())}-{os.getpid()}"
|
||||
)
|
||||
try:
|
||||
shutil.copy2(host_registry_path, backup)
|
||||
except Exception:
|
||||
raise SystemExit(32)
|
||||
updated = dict(projects)
|
||||
updated[repository] = project
|
||||
temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
|
||||
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
|
||||
try:
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.S_IMODE(registry_stat.st_mode))
|
||||
os.replace(temporary, host_registry_path)
|
||||
directory_fd = os.open(str(host_registry_path.parent), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(34)
|
||||
|
||||
if existing_state is None:
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
if state_root.is_symlink() or not state_root.is_dir():
|
||||
raise SystemExit(50)
|
||||
temporary_state = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
|
||||
payload = {
|
||||
"repository": repository,
|
||||
"revision": legacy_revision,
|
||||
"status": "success",
|
||||
}
|
||||
try:
|
||||
with temporary_state.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary_state, 0o600)
|
||||
os.replace(temporary_state, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary_state.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(51)
|
||||
|
||||
try:
|
||||
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
||||
if controller_path.is_symlink() or not controller_path.is_file():
|
||||
raise RuntimeError("controller module is unavailable")
|
||||
controller_spec = importlib.util.spec_from_file_location(
|
||||
"appops_locked_privatemuse_bootstrap", controller_path
|
||||
)
|
||||
if controller_spec is None or controller_spec.loader is None:
|
||||
raise RuntimeError("controller module cannot be loaded")
|
||||
controller = importlib.util.module_from_spec(controller_spec)
|
||||
controller_spec.loader.exec_module(controller)
|
||||
controller.bootstrap_legacy_state(repository, project, legacy_revision)
|
||||
except Exception:
|
||||
raise SystemExit(52)
|
||||
|
||||
try:
|
||||
final_state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(53)
|
||||
if not (
|
||||
isinstance(final_state, dict)
|
||||
and final_state.get("schema_version") == 3
|
||||
and final_state.get("repository") == repository
|
||||
and final_state.get("live_revision") == legacy_revision
|
||||
and final_state.get("live_status") == "healthy"
|
||||
and final_state.get("live_legacy_provenance") is True
|
||||
):
|
||||
raise SystemExit(53)
|
||||
|
||||
print("enrolled", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class PrivateMuseDeploymentEnrollment:
|
||||
"""One-time, fail-closed enrollment of the existing PrivateMuse production runtime."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": redact_message(
|
||||
str(status.get("reason") or status.get("status") or "unavailable")
|
||||
),
|
||||
}
|
||||
controller = dict(status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": "deployment controller is not running",
|
||||
}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not container_id:
|
||||
return {
|
||||
"status": "connector_unavailable",
|
||||
"reason": "Portainer deployment control is unavailable",
|
||||
}
|
||||
|
||||
command = [
|
||||
"python3",
|
||||
"-c",
|
||||
_ENROLL_SCRIPT,
|
||||
_REPOSITORY,
|
||||
_LEGACY_REVISION,
|
||||
json.dumps(_PROJECT, separators=(",", ":"), sort_keys=True),
|
||||
]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": command,
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "PrivateMuse enrollment could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 120
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "PrivateMuse enrollment timed out"}
|
||||
reasons = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
21: "deployment projects registry is not a regular file",
|
||||
22: "deployment projects registry does not contain an object",
|
||||
23: "PrivateMuse already has a different deployment target",
|
||||
24: "mounted PrivateMuse bare repository is unavailable",
|
||||
25: "verified PrivateMuse legacy revision cannot be resolved",
|
||||
26: "PrivateMuse main cannot be resolved",
|
||||
27: "verified PrivateMuse legacy revision is not an ancestor of main",
|
||||
28: "required PrivateMuse production Compose file is absent from main",
|
||||
29: "deployment registry lock file cannot be opened",
|
||||
30: "deployment registry lock cannot be acquired",
|
||||
31: "deployment projects registry cannot be read as JSON",
|
||||
32: "deployment projects registry backup cannot be created",
|
||||
34: "deployment projects registry replacement failed",
|
||||
36: "deployment projects registry host alias is unavailable or mismatched",
|
||||
40: "PrivateMuse fixed enrollment contract was altered",
|
||||
41: "PrivateMuse production environment file is unavailable or unsafe",
|
||||
42: "PrivateMuse deployment state file is unavailable or invalid",
|
||||
43: "PrivateMuse deployment state conflicts with the verified legacy baseline",
|
||||
44: "PrivateMuse Compose project inventory could not be read",
|
||||
45: "PrivateMuse Compose project contains an unexpected container",
|
||||
46: "required PrivateMuse production container is missing or unhealthy",
|
||||
47: "stale PrivateMuse audio container could not be inspected",
|
||||
48: "a stale PrivateMuse audio provider is still running and was not removed",
|
||||
49: "a verified stopped PrivateMuse audio container could not be removed",
|
||||
50: "deployment state directory is unavailable or unsafe",
|
||||
51: "verified PrivateMuse legacy state could not be written durably",
|
||||
52: "deployment controller rejected PrivateMuse legacy-state bootstrap",
|
||||
53: "PrivateMuse legacy-state bootstrap did not produce the expected managed state",
|
||||
54: "PrivateMuse deployment target is not an object",
|
||||
55: "PrivateMuse deployment target is owned by the controller v2 managed-target contract",
|
||||
60: "PrivateMuse deployment target differs in branch",
|
||||
61: "PrivateMuse deployment target differs in slug",
|
||||
62: "PrivateMuse deployment target differs in source_dir",
|
||||
63: "PrivateMuse deployment target differs in exact_source",
|
||||
64: "PrivateMuse deployment target differs in env_source",
|
||||
65: "PrivateMuse deployment target differs in compose_files",
|
||||
66: "PrivateMuse deployment target differs in compose_project",
|
||||
67: "PrivateMuse deployment target differs in containers",
|
||||
68: "PrivateMuse deployment target differs in verify_oci_revision",
|
||||
69: "PrivateMuse deployment target differs in health_timeout",
|
||||
70: "PrivateMuse deployment target differs in environment",
|
||||
71: "PrivateMuse deployment target matches canonical fields but has additional or invalid controller metadata",
|
||||
}
|
||||
if exit_code != 0:
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": reasons.get(
|
||||
exit_code,
|
||||
"PrivateMuse enrollment failed closed before a production application deploy",
|
||||
),
|
||||
}
|
||||
return {
|
||||
"status": "enrolled",
|
||||
"repository": _REPOSITORY,
|
||||
"legacy_revision": _LEGACY_REVISION,
|
||||
"compose_project": _PROJECT["compose_project"],
|
||||
"containers": list(_PROJECT["containers"]),
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,324 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/privatemuse"
|
||||
_ENDPOINT = "http://198.19.0.1:1299/v1/gitea/push/jens/privatemuse"
|
||||
|
||||
# Fixed, one-target repair. The model/operator cannot supply repository, URL,
|
||||
# secret, token, hook id or command text. Both credentials stay inside the
|
||||
# existing root-owned deployment-control boundary.
|
||||
_REPAIR_SCRIPT = r"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
repository, endpoint = sys.argv[1:3]
|
||||
if repository != "jens/privatemuse" or endpoint != "http://198.19.0.1:1299/v1/gitea/push/jens/privatemuse":
|
||||
raise SystemExit(40)
|
||||
|
||||
projects_path = Path("/opt/gitea-deploy/projects.json")
|
||||
secret_root = Path("/opt/gitea-deploy/webhook-secrets")
|
||||
secret_path = secret_root / "jens__privatemuse"
|
||||
admin_token_path = Path("/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token")
|
||||
expected_validation = {
|
||||
"validation_gate_version": 1,
|
||||
"required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
|
||||
def safe_regular(path: Path, *, private: bool = False) -> None:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise SystemExit(20)
|
||||
info = path.stat()
|
||||
if os.name != "nt" and info.st_uid != os.geteuid():
|
||||
raise SystemExit(21)
|
||||
if private and os.name != "nt" and stat.S_IMODE(info.st_mode) & 0o077:
|
||||
raise SystemExit(22)
|
||||
|
||||
safe_regular(projects_path)
|
||||
safe_regular(secret_path, private=True)
|
||||
safe_regular(admin_token_path, private=True)
|
||||
|
||||
try:
|
||||
projects = json.loads(projects_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(23)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
repository_id = project.get("gitea_repository_id") if isinstance(project, dict) else None
|
||||
if (
|
||||
not isinstance(project, dict)
|
||||
or project.get("branch") != "main"
|
||||
or isinstance(repository_id, bool)
|
||||
or not isinstance(repository_id, int)
|
||||
or repository_id <= 0
|
||||
or project.get("gitea_webhook_secret_file") != "jens__privatemuse"
|
||||
or project.get("validation") != expected_validation
|
||||
):
|
||||
raise SystemExit(24)
|
||||
|
||||
secret = secret_path.read_text(encoding="utf-8").strip()
|
||||
token = admin_token_path.read_text(encoding="utf-8").strip()
|
||||
if not 32 <= len(secret) <= 256 or any(ord(ch) < 33 or ord(ch) > 126 for ch in secret):
|
||||
raise SystemExit(25)
|
||||
if not 20 <= len(token) <= 512 or any(ch in token for ch in "\r\n"):
|
||||
raise SystemExit(26)
|
||||
|
||||
# Prove the effective Gitea outbound boundary directly instead of inferring it
|
||||
# from app.ini syntax. This is stronger than a text parse: the Gitea container
|
||||
# must actually reach the fixed private controller endpoint and receive the
|
||||
# expected controller protocol identity. No credential is sent.
|
||||
health = subprocess.run(
|
||||
[
|
||||
"docker", "exec", "Gitea",
|
||||
"curl", "--fail", "--silent", "--show-error", "--max-time", "5",
|
||||
"http://198.19.0.1:1299/health",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if health.returncode != 0:
|
||||
raise SystemExit(28)
|
||||
try:
|
||||
health_payload = json.loads(health.stdout)
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(29)
|
||||
if (
|
||||
not isinstance(health_payload, dict)
|
||||
or health_payload.get("status") != "ok"
|
||||
or health_payload.get("protocolVersion") != "itworx-controller-request/v2"
|
||||
):
|
||||
raise SystemExit(29)
|
||||
|
||||
def request(method: str, path: str, payload=None):
|
||||
cmd = [
|
||||
"docker", "exec", "-i", "Gitea",
|
||||
"curl", "--fail", "--silent", "--show-error", "--max-time", "15",
|
||||
"--request", method,
|
||||
"--header", f"Authorization: token {token}",
|
||||
"--header", "Accept: application/json",
|
||||
]
|
||||
raw = None
|
||||
if payload is not None:
|
||||
cmd.extend(["--header", "Content-Type: application/json", "--data-binary", "@-"])
|
||||
raw = json.dumps(payload, separators=(",", ":"))
|
||||
cmd.append(f"http://127.0.0.1:3000/api/v1{path}")
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=raw,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(30)
|
||||
if not result.stdout.strip():
|
||||
return None
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(31)
|
||||
|
||||
repo = request("GET", "/repos/Jens/PrivateMuse")
|
||||
if (
|
||||
not isinstance(repo, dict)
|
||||
or repo.get("id") != repository_id
|
||||
or str(repo.get("full_name") or "").lower() != repository
|
||||
or repo.get("default_branch") != "main"
|
||||
):
|
||||
raise SystemExit(32)
|
||||
|
||||
hooks = request("GET", "/repos/Jens/PrivateMuse/hooks")
|
||||
if not isinstance(hooks, list):
|
||||
raise SystemExit(33)
|
||||
|
||||
exact = []
|
||||
semantic_other = []
|
||||
for hook in hooks:
|
||||
if not isinstance(hook, dict):
|
||||
continue
|
||||
hook_config = hook.get("config")
|
||||
url = str(hook_config.get("url") or "") if isinstance(hook_config, dict) else ""
|
||||
if url == endpoint:
|
||||
exact.append(hook)
|
||||
elif url.endswith("/v1/gitea/push/jens/privatemuse"):
|
||||
semantic_other.append(hook)
|
||||
if len(exact) > 1 or semantic_other:
|
||||
raise SystemExit(34)
|
||||
|
||||
payload = {
|
||||
"active": True,
|
||||
"branch_filter": "main",
|
||||
"events": ["push"],
|
||||
"config": {
|
||||
"url": endpoint,
|
||||
"content_type": "json",
|
||||
"secret": secret,
|
||||
"insecure_ssl": "0",
|
||||
},
|
||||
}
|
||||
|
||||
if exact:
|
||||
hook_id = exact[0].get("id")
|
||||
if isinstance(hook_id, bool) or not isinstance(hook_id, int) or hook_id <= 0:
|
||||
raise SystemExit(35)
|
||||
updated = request("PATCH", f"/repos/Jens/PrivateMuse/hooks/{hook_id}", payload)
|
||||
else:
|
||||
created = request(
|
||||
"POST",
|
||||
"/repos/Jens/PrivateMuse/hooks",
|
||||
{"type": "gitea", **payload},
|
||||
)
|
||||
hook_id = created.get("id") if isinstance(created, dict) else None
|
||||
if isinstance(hook_id, bool) or not isinstance(hook_id, int) or hook_id <= 0:
|
||||
raise SystemExit(36)
|
||||
updated = created
|
||||
|
||||
if not isinstance(updated, dict):
|
||||
updated = request("GET", f"/repos/Jens/PrivateMuse/hooks/{hook_id}")
|
||||
config_out = updated.get("config") if isinstance(updated, dict) else None
|
||||
events_out = updated.get("events") if isinstance(updated, dict) else None
|
||||
if (
|
||||
not isinstance(updated, dict)
|
||||
or updated.get("id") != hook_id
|
||||
or updated.get("active") is not True
|
||||
or str(updated.get("branch_filter") or "") != "main"
|
||||
or not isinstance(config_out, dict)
|
||||
or config_out.get("url") != endpoint
|
||||
or str(config_out.get("content_type") or "").lower() != "json"
|
||||
or events_out != ["push"]
|
||||
):
|
||||
raise SystemExit(37)
|
||||
|
||||
# Never print the hook payload, token or secret. Hook id is non-sensitive
|
||||
# evidence that the bounded Gitea object exists.
|
||||
print(f"privatemuse_runnerless_webhook_ready hook_id={hook_id}", flush=True)
|
||||
"""
|
||||
|
||||
|
||||
class PrivateMuseWebhookRepair:
|
||||
"""Repair only the fixed PrivateMuse runnerless repository webhook."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
controller = (
|
||||
dict(status.get("container") or {})
|
||||
if status.get("status") == "ok"
|
||||
else {}
|
||||
)
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if (
|
||||
controller.get("name") != _CONTROLLER
|
||||
or controller.get("state") != "running"
|
||||
or not container_id
|
||||
or not config
|
||||
):
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": "deployment controller is unavailable",
|
||||
}
|
||||
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
command = ["python3", "-c", _REPAIR_SCRIPT, _REPOSITORY, _ENDPOINT]
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": command,
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {
|
||||
"status": "error",
|
||||
"reason": "PrivateMuse webhook repair could not start",
|
||||
}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
deadline = time.monotonic() + 60
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json",
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "PrivateMuse webhook repair timed out; inspect before retrying",
|
||||
}
|
||||
if exit_code == 0:
|
||||
return {
|
||||
"status": "repaired",
|
||||
"repository": _REPOSITORY,
|
||||
"endpoint": _ENDPOINT,
|
||||
"branch": "main",
|
||||
"event": "push",
|
||||
}
|
||||
|
||||
reasons = {
|
||||
20: "required controller/Gitea file is unavailable or unsafe",
|
||||
21: "required repair file is not owned by the controller user",
|
||||
22: "required credential file permissions are too broad",
|
||||
23: "deployment projects registry is invalid",
|
||||
24: "PrivateMuse runnerless target metadata differs from the fixed contract",
|
||||
25: "PrivateMuse controller webhook secret is invalid",
|
||||
26: "Gitea admin credential is unavailable or invalid",
|
||||
27: "reserved legacy Gitea config validation failure",
|
||||
28: "Gitea cannot reach the fixed private deployment controller endpoint",
|
||||
29: "Private deployment controller health identity is invalid or unavailable",
|
||||
30: "bounded Gitea API request failed",
|
||||
31: "Gitea returned an invalid JSON response",
|
||||
32: "Gitea repository identity or default branch differs from the fixed contract",
|
||||
33: "Gitea webhook inventory is invalid",
|
||||
34: "PrivateMuse has ambiguous or stale runnerless webhook endpoints",
|
||||
35: "existing PrivateMuse runnerless hook id is invalid",
|
||||
36: "Gitea did not return a valid hook after creation",
|
||||
37: "PrivateMuse runnerless hook verification failed after repair",
|
||||
40: "fixed PrivateMuse webhook repair contract was altered",
|
||||
}
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": reasons.get(
|
||||
exit_code,
|
||||
"PrivateMuse runnerless webhook repair failed closed",
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
"""Durable AppOps persistence for the one fixed RepoPulse adoption intent.
|
||||
|
||||
This module validates and persists protocol state only. It has no API route,
|
||||
controller client, product enable flag, or external side effect. A future
|
||||
adapter must authenticate controller evidence before passing it here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import RepoPulseAdoptionJournal
|
||||
from . import repopulse_legacy_adoption_protocol as protocol
|
||||
|
||||
_SLOT = "repopulse"
|
||||
|
||||
|
||||
class RepoPulseAdoptionJournalError(ValueError):
|
||||
"""Sanitized journal state or persistence error."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _rollback(session: Session) -> None:
|
||||
try:
|
||||
session.rollback()
|
||||
except SQLAlchemyError:
|
||||
pass
|
||||
|
||||
|
||||
def _raise_protocol_error(session: Session, error: protocol.AdoptionProtocolError) -> None:
|
||||
_rollback(session)
|
||||
raise RepoPulseAdoptionJournalError(error.code) from None
|
||||
|
||||
|
||||
def _raise_storage_error(session: Session) -> None:
|
||||
_rollback(session)
|
||||
raise RepoPulseAdoptionJournalError("journal_storage_unavailable") from None
|
||||
|
||||
|
||||
def _load(session: Session) -> tuple[RepoPulseAdoptionJournal | None, dict[str, Any] | None]:
|
||||
row = session.get(RepoPulseAdoptionJournal, _SLOT, populate_existing=True)
|
||||
if row is None:
|
||||
return None, None
|
||||
|
||||
if (
|
||||
row.slot != _SLOT
|
||||
or row.repository != protocol.REPOSITORY
|
||||
or row.application_id != protocol.APPLICATION_ID
|
||||
or not isinstance(row.revision, int)
|
||||
or isinstance(row.revision, bool)
|
||||
or row.revision < 1
|
||||
or not isinstance(row.intent_json, dict)
|
||||
):
|
||||
raise RepoPulseAdoptionJournalError("journal_identity_or_shape_invalid")
|
||||
|
||||
intent = row.intent_json
|
||||
try:
|
||||
protocol.intent_binding_sha256(intent)
|
||||
except protocol.AdoptionProtocolError:
|
||||
raise RepoPulseAdoptionJournalError("journal_intent_invalid") from None
|
||||
if (
|
||||
row.intent_id != intent.get("intentId")
|
||||
or row.idempotency_sha256 != intent.get("idempotencySha256")
|
||||
or row.preflight_sha256 != intent.get("preflightSha256")
|
||||
or row.phase != intent.get("phase")
|
||||
or row.status != intent.get("status")
|
||||
):
|
||||
raise RepoPulseAdoptionJournalError("journal_projection_mismatch")
|
||||
return row, intent
|
||||
|
||||
|
||||
def _status(
|
||||
intent: dict[str, Any] | None,
|
||||
revision: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
recovery = intent.get("recovery") if intent else None
|
||||
return {
|
||||
"apiVersion": protocol.PROTOCOL_VERSION,
|
||||
"repository": protocol.REPOSITORY,
|
||||
"applicationId": protocol.APPLICATION_ID,
|
||||
"status": intent["status"] if intent else "unreserved",
|
||||
"phase": intent["phase"] if intent else None,
|
||||
"intentId": intent["intentId"] if intent else None,
|
||||
"revision": revision,
|
||||
"preflightSha256": intent["preflightSha256"] if intent else None,
|
||||
"intentBindingSha256": (
|
||||
protocol.intent_binding_sha256(intent) if intent else None
|
||||
),
|
||||
"recoveryReason": recovery.get("reason") if isinstance(recovery, dict) else None,
|
||||
"failureEvidenceSha256": (
|
||||
recovery.get("failureEvidenceSha256") if isinstance(recovery, dict) else None
|
||||
),
|
||||
"controllerReconciliationRequired": (
|
||||
recovery.get("controllerReconciliationRequired") is True
|
||||
if isinstance(recovery, dict)
|
||||
else False
|
||||
),
|
||||
"legacyVolumePreserved": True,
|
||||
}
|
||||
|
||||
|
||||
def get_repopulse_adoption_status(session: Session) -> dict[str, Any]:
|
||||
"""Return fixed identity and safe digests, never stored evidence payloads."""
|
||||
try:
|
||||
row, intent = _load(session)
|
||||
return _status(intent, row.revision if row else 0)
|
||||
except RepoPulseAdoptionJournalError:
|
||||
_rollback(session)
|
||||
raise
|
||||
except SQLAlchemyError:
|
||||
_raise_storage_error(session)
|
||||
|
||||
|
||||
def reserve_repopulse_adoption(
|
||||
session: Session,
|
||||
preflight_report: object,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically create or replay the sole fixed RepoPulse intent."""
|
||||
try:
|
||||
candidate = protocol.create_intent(preflight_report, idempotency_key)
|
||||
now = utc_now()
|
||||
statement = sqlite_insert(RepoPulseAdoptionJournal).values(
|
||||
slot=_SLOT,
|
||||
repository=protocol.REPOSITORY,
|
||||
application_id=protocol.APPLICATION_ID,
|
||||
intent_id=candidate["intentId"],
|
||||
idempotency_sha256=candidate["idempotencySha256"],
|
||||
preflight_sha256=candidate["preflightSha256"],
|
||||
phase=candidate["phase"],
|
||||
status=candidate["status"],
|
||||
revision=1,
|
||||
intent_json=candidate,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
# The insert acquires SQLite's writer reservation before the read. On
|
||||
# conflict it is a no-op, so concurrent first reservations serialize
|
||||
# without replacing the persisted intent.
|
||||
session.execute(
|
||||
statement.on_conflict_do_nothing(
|
||||
index_elements=[RepoPulseAdoptionJournal.slot]
|
||||
)
|
||||
)
|
||||
persisted_row, persisted_intent = _load(session)
|
||||
if persisted_row is None or persisted_intent is None:
|
||||
raise RepoPulseAdoptionJournalError("journal_persistence_failed")
|
||||
replay = protocol.reserve_or_replay(
|
||||
persisted_intent, preflight_report, idempotency_key
|
||||
)
|
||||
session.commit()
|
||||
return _status(replay, persisted_row.revision)
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
_raise_protocol_error(session, error)
|
||||
except RepoPulseAdoptionJournalError:
|
||||
_rollback(session)
|
||||
raise
|
||||
except SQLAlchemyError:
|
||||
_raise_storage_error(session)
|
||||
|
||||
|
||||
def _persist_updated_intent(
|
||||
session: Session,
|
||||
row: RepoPulseAdoptionJournal,
|
||||
current: dict[str, Any],
|
||||
updated: dict[str, Any],
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
if updated == current:
|
||||
# A protocol-approved replay does not create a new journal revision.
|
||||
return _status(current, row.revision)
|
||||
if (
|
||||
not isinstance(expected_revision, int)
|
||||
or isinstance(expected_revision, bool)
|
||||
or expected_revision < 1
|
||||
or expected_revision != row.revision
|
||||
):
|
||||
raise RepoPulseAdoptionJournalError("journal_revision_conflict")
|
||||
|
||||
result = session.execute(
|
||||
update(RepoPulseAdoptionJournal)
|
||||
.where(
|
||||
RepoPulseAdoptionJournal.slot == _SLOT,
|
||||
RepoPulseAdoptionJournal.revision == expected_revision,
|
||||
RepoPulseAdoptionJournal.intent_id == current["intentId"],
|
||||
)
|
||||
.values(
|
||||
intent_id=updated["intentId"],
|
||||
idempotency_sha256=updated["idempotencySha256"],
|
||||
preflight_sha256=updated["preflightSha256"],
|
||||
phase=updated["phase"],
|
||||
status=updated["status"],
|
||||
intent_json=updated,
|
||||
revision=expected_revision + 1,
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
raise RepoPulseAdoptionJournalError("journal_revision_conflict")
|
||||
session.commit()
|
||||
return _status(updated, expected_revision + 1)
|
||||
|
||||
|
||||
def _update_intent(
|
||||
session: Session,
|
||||
transform: Callable[[dict[str, Any]], dict[str, Any]],
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
row, current = _load(session)
|
||||
if row is None or current is None:
|
||||
raise RepoPulseAdoptionJournalError("adoption_intent_not_reserved")
|
||||
updated = transform(current)
|
||||
return _persist_updated_intent(
|
||||
session, row, current, updated, expected_revision
|
||||
)
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
_raise_protocol_error(session, error)
|
||||
except RepoPulseAdoptionJournalError:
|
||||
_rollback(session)
|
||||
raise
|
||||
except SQLAlchemyError:
|
||||
_raise_storage_error(session)
|
||||
|
||||
|
||||
def advance_repopulse_adoption(
|
||||
session: Session,
|
||||
step: str,
|
||||
evidence: object,
|
||||
*,
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one protocol-validated checkpoint using revision compare-and-swap."""
|
||||
return _update_intent(
|
||||
session,
|
||||
lambda current: protocol.advance_intent(current, step, evidence),
|
||||
expected_revision,
|
||||
)
|
||||
|
||||
|
||||
def confirm_repopulse_controller_intent(
|
||||
session: Session,
|
||||
quiescence_evidence: object,
|
||||
*,
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist controller durability confirmation at the quiescence checkpoint.
|
||||
|
||||
The pure protocol checks exact intent identity and binding digest. This
|
||||
function does not authenticate or call the controller; a future adapter
|
||||
must authenticate the supplied controller evidence first.
|
||||
"""
|
||||
return advance_repopulse_adoption(
|
||||
session,
|
||||
"quiescence",
|
||||
quiescence_evidence,
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
|
||||
|
||||
def mark_repopulse_adoption_recovery(
|
||||
session: Session,
|
||||
reason: str,
|
||||
failure_evidence: object,
|
||||
*,
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist only the protocol's failure digest; reject conflicting recovery rewrites."""
|
||||
def transform(current: dict[str, Any]) -> dict[str, Any]:
|
||||
updated = protocol.mark_recovery_required(current, reason, failure_evidence)
|
||||
if current["status"] == "recovery_required" and updated != current:
|
||||
raise protocol.AdoptionProtocolError("idempotency_conflict")
|
||||
return updated
|
||||
|
||||
return _update_intent(session, transform, expected_revision)
|
||||
|
||||
|
||||
def resume_repopulse_adoption(
|
||||
session: Session,
|
||||
controller_receipt: object,
|
||||
*,
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Resume the journal only from a protocol-validated, exactly bound controller receipt."""
|
||||
return _update_intent(
|
||||
session,
|
||||
lambda current: protocol.resume_intent(current, controller_receipt),
|
||||
expected_revision,
|
||||
)
|
||||
@@ -0,0 +1,629 @@
|
||||
"""Atomic AppOps ownership commit for the one RepoPulse adoption.
|
||||
|
||||
This is a database-only protocol step. The caller must authenticate the live
|
||||
controller receipt before supplying the fixed registration evidence. No route,
|
||||
controller client, secret writer, or generic enrollment orchestration is used.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_, update
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting, Enrollment, Policy, RepoPulseAdoptionJournal
|
||||
from ..schemas.enrollment import EnrollmentPlanV1, EnrollmentTrustConfigV1
|
||||
from . import repopulse_adoption_journal as journal
|
||||
from . import repopulse_legacy_adoption_protocol as protocol
|
||||
from .enrollment_profiles import PROFILE_VERSION
|
||||
from .secret_store import has_secret
|
||||
|
||||
_SLOT = "repopulse"
|
||||
_TRUST_SETTING_KEY = "deployment_enrollment:trust"
|
||||
_CONTROLLER_IDENTITY = "unraid-controller"
|
||||
_TARGET_SECRET_REF = "ENROLL_REPOPULSE_TARGET_HMAC"
|
||||
_CREATED_BY = "repopulse-legacy-adoption/v1"
|
||||
_APPLY_POLICY_ID = "enrollment-repopulse-deployment-apply"
|
||||
_ROLLBACK_POLICY_ID = "enrollment-repopulse-deployment-rollback"
|
||||
_POLICY_REASON = "Enrollment-generated exact-target deployment policy."
|
||||
|
||||
|
||||
class RepoPulseOwnershipError(ValueError):
|
||||
"""Sanitized fixed ownership precondition or persistence error."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _need(condition: bool, code: str) -> None:
|
||||
if not condition:
|
||||
raise RepoPulseOwnershipError(code)
|
||||
|
||||
|
||||
def _canonical_plan(plan: EnrollmentPlanV1) -> tuple[dict[str, Any], str]:
|
||||
payload = plan.model_dump(mode="json")
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return payload, hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _fixed_plan(
|
||||
session: Session, intent: dict[str, Any]
|
||||
) -> tuple[EnrollmentPlanV1, dict[str, Any], str]:
|
||||
preflight = intent["evidence"].get("preflight")
|
||||
gates = intent["evidence"].get("targetGates")
|
||||
registration = intent["evidence"].get("controllerRegistration")
|
||||
_need(
|
||||
isinstance(preflight, dict) and isinstance(gates, dict),
|
||||
"adoption_prerequisite_missing",
|
||||
)
|
||||
_need(isinstance(registration, dict), "controller_registration_missing")
|
||||
|
||||
trust_row = session.get(AppSetting, _TRUST_SETTING_KEY)
|
||||
_need(
|
||||
trust_row is not None and trust_row.is_secret is False,
|
||||
"enrollment_trust_unavailable",
|
||||
)
|
||||
try:
|
||||
trust = EnrollmentTrustConfigV1.model_validate(trust_row.value_json)
|
||||
except (TypeError, ValueError):
|
||||
raise RepoPulseOwnershipError("enrollment_trust_invalid") from None
|
||||
_need(
|
||||
trust.controller_identity == _CONTROLLER_IDENTITY,
|
||||
"controller_identity_mismatch",
|
||||
)
|
||||
_need(
|
||||
"repopulse-authenticated" in trust.autonomous_profiles
|
||||
and "itworx-mcp-hub-npm" in trust.approved_networks
|
||||
and "example.com" in trust.allowed_domain_suffixes
|
||||
and trust.repopulse_proxy_access_list_id is not None,
|
||||
"repopulse_trust_boundary_mismatch",
|
||||
)
|
||||
|
||||
revision = preflight.get("main_revision")
|
||||
_need(
|
||||
isinstance(revision, str) and len(revision) == 40, "preflight_revision_invalid"
|
||||
)
|
||||
try:
|
||||
plan = EnrollmentPlanV1.model_validate(
|
||||
{
|
||||
"application_id": protocol.APPLICATION_ID,
|
||||
"display_name": "RepoPulse",
|
||||
"environment": "production",
|
||||
"repository": "Jens/RepoPulse",
|
||||
"revision": revision,
|
||||
"profile": "repopulse-authenticated",
|
||||
"target": {
|
||||
"target_identity": _CONTROLLER_IDENTITY,
|
||||
"container_name": "repopulse",
|
||||
},
|
||||
"storage": {"runtime_uid": 10001, "runtime_gid": 10001},
|
||||
"network": {
|
||||
"docker_network": "itworx-mcp-hub-npm",
|
||||
"internal_port": 8000,
|
||||
},
|
||||
"artifact": {
|
||||
"release_version": "1.5.0",
|
||||
"image_repository": "repopulse-managed",
|
||||
"image_tag": "production",
|
||||
"health_path": "/health",
|
||||
},
|
||||
"exposure": {
|
||||
"hostname": "repopulse.example.com",
|
||||
"upstream_service": "repopulse",
|
||||
},
|
||||
"secrets": {"additional_secret_refs": ["REPOPULSE_GITEA_TOKEN"]},
|
||||
}
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
raise RepoPulseOwnershipError("fixed_repopulse_plan_invalid") from None
|
||||
|
||||
payload, plan_sha256 = _canonical_plan(plan)
|
||||
_need(
|
||||
gates.get("profileId") == "repopulse-authenticated"
|
||||
and gates.get("profileVersion") == PROFILE_VERSION
|
||||
and gates.get("planSha256") == plan_sha256
|
||||
and registration.get("planSha256") == plan_sha256,
|
||||
"fixed_repopulse_plan_digest_mismatch",
|
||||
)
|
||||
expected_enrollment_id = f"enr-{intent['intentId'][4:]}"
|
||||
_need(
|
||||
registration.get("enrollmentId") == expected_enrollment_id,
|
||||
"fixed_enrollment_id_mismatch",
|
||||
)
|
||||
return plan, payload, plan_sha256
|
||||
|
||||
|
||||
def _expected_provenance(
|
||||
intent: dict[str, Any], registration: dict[str, Any], plan_sha256: str
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"createdBy": _CREATED_BY,
|
||||
"adoptionIntentId": intent["intentId"],
|
||||
"profileVersion": PROFILE_VERSION,
|
||||
"policyTemplateVersion": "exact-target/v1",
|
||||
"sourceRevision": intent["evidence"]["preflight"]["main_revision"],
|
||||
"planSha256": plan_sha256,
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"deploymentReceiptSha256": registration["deploymentReceiptSha256"],
|
||||
}
|
||||
|
||||
|
||||
def _expected_resources(registration: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"appdataPath": "/srv/example/appdata/repopulse",
|
||||
"dataPath": "/srv/example/appdata/repopulse/data",
|
||||
"network": "itworx-mcp-hub-npm",
|
||||
"hostname": "repopulse.example.com",
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"deploymentReceiptSha256": registration["deploymentReceiptSha256"],
|
||||
}
|
||||
|
||||
|
||||
def _expected_policy_rows(
|
||||
intent: dict[str, Any], registration: dict[str, Any], plan_sha256: str
|
||||
) -> tuple[Policy, Policy]:
|
||||
provenance = _expected_provenance(intent, registration, plan_sha256)
|
||||
return (
|
||||
Policy(
|
||||
id=_APPLY_POLICY_ID,
|
||||
action="deployment.apply",
|
||||
target=protocol.APPLICATION_ID,
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason=_POLICY_REASON,
|
||||
provenance_json=provenance,
|
||||
),
|
||||
Policy(
|
||||
id=_ROLLBACK_POLICY_ID,
|
||||
action="deployment.rollback",
|
||||
target=protocol.APPLICATION_ID,
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason=_POLICY_REASON,
|
||||
provenance_json=provenance,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _policy_matches(actual: Policy | None, expected: Policy) -> bool:
|
||||
return bool(
|
||||
actual is not None
|
||||
and actual.id == expected.id
|
||||
and actual.action == expected.action
|
||||
and actual.target == expected.target
|
||||
and actual.voice == expected.voice
|
||||
and actual.manual == expected.manual
|
||||
and actual.risk == expected.risk
|
||||
and actual.reason == expected.reason
|
||||
and actual.provenance_json == expected.provenance_json
|
||||
)
|
||||
|
||||
|
||||
def _expected_enrollment(
|
||||
intent: dict[str, Any],
|
||||
registration: dict[str, Any],
|
||||
plan: EnrollmentPlanV1,
|
||||
plan_json: dict[str, Any],
|
||||
plan_sha256: str,
|
||||
) -> Enrollment:
|
||||
return Enrollment(
|
||||
id=registration["enrollmentId"],
|
||||
application_id=protocol.APPLICATION_ID,
|
||||
repository=protocol.REPOSITORY,
|
||||
revision=plan.revision,
|
||||
profile="repopulse-authenticated",
|
||||
controller_identity=_CONTROLLER_IDENTITY,
|
||||
plan_sha256=plan_sha256,
|
||||
status="ready",
|
||||
target_secret_ref=_TARGET_SECRET_REF,
|
||||
oidc_secret_ref=None,
|
||||
plan_json=plan_json,
|
||||
resources_json=_expected_resources(registration),
|
||||
provenance_json=_expected_provenance(intent, registration, plan_sha256),
|
||||
)
|
||||
|
||||
|
||||
def _enrollment_matches(actual: Enrollment | None, expected: Enrollment) -> bool:
|
||||
return bool(
|
||||
actual is not None
|
||||
and actual.id == expected.id
|
||||
and actual.application_id == expected.application_id
|
||||
and actual.repository == expected.repository
|
||||
and actual.revision == expected.revision
|
||||
and actual.profile == expected.profile
|
||||
and actual.controller_identity == expected.controller_identity
|
||||
and actual.plan_sha256 == expected.plan_sha256
|
||||
and actual.status == expected.status
|
||||
and actual.target_secret_ref == expected.target_secret_ref
|
||||
and actual.oidc_secret_ref is None
|
||||
and actual.plan_json == expected.plan_json
|
||||
and actual.resources_json == expected.resources_json
|
||||
and actual.provenance_json == expected.provenance_json
|
||||
and actual.last_verified_at is not None
|
||||
and actual.completed_at is not None
|
||||
)
|
||||
|
||||
|
||||
def _verify_owned_rows(
|
||||
session: Session,
|
||||
intent: dict[str, Any],
|
||||
registration: dict[str, Any],
|
||||
plan: EnrollmentPlanV1,
|
||||
plan_json: dict[str, Any],
|
||||
plan_sha256: str,
|
||||
) -> tuple[Enrollment, tuple[Policy, Policy]]:
|
||||
expected_enrollment = _expected_enrollment(
|
||||
intent, registration, plan, plan_json, plan_sha256
|
||||
)
|
||||
matching_enrollments = session.exec(
|
||||
select(Enrollment).where(
|
||||
or_(
|
||||
Enrollment.application_id == protocol.APPLICATION_ID,
|
||||
Enrollment.repository == protocol.REPOSITORY,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
_need(
|
||||
len(matching_enrollments) == 1
|
||||
and _enrollment_matches(matching_enrollments[0], expected_enrollment),
|
||||
"ownership_enrollment_readback_mismatch",
|
||||
)
|
||||
expected_policies = _expected_policy_rows(intent, registration, plan_sha256)
|
||||
_need(
|
||||
all(
|
||||
_policy_matches(session.get(Policy, expected.id), expected)
|
||||
for expected in expected_policies
|
||||
),
|
||||
"ownership_policy_readback_mismatch",
|
||||
)
|
||||
_need(
|
||||
has_secret(session, _TARGET_SECRET_REF), "target_secret_reference_unavailable"
|
||||
)
|
||||
return matching_enrollments[0], expected_policies
|
||||
|
||||
|
||||
def _verify_live_registration(
|
||||
intent: dict[str, Any], live_registration_evidence: object
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
if intent.get("status") == "completed":
|
||||
checked = protocol.validate_controller_registration_evidence(
|
||||
intent, live_registration_evidence
|
||||
)
|
||||
replayed_registration = checked
|
||||
else:
|
||||
replayed = protocol.advance_intent(
|
||||
intent, "controller_registration", live_registration_evidence
|
||||
)
|
||||
replayed_registration = replayed["evidence"].get("controllerRegistration")
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
_need(
|
||||
replayed_registration == intent["evidence"].get("controllerRegistration"),
|
||||
"idempotency_conflict",
|
||||
)
|
||||
return replayed_registration
|
||||
|
||||
|
||||
def _ownership_evidence(
|
||||
registration: dict[str, Any], plan_sha256: str
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"repository": protocol.REPOSITORY,
|
||||
"applicationId": protocol.APPLICATION_ID,
|
||||
"enrollmentId": registration["enrollmentId"],
|
||||
"profileId": registration["profileId"],
|
||||
"planSha256": plan_sha256,
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"enrollmentPersisted": True,
|
||||
"exactApplyPolicyPersisted": True,
|
||||
"exactRollbackPolicyPersisted": True,
|
||||
"secretReferenceReady": True,
|
||||
}
|
||||
|
||||
|
||||
def _advance_or_verify_ownership(
|
||||
intent: dict[str, Any], evidence: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
if intent.get("status") == "completed":
|
||||
checked = protocol.validate_appops_ownership_evidence(intent, evidence)
|
||||
_need(
|
||||
checked == intent["evidence"].get("appopsOwnership"),
|
||||
"idempotency_conflict",
|
||||
)
|
||||
return intent
|
||||
return protocol.advance_intent(intent, "appops_ownership", evidence)
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
|
||||
|
||||
def _write_journal_update(
|
||||
session: Session,
|
||||
row: RepoPulseAdoptionJournal,
|
||||
current: dict[str, Any],
|
||||
updated: dict[str, Any],
|
||||
expected_revision: int,
|
||||
) -> int:
|
||||
result = session.execute(
|
||||
update(RepoPulseAdoptionJournal)
|
||||
.where(
|
||||
RepoPulseAdoptionJournal.slot == _SLOT,
|
||||
RepoPulseAdoptionJournal.revision == expected_revision,
|
||||
RepoPulseAdoptionJournal.intent_id == current["intentId"],
|
||||
)
|
||||
.values(
|
||||
idempotency_sha256=updated["idempotencySha256"],
|
||||
preflight_sha256=updated["preflightSha256"],
|
||||
phase=updated["phase"],
|
||||
status=updated["status"],
|
||||
intent_json=updated,
|
||||
revision=expected_revision + 1,
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
_need(result.rowcount == 1, "journal_revision_conflict")
|
||||
return expected_revision + 1
|
||||
|
||||
|
||||
def _readback_owned(
|
||||
session: Session, registration: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
session.rollback()
|
||||
session.expire_all()
|
||||
try:
|
||||
row, intent = journal._load(session)
|
||||
if row is None or intent is None:
|
||||
return None
|
||||
live = _verify_live_registration(intent, registration)
|
||||
plan, plan_json, plan_sha256 = _fixed_plan(session, intent)
|
||||
_verify_owned_rows(session, intent, live, plan, plan_json, plan_sha256)
|
||||
_advance_or_verify_ownership(intent, _ownership_evidence(live, plan_sha256))
|
||||
if intent["phase"] not in {"appops_owned", "completed"}:
|
||||
return None
|
||||
if intent["phase"] == "completed":
|
||||
_assert_completion_binding(intent, live, registration)
|
||||
return journal.get_repopulse_adoption_status(session)
|
||||
except (
|
||||
RepoPulseOwnershipError,
|
||||
journal.RepoPulseAdoptionJournalError,
|
||||
protocol.AdoptionProtocolError,
|
||||
SQLAlchemyError,
|
||||
):
|
||||
session.rollback()
|
||||
return None
|
||||
|
||||
|
||||
def _record_uncertain_commit(
|
||||
session: Session, expected_revision: int, receipt_sha256: str
|
||||
) -> None:
|
||||
session.rollback()
|
||||
session.expire_all()
|
||||
try:
|
||||
row, current = journal._load(session)
|
||||
if (
|
||||
row is None
|
||||
or current is None
|
||||
or row.revision != expected_revision
|
||||
or current.get("phase") != "target_registered"
|
||||
or current.get("status") != "in_progress"
|
||||
):
|
||||
return
|
||||
journal.mark_repopulse_adoption_recovery(
|
||||
session,
|
||||
"appops_commit_uncertain",
|
||||
{"controllerReceiptSha256": receipt_sha256},
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
except (
|
||||
RepoPulseOwnershipError,
|
||||
journal.RepoPulseAdoptionJournalError,
|
||||
protocol.AdoptionProtocolError,
|
||||
SQLAlchemyError,
|
||||
):
|
||||
session.rollback()
|
||||
|
||||
|
||||
def _assert_completion_binding(
|
||||
intent: dict[str, Any],
|
||||
registration: dict[str, Any],
|
||||
live_registration: dict[str, Any],
|
||||
) -> None:
|
||||
stored = intent.get("evidence", {}).get("completion")
|
||||
_need(isinstance(stored, dict), "completion_evidence_missing")
|
||||
expected = {
|
||||
"intentId": intent["intentId"],
|
||||
"repository": protocol.REPOSITORY,
|
||||
"applicationId": protocol.APPLICATION_ID,
|
||||
"enrollmentId": registration["enrollmentId"],
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"enrollmentStillPersisted": True,
|
||||
"exactApplyPolicyStillPersisted": True,
|
||||
"exactRollbackPolicyStillPersisted": True,
|
||||
"controllerReceiptStillMatches": (
|
||||
live_registration == registration
|
||||
and live_registration["receiptSha256"] == registration["receiptSha256"]
|
||||
),
|
||||
"legacyVolumeRetained": live_registration["legacyVolumeRetained"],
|
||||
}
|
||||
_need(stored == expected, "completion_binding_mismatch")
|
||||
|
||||
|
||||
def commit_repopulse_adoption_ownership(
|
||||
session: Session,
|
||||
authenticated_controller_registration: object,
|
||||
*,
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically install fixed enrollment/policies and advance to `appops_owned`.
|
||||
|
||||
The caller is responsible for authenticating the live controller response.
|
||||
The protocol validator then binds every field to the persisted fixed intent.
|
||||
Exact replay reads back the DB rows and validates the fresh receipt again.
|
||||
"""
|
||||
try:
|
||||
row, current = journal._load(session)
|
||||
_need(row is not None and current is not None, "adoption_intent_not_reserved")
|
||||
live = _verify_live_registration(current, authenticated_controller_registration)
|
||||
plan, plan_json, plan_sha256 = _fixed_plan(session, current)
|
||||
_need(
|
||||
has_secret(session, _TARGET_SECRET_REF),
|
||||
"target_secret_reference_unavailable",
|
||||
)
|
||||
|
||||
if current["phase"] in {"appops_owned", "completed"}:
|
||||
_verify_owned_rows(session, current, live, plan, plan_json, plan_sha256)
|
||||
_advance_or_verify_ownership(
|
||||
current, _ownership_evidence(live, plan_sha256)
|
||||
)
|
||||
if current["phase"] == "completed":
|
||||
_assert_completion_binding(current, live, live)
|
||||
return journal.get_repopulse_adoption_status(session)
|
||||
|
||||
_need(
|
||||
current["phase"] == "target_registered"
|
||||
and current["status"] == "in_progress",
|
||||
"adoption_not_ready_for_ownership",
|
||||
)
|
||||
_need(
|
||||
isinstance(expected_revision, int)
|
||||
and not isinstance(expected_revision, bool)
|
||||
and expected_revision == row.revision,
|
||||
"journal_revision_conflict",
|
||||
)
|
||||
|
||||
conflicting_enrollments = session.exec(
|
||||
select(Enrollment).where(
|
||||
or_(
|
||||
Enrollment.application_id == protocol.APPLICATION_ID,
|
||||
Enrollment.repository == protocol.REPOSITORY,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
_need(not conflicting_enrollments, "ownership_resource_conflict")
|
||||
expected_enrollment = _expected_enrollment(
|
||||
current, live, plan, plan_json, plan_sha256
|
||||
)
|
||||
expected_policies = _expected_policy_rows(current, live, plan_sha256)
|
||||
_need(
|
||||
all(session.get(Policy, item.id) is None for item in expected_policies),
|
||||
"ownership_resource_conflict",
|
||||
)
|
||||
|
||||
now = utc_now()
|
||||
expected_enrollment.created_at = now
|
||||
expected_enrollment.updated_at = now
|
||||
expected_enrollment.last_verified_at = now
|
||||
expected_enrollment.completed_at = now
|
||||
for policy in expected_policies:
|
||||
session.add(policy)
|
||||
session.add(expected_enrollment)
|
||||
session.flush()
|
||||
|
||||
updated = _advance_or_verify_ownership(
|
||||
current, _ownership_evidence(live, plan_sha256)
|
||||
)
|
||||
next_revision = _write_journal_update(
|
||||
session, row, current, updated, expected_revision
|
||||
)
|
||||
try:
|
||||
session.commit()
|
||||
except (SQLAlchemyError, RuntimeError, OSError):
|
||||
replay = _readback_owned(session, live)
|
||||
if replay is not None:
|
||||
return replay
|
||||
_record_uncertain_commit(session, expected_revision, live["receiptSha256"])
|
||||
raise RepoPulseOwnershipError("appops_commit_uncertain") from None
|
||||
return journal._status(updated, next_revision)
|
||||
except RepoPulseOwnershipError:
|
||||
session.rollback()
|
||||
raise
|
||||
except journal.RepoPulseAdoptionJournalError as error:
|
||||
session.rollback()
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
session.rollback()
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
except SQLAlchemyError:
|
||||
session.rollback()
|
||||
raise RepoPulseOwnershipError("ownership_storage_unavailable") from None
|
||||
|
||||
|
||||
def complete_repopulse_adoption(
|
||||
session: Session,
|
||||
authenticated_controller_registration: object,
|
||||
*,
|
||||
expected_revision: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist AppOps completion after ownership, with fresh receipt readback.
|
||||
|
||||
Controller authentication and its completed adoption/deployment receipt
|
||||
verification remain caller responsibilities. This step independently
|
||||
rechecks the receipt binding and exact owned rows on every replay.
|
||||
"""
|
||||
try:
|
||||
row, current = journal._load(session)
|
||||
_need(row is not None and current is not None, "adoption_intent_not_reserved")
|
||||
live = _verify_live_registration(current, authenticated_controller_registration)
|
||||
plan, plan_json, plan_sha256 = _fixed_plan(session, current)
|
||||
_verify_owned_rows(session, current, live, plan, plan_json, plan_sha256)
|
||||
_advance_or_verify_ownership(current, _ownership_evidence(live, plan_sha256))
|
||||
_need(
|
||||
(current["phase"] == "appops_owned" and current["status"] == "in_progress")
|
||||
or (current["phase"] == "completed" and current["status"] == "completed"),
|
||||
"adoption_not_ready_to_complete",
|
||||
)
|
||||
completion_evidence = {
|
||||
"intentId": current["intentId"],
|
||||
"repository": protocol.REPOSITORY,
|
||||
"applicationId": protocol.APPLICATION_ID,
|
||||
"enrollmentId": live["enrollmentId"],
|
||||
"controllerReceiptSha256": live["receiptSha256"],
|
||||
"enrollmentStillPersisted": True,
|
||||
"exactApplyPolicyStillPersisted": True,
|
||||
"exactRollbackPolicyStillPersisted": True,
|
||||
"controllerReceiptStillMatches": True,
|
||||
"legacyVolumeRetained": live["legacyVolumeRetained"],
|
||||
}
|
||||
if current["phase"] == "completed":
|
||||
_assert_completion_binding(current, live, live)
|
||||
_need(
|
||||
current["evidence"].get("completion") == completion_evidence,
|
||||
"completion_binding_mismatch",
|
||||
)
|
||||
return journal.get_repopulse_adoption_status(session)
|
||||
|
||||
_need(
|
||||
isinstance(expected_revision, int)
|
||||
and not isinstance(expected_revision, bool)
|
||||
and expected_revision == row.revision,
|
||||
"journal_revision_conflict",
|
||||
)
|
||||
try:
|
||||
updated = protocol.complete_intent(current, completion_evidence)
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
return journal._persist_updated_intent(
|
||||
session, row, current, updated, expected_revision
|
||||
)
|
||||
except RepoPulseOwnershipError:
|
||||
session.rollback()
|
||||
raise
|
||||
except journal.RepoPulseAdoptionJournalError as error:
|
||||
session.rollback()
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
except protocol.AdoptionProtocolError as error:
|
||||
session.rollback()
|
||||
raise RepoPulseOwnershipError(error.code) from None
|
||||
except SQLAlchemyError:
|
||||
session.rollback()
|
||||
raise RepoPulseOwnershipError("ownership_storage_unavailable") from None
|
||||
@@ -0,0 +1,735 @@
|
||||
"""Read-only, fixed-identity assessment of the existing RepoPulse deployment.
|
||||
|
||||
This module is deliberately separate from generic enrollment and contains no
|
||||
adoption, registry-write, or container-mutation path. Its collector runs only
|
||||
fixed read operations in the existing deployment controller; its evaluator is
|
||||
pure so sanitized observations can be covered without a live Unraid host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/repopulse"
|
||||
_CONTAINER = "repopulse"
|
||||
_VOLUME = "repopulse_data"
|
||||
_SOURCE_DIR = "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse"
|
||||
_APPDATA_ROOT = "/srv/example/appdata/repopulse"
|
||||
_EXPECTED_TARGET = {
|
||||
"branch": "main",
|
||||
"slug": "repopulse",
|
||||
"source_dir": _SOURCE_DIR,
|
||||
"exact_source": True,
|
||||
"env_source": f"{_APPDATA_ROOT}/deploy.env",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"compose_project": "repopulse",
|
||||
"containers": [_CONTAINER],
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 180,
|
||||
"environment_keys": [],
|
||||
}
|
||||
_EXPECTED_REGISTRY_KEYS = {
|
||||
"branch",
|
||||
"slug",
|
||||
"source_dir",
|
||||
"exact_source",
|
||||
"env_source",
|
||||
"compose_files",
|
||||
"compose_project",
|
||||
"containers",
|
||||
"verify_oci_revision",
|
||||
"health_timeout",
|
||||
"gitea_repository_id",
|
||||
"gitea_webhook_secret_file",
|
||||
"validation",
|
||||
}
|
||||
_EXPECTED_VALIDATION = {
|
||||
"validation_gate_version": 1,
|
||||
"required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
_SHA = re.compile(r"^[0-9a-f]{40}$")
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_PREFLIGHT_SCRIPT = r'''
|
||||
import http.client
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY = "jens/repopulse"
|
||||
CONTAINER = "repopulse"
|
||||
VOLUME = "repopulse_data"
|
||||
APPDATA_ROOT = Path("/srv/example/appdata/repopulse")
|
||||
SCAN_ROOT = Path("/mnt/cache/appdata/gitea-deploy-runners/sources")
|
||||
REGISTRY = Path("/opt/gitea-deploy/projects.json")
|
||||
HOST_REGISTRY = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
STATE = Path("/mnt/cache/appdata/gitea-deploy-runners/state/repopulse.json")
|
||||
BARE_REPO = Path("/gitea-repositories/jens/repopulse.git")
|
||||
SHA = re.compile(r"^[0-9a-f]{40}$")
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 4 * 1024 * 1024:
|
||||
return None
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def run_fixed(argv, timeout=10):
|
||||
try:
|
||||
result = subprocess.run(
|
||||
argv,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
env={
|
||||
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"HOME": "/root",
|
||||
"GIT_OPTIONAL_LOCKS": "0",
|
||||
"GIT_TERMINAL_PROMPT": "0",
|
||||
"GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_CONFIG_GLOBAL": "/dev/null",
|
||||
},
|
||||
)
|
||||
return result.returncode, result.stdout.strip()
|
||||
except Exception:
|
||||
return -1, ""
|
||||
|
||||
def inspect_json(kind, name):
|
||||
code, output = run_fixed([
|
||||
"docker", "--host", "unix:///var/run/docker.sock", "inspect", "--type", kind, name
|
||||
], timeout=12)
|
||||
if code != 0:
|
||||
return None
|
||||
try:
|
||||
value = json.loads(output)
|
||||
return value[0] if isinstance(value, list) and len(value) == 1 and isinstance(value[0], dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def mode_allows(path, *, uid, gid, required_bits):
|
||||
"""Conservative POSIX mode check for a fixed non-root runtime identity."""
|
||||
try:
|
||||
info = path.lstat()
|
||||
if path.is_symlink() or not info or not stat.S_ISDIR(info.st_mode):
|
||||
return False
|
||||
mode = stat.S_IMODE(info.st_mode)
|
||||
if info.st_uid == uid:
|
||||
bits = (mode >> 6) & 0o7
|
||||
elif info.st_gid == gid:
|
||||
bits = (mode >> 3) & 0o7
|
||||
else:
|
||||
bits = mode & 0o7
|
||||
return bits & required_bits == required_bits
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def directory_readable_as(path, *, uid, gid):
|
||||
"""Check traversal of every parent and read/traverse rights at the leaf."""
|
||||
absolute = path.absolute()
|
||||
chain = list(reversed(absolute.parents)) + [absolute]
|
||||
for item in chain[:-1]:
|
||||
if not mode_allows(item, uid=uid, gid=gid, required_bits=0o1):
|
||||
return False
|
||||
return mode_allows(chain[-1], uid=uid, gid=gid, required_bits=0o5)
|
||||
|
||||
def repository_facts():
|
||||
projects = read_json(REGISTRY)
|
||||
entry = projects.get(REPOSITORY) if isinstance(projects, dict) else None
|
||||
registry = {"present": isinstance(projects, dict) and REPOSITORY in projects}
|
||||
try:
|
||||
registry["alias_matches"] = (
|
||||
not REGISTRY.is_symlink()
|
||||
and not HOST_REGISTRY.is_symlink()
|
||||
and REGISTRY.is_file()
|
||||
and HOST_REGISTRY.is_file()
|
||||
and os.path.samefile(REGISTRY, HOST_REGISTRY)
|
||||
)
|
||||
except Exception:
|
||||
registry["alias_matches"] = False
|
||||
if isinstance(entry, dict):
|
||||
registry["entry_type"] = "object"
|
||||
registry["keys"] = sorted(str(key) for key in entry.keys())
|
||||
registry["target"] = {
|
||||
key: entry.get(key)
|
||||
for key in (
|
||||
"branch", "slug", "source_dir", "exact_source", "env_source",
|
||||
"compose_files", "compose_project", "containers",
|
||||
"verify_oci_revision", "health_timeout",
|
||||
)
|
||||
}
|
||||
environment = entry.get("environment")
|
||||
registry["environment_keys"] = (
|
||||
sorted(str(key) for key in environment)
|
||||
if isinstance(environment, dict)
|
||||
else [] if "environment" not in entry else None
|
||||
)
|
||||
validation = entry.get("validation")
|
||||
registry["validation"] = validation if isinstance(validation, dict) else None
|
||||
registry["repository_id"] = entry.get("gitea_repository_id")
|
||||
registry["webhook_secret_file_matches"] = entry.get("gitea_webhook_secret_file") == "jens__repopulse"
|
||||
else:
|
||||
registry["entry_type"] = "missing" if entry is None else type(entry).__name__
|
||||
|
||||
state = read_json(STATE)
|
||||
if isinstance(state, dict):
|
||||
state = {
|
||||
key: state.get(key)
|
||||
for key in (
|
||||
"schema_version", "repository", "revision", "live_revision",
|
||||
"live_status", "live_legacy_provenance", "status",
|
||||
)
|
||||
}
|
||||
else:
|
||||
state = None
|
||||
|
||||
live_revision = state.get("live_revision") if isinstance(state, dict) else None
|
||||
git = {"bare_repo_present": BARE_REPO.is_dir() and not BARE_REPO.is_symlink()}
|
||||
main_code, main_sha = run_fixed(["git", f"--git-dir={BARE_REPO}", "rev-parse", "refs/heads/main"])
|
||||
git["main_revision"] = main_sha if main_code == 0 and SHA.fullmatch(main_sha) else None
|
||||
git["live_revision_resolves"] = False
|
||||
git["live_is_ancestor"] = False
|
||||
git["compose_present_on_main"] = False
|
||||
if isinstance(live_revision, str) and SHA.fullmatch(live_revision) and git["main_revision"]:
|
||||
resolve_code, resolved = run_fixed([
|
||||
"git", f"--git-dir={BARE_REPO}", "rev-parse", f"{live_revision}^{{commit}}"
|
||||
])
|
||||
git["live_revision_resolves"] = resolve_code == 0 and resolved == live_revision
|
||||
if git["live_revision_resolves"]:
|
||||
ancestor_code, _ = run_fixed([
|
||||
"git", f"--git-dir={BARE_REPO}", "merge-base", "--is-ancestor",
|
||||
live_revision, git["main_revision"],
|
||||
])
|
||||
git["live_is_ancestor"] = ancestor_code == 0
|
||||
compose_code, _ = run_fixed([
|
||||
"git", f"--git-dir={BARE_REPO}", "cat-file", "-e",
|
||||
f"{git['main_revision']}:docker-compose.yml",
|
||||
])
|
||||
git["compose_present_on_main"] = compose_code == 0
|
||||
git["live_revision"] = live_revision if isinstance(live_revision, str) and SHA.fullmatch(live_revision) else None
|
||||
|
||||
container = inspect_json("container", CONTAINER)
|
||||
runtime = None
|
||||
if isinstance(container, dict):
|
||||
config = container.get("Config") if isinstance(container.get("Config"), dict) else {}
|
||||
host_config = container.get("HostConfig") if isinstance(container.get("HostConfig"), dict) else {}
|
||||
state_payload = container.get("State") if isinstance(container.get("State"), dict) else {}
|
||||
labels = config.get("Labels") if isinstance(config.get("Labels"), dict) else {}
|
||||
mounts = []
|
||||
for mount in container.get("Mounts", []):
|
||||
if isinstance(mount, dict) and mount.get("Destination") in ("/data", "/scan"):
|
||||
mounts.append({
|
||||
"destination": mount.get("Destination"),
|
||||
"type": mount.get("Type"),
|
||||
"name": mount.get("Name"),
|
||||
"source": mount.get("Source"),
|
||||
"read_write": mount.get("RW"),
|
||||
})
|
||||
envs = config.get("Env") if isinstance(config.get("Env"), list) else []
|
||||
operator_token_configured = any(
|
||||
isinstance(value, str) and value.startswith("REPOPULSE_OPERATOR_TOKEN=")
|
||||
and bool(value.split("=", 1)[1])
|
||||
for value in envs
|
||||
)
|
||||
bindings = host_config.get("PortBindings") if isinstance(host_config.get("PortBindings"), dict) else {}
|
||||
host_ports = []
|
||||
for _container_port, endpoints in bindings.items():
|
||||
if not isinstance(endpoints, list):
|
||||
continue
|
||||
for endpoint in endpoints:
|
||||
if isinstance(endpoint, dict):
|
||||
ip, port = endpoint.get("HostIp"), endpoint.get("HostPort")
|
||||
if isinstance(port, str) and port.isdigit():
|
||||
host_ports.append({"ip": (ip or "0.0.0.0")[:64] if isinstance(ip, str) else "unknown", "port": int(port)})
|
||||
security_options = host_config.get("SecurityOpt") if isinstance(host_config.get("SecurityOpt"), list) else []
|
||||
runtime = {
|
||||
"name_matches": container.get("Name") == "/repopulse",
|
||||
"compose_project_matches": labels.get("com.docker.compose.project") == "repopulse",
|
||||
"container_id_sha256": (
|
||||
hashlib.sha256(container["Id"].encode("ascii")).hexdigest()
|
||||
if isinstance(container.get("Id"), str)
|
||||
and re.fullmatch(r"[0-9a-f]{64}", container["Id"])
|
||||
else None
|
||||
),
|
||||
"image_sha256": (
|
||||
hashlib.sha256(config["Image"].encode("utf-8")).hexdigest()
|
||||
if isinstance(config.get("Image"), str) and len(config["Image"]) <= 160
|
||||
else None
|
||||
),
|
||||
"state": state_payload.get("Status") if state_payload.get("Status") in ("running", "exited", "created", "restarting", "paused", "dead") else "unknown",
|
||||
"health": (state_payload.get("Health") or {}).get("Status") if isinstance(state_payload.get("Health"), dict) else "missing",
|
||||
"image": config.get("Image") if isinstance(config.get("Image"), str) and len(config.get("Image")) <= 160 else None,
|
||||
"network_mode": host_config.get("NetworkMode") if isinstance(host_config.get("NetworkMode"), str) and len(host_config.get("NetworkMode")) <= 160 else "unknown",
|
||||
"mounts": mounts,
|
||||
"read_only_root": host_config.get("ReadonlyRootfs") is True,
|
||||
"cap_drop_all": "ALL" in (host_config.get("CapDrop") if isinstance(host_config.get("CapDrop"), list) else []),
|
||||
"no_new_privileges": any(isinstance(value, str) and value.casefold() == "no-new-privileges:true" for value in security_options),
|
||||
"host_ports": sorted(host_ports, key=lambda item: (item["ip"], item["port"])),
|
||||
"operator_token_configured": operator_token_configured,
|
||||
}
|
||||
|
||||
volume = inspect_json("volume", VOLUME)
|
||||
volume_facts = None
|
||||
if isinstance(volume, dict):
|
||||
volume_facts = {
|
||||
"name_matches": volume.get("Name") == VOLUME,
|
||||
"mountpoint_matches": volume.get("Mountpoint") == "/var/lib/docker/volumes/repopulse_data/_data",
|
||||
}
|
||||
|
||||
try:
|
||||
root_ok = APPDATA_ROOT.is_dir() and not APPDATA_ROOT.is_symlink()
|
||||
unexpected_entry_count = 0
|
||||
if root_ok:
|
||||
# At most four names are needed to check the expected appdata
|
||||
# entries without materializing a large directory listing.
|
||||
with os.scandir(APPDATA_ROOT) as entries:
|
||||
for index, item in enumerate(entries):
|
||||
if item.name not in {"deploy.env", "deploy.managed.env", "data", "sources"}:
|
||||
unexpected_entry_count = 1
|
||||
break
|
||||
if index >= 3:
|
||||
unexpected_entry_count = 1
|
||||
break
|
||||
env_path = APPDATA_ROOT / "deploy.env"
|
||||
managed_env_path = APPDATA_ROOT / "deploy.managed.env"
|
||||
data_path = APPDATA_ROOT / "data"
|
||||
sources_path = APPDATA_ROOT / "sources"
|
||||
sources_exists = sources_path.exists() or sources_path.is_symlink()
|
||||
appdata = {
|
||||
"root_is_safe_directory": root_ok,
|
||||
"deploy_env_is_regular_file": env_path.is_file() and not env_path.is_symlink(),
|
||||
"managed_env_exists": managed_env_path.exists() or managed_env_path.is_symlink(),
|
||||
"managed_env_is_regular_file": managed_env_path.is_file() and not managed_env_path.is_symlink(),
|
||||
"managed_env_is_private": (
|
||||
managed_env_path.is_file()
|
||||
and not managed_env_path.is_symlink()
|
||||
and (
|
||||
os.name == "nt"
|
||||
or (
|
||||
managed_env_path.stat().st_uid == os.geteuid()
|
||||
and stat.S_IMODE(managed_env_path.stat().st_mode) & 0o077 == 0
|
||||
)
|
||||
)
|
||||
),
|
||||
"target_data_exists": data_path.exists() or data_path.is_symlink(),
|
||||
"target_data_is_symlink": data_path.is_symlink(),
|
||||
"sources_exists": sources_exists,
|
||||
"sources_is_safe_directory": not sources_exists or (sources_path.is_dir() and not sources_path.is_symlink()),
|
||||
"unexpected_entry_count": unexpected_entry_count,
|
||||
}
|
||||
except Exception:
|
||||
appdata = None
|
||||
|
||||
scan_access = {
|
||||
"runtime_uid": 10001,
|
||||
"runtime_gid": 10001,
|
||||
"root_readable": directory_readable_as(SCAN_ROOT, uid=10001, gid=10001),
|
||||
"repopulse_source_readable": directory_readable_as(SCAN_ROOT / "repopulse", uid=10001, gid=10001),
|
||||
"privatemuse_source_readable": directory_readable_as(SCAN_ROOT / "privatemuse", uid=10001, gid=10001),
|
||||
}
|
||||
|
||||
health = None
|
||||
meta = None
|
||||
try:
|
||||
connection = http.client.HTTPConnection("127.0.0.1", 1254, timeout=5)
|
||||
connection.request("GET", "/health", headers={"Connection": "close"})
|
||||
response = connection.getresponse()
|
||||
body = response.read(8192)
|
||||
health_payload = json.loads(body.decode("utf-8")) if response.status == 200 else None
|
||||
health = {
|
||||
"http_ok": response.status == 200,
|
||||
"status": health_payload.get("status") if isinstance(health_payload, dict) and health_payload.get("status") in ("ok", "degraded") else "unknown",
|
||||
}
|
||||
connection.close()
|
||||
|
||||
connection = http.client.HTTPConnection("127.0.0.1", 1254, timeout=5)
|
||||
connection.request("GET", "/api/v1/meta", headers={"Connection": "close"})
|
||||
response = connection.getresponse()
|
||||
body = response.read(65536)
|
||||
payload = json.loads(body.decode("utf-8")) if response.status == 200 else None
|
||||
connection.close()
|
||||
if isinstance(payload, dict):
|
||||
def safe_int(value):
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
|
||||
repo_states = payload.get("repository_states") if isinstance(payload.get("repository_states"), dict) else {}
|
||||
storage = payload.get("storage") if isinstance(payload.get("storage"), dict) else {}
|
||||
recovery = payload.get("database_recovery") if isinstance(payload.get("database_recovery"), dict) else {}
|
||||
meta = {
|
||||
"http_ok": response.status == 200,
|
||||
"product_matches": payload.get("name") == "RepoPulse",
|
||||
"repository_count": safe_int(payload.get("repository_count")),
|
||||
"repository_states": {key: safe_int(repo_states.get(key)) for key in ("current", "stale", "queued", "scanning", "failed", "never_scanned", "archived", "unavailable")},
|
||||
"queue": {key: safe_int(payload.get(name)) for key, name in (("queued", "queue_depth"), ("active", "active_jobs"), ("completed", "completed_jobs"), ("failed", "failed_jobs"))},
|
||||
"storage": {key: safe_int(storage.get(key)) for key in ("database_bytes", "journal_bytes", "scan_count", "compacted_scan_count", "mirror_count", "mirror_cache_bytes", "worktree_count")},
|
||||
"database_recovery_status": recovery.get("status") if recovery.get("status") in ("healthy", "acknowledged", "degraded_recovered", "unknown") else "unknown",
|
||||
"mutation_authentication": payload.get("mutation_authentication") if payload.get("mutation_authentication") in ("operator_token", "perimeter_required") else "unknown",
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
safe = {
|
||||
"registry": registry,
|
||||
"state": state,
|
||||
"git": git,
|
||||
"runtime": runtime,
|
||||
"volume": volume_facts,
|
||||
"appdata": appdata,
|
||||
"scan_access": scan_access,
|
||||
"health": health,
|
||||
"meta": meta,
|
||||
}
|
||||
print(json.dumps(safe, sort_keys=True, separators=(",", ":")))
|
||||
|
||||
repository_facts()
|
||||
'''
|
||||
|
||||
|
||||
def _check(code: str, passed: bool | None, detail: str) -> dict[str, str]:
|
||||
status = "passed" if passed is True else "blocked" if passed is False else "unknown"
|
||||
return {"code": code, "status": status, "detail": detail}
|
||||
|
||||
|
||||
def _unavailable_report(code: str, detail: str) -> dict[str, Any]:
|
||||
check = _check(code, False, detail)
|
||||
check["gate"] = "legacy_compatibility"
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption-preflight/v1",
|
||||
"repository": _REPOSITORY,
|
||||
"status": "unavailable",
|
||||
"legacyCompatibility": {"status": "unavailable", "checks": [check]},
|
||||
"targetAcceptance": {
|
||||
"status": "not_assessed",
|
||||
"checks": [],
|
||||
"externalGates": ["Verify the exact authenticated NPM HTTPS route and access list separately."],
|
||||
},
|
||||
"readOnly": True,
|
||||
"adoptionAuthorized": False,
|
||||
"checks": [check],
|
||||
"evidence": {},
|
||||
}
|
||||
|
||||
|
||||
def _is_nonnegative_int(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||
|
||||
|
||||
def _matches_string_set(value: object, expected: set[str]) -> bool:
|
||||
return (
|
||||
isinstance(value, list)
|
||||
and all(isinstance(item, str) for item in value)
|
||||
and len(value) == len(expected)
|
||||
and set(value) == expected
|
||||
)
|
||||
|
||||
|
||||
def _same_json(left: object, right: object) -> bool:
|
||||
try:
|
||||
return json.dumps(left, sort_keys=True, separators=(",", ":"), allow_nan=False) == json.dumps(
|
||||
right, sort_keys=True, separators=(",", ":"), allow_nan=False
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_facts(facts: object) -> dict[str, Any]:
|
||||
"""Evaluate a sanitized snapshot; never trusts caller-provided paths or commands."""
|
||||
if not isinstance(facts, dict):
|
||||
facts = {}
|
||||
legacy_checks: list[dict[str, str]] = []
|
||||
acceptance_checks: list[dict[str, str]] = []
|
||||
|
||||
def add_check(
|
||||
target: list[dict[str, str]], gate: str, code: str, passed: bool | None, detail: str
|
||||
) -> None:
|
||||
result = _check(code, passed, detail)
|
||||
result["gate"] = gate
|
||||
target.append(result)
|
||||
|
||||
registry = facts.get("registry") if isinstance(facts.get("registry"), dict) else {}
|
||||
registry_target = registry.get("target") if isinstance(registry.get("target"), dict) else {}
|
||||
expected_registry_target = {
|
||||
key: value for key, value in _EXPECTED_TARGET.items() if key != "environment_keys"
|
||||
}
|
||||
registry_matches = (
|
||||
registry.get("present") is True
|
||||
and registry.get("alias_matches") is True
|
||||
and registry.get("entry_type") == "object"
|
||||
and _matches_string_set(registry.get("keys"), _EXPECTED_REGISTRY_KEYS)
|
||||
and _same_json(registry_target, expected_registry_target)
|
||||
and registry.get("environment_keys") == []
|
||||
and _same_json(registry.get("validation"), _EXPECTED_VALIDATION)
|
||||
and registry.get("repository_id") == 64
|
||||
and registry.get("webhook_secret_file_matches") is True
|
||||
)
|
||||
add_check(legacy_checks, "legacy_compatibility", "legacy_registry", registry_matches, "The controller registry must match the fixed jens/repopulse target and runnerless validation policy.")
|
||||
|
||||
state = facts.get("state") if isinstance(facts.get("state"), dict) else {}
|
||||
live_revision = state.get("live_revision")
|
||||
state_matches = (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == _REPOSITORY
|
||||
and isinstance(state.get("revision"), str)
|
||||
and state.get("revision") == live_revision
|
||||
and isinstance(live_revision, str)
|
||||
and _SHA.fullmatch(live_revision) is not None
|
||||
and state.get("live_status") == "healthy"
|
||||
and state.get("status") == "success"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
)
|
||||
add_check(legacy_checks, "legacy_compatibility", "controller_state", state_matches, "The current controller state must be a healthy schema v3 success for the fixed repository.")
|
||||
|
||||
git = facts.get("git") if isinstance(facts.get("git"), dict) else {}
|
||||
main_revision = git.get("main_revision")
|
||||
git_matches = (
|
||||
git.get("bare_repo_present") is True
|
||||
and isinstance(main_revision, str)
|
||||
and _SHA.fullmatch(main_revision) is not None
|
||||
and git.get("live_revision") == live_revision
|
||||
and git.get("live_revision_resolves") is True
|
||||
and git.get("live_is_ancestor") is True
|
||||
and git.get("compose_present_on_main") is True
|
||||
)
|
||||
add_check(legacy_checks, "legacy_compatibility", "gitea_ancestry", git_matches, "The recorded live SHA must resolve in Gitea, be an ancestor of the current main SHA, and main must contain the required Compose file.")
|
||||
|
||||
runtime = facts.get("runtime") if isinstance(facts.get("runtime"), dict) else {}
|
||||
runtime_identity = _check(
|
||||
"runtime_identity_and_health",
|
||||
runtime.get("name_matches") is True
|
||||
and runtime.get("compose_project_matches") is True
|
||||
and isinstance(runtime.get("container_id_sha256"), str)
|
||||
and _SHA256.fullmatch(runtime["container_id_sha256"]) is not None
|
||||
and isinstance(runtime.get("image_sha256"), str)
|
||||
and _SHA256.fullmatch(runtime["image_sha256"]) is not None
|
||||
and isinstance(runtime.get("image"), str)
|
||||
and runtime.get("image").split("@", 1)[0].split(":", 1)[0] == "repopulse-repopulse"
|
||||
and runtime.get("state") == "running"
|
||||
and runtime.get("health") == "healthy",
|
||||
"The exact repopulse container must be running and healthy under the repopulse Compose project.",
|
||||
)
|
||||
runtime_identity["gate"] = "legacy_compatibility"
|
||||
legacy_checks.append(runtime_identity)
|
||||
mounts = runtime.get("mounts") if isinstance(runtime.get("mounts"), list) else []
|
||||
data_mount = next((mount for mount in mounts if isinstance(mount, dict) and mount.get("destination") == "/data"), {})
|
||||
scan_mount = next((mount for mount in mounts if isinstance(mount, dict) and mount.get("destination") == "/scan"), {})
|
||||
volume = facts.get("volume") if isinstance(facts.get("volume"), dict) else {}
|
||||
storage_matches = (
|
||||
data_mount.get("type") == "volume"
|
||||
and data_mount.get("name") == _VOLUME
|
||||
and data_mount.get("read_write") is True
|
||||
and scan_mount.get("type") == "bind"
|
||||
and scan_mount.get("source") == _SOURCE_DIR.replace("/repopulse", "")
|
||||
and scan_mount.get("read_write") is False
|
||||
and volume.get("name_matches") is True
|
||||
and volume.get("mountpoint_matches") is True
|
||||
)
|
||||
add_check(legacy_checks, "legacy_compatibility", "legacy_storage_topology", storage_matches, "The legacy data volume and read-only scan source must match the audited RepoPulse mounts.")
|
||||
|
||||
safe_runtime = (
|
||||
runtime.get("read_only_root") is True
|
||||
and runtime.get("cap_drop_all") is True
|
||||
and runtime.get("no_new_privileges") is True
|
||||
)
|
||||
add_check(acceptance_checks, "target_acceptance", "runtime_hardening", safe_runtime, "Managed cutover requires a read-only root filesystem, all capabilities dropped, and no-new-privileges; current legacy settings are reported without failing compatibility.")
|
||||
|
||||
host_ports = runtime.get("host_ports") if isinstance(runtime.get("host_ports"), list) else []
|
||||
add_check(acceptance_checks, "target_acceptance", "host_exposure", len(host_ports) == 0, "Managed cutover must remove the legacy published host port; current bindings are included in the evidence.")
|
||||
add_check(acceptance_checks, "target_acceptance", "managed_network", isinstance(runtime.get("network_mode"), str) and runtime.get("network_mode") not in ("host", "unknown"), "Managed cutover requires an inspected private network instead of host networking.")
|
||||
|
||||
appdata = facts.get("appdata") if isinstance(facts.get("appdata"), dict) else {}
|
||||
sources_safe = (
|
||||
isinstance(appdata.get("sources_exists"), bool)
|
||||
and appdata.get("sources_is_safe_directory") is True
|
||||
)
|
||||
managed_env_exists = appdata.get("managed_env_exists")
|
||||
managed_env_is_regular_file = appdata.get("managed_env_is_regular_file")
|
||||
managed_env_is_private = appdata.get("managed_env_is_private")
|
||||
managed_env_safe = (
|
||||
isinstance(managed_env_exists, bool)
|
||||
and isinstance(managed_env_is_regular_file, bool)
|
||||
and isinstance(managed_env_is_private, bool)
|
||||
and (
|
||||
(
|
||||
managed_env_exists is False
|
||||
and managed_env_is_regular_file is False
|
||||
and managed_env_is_private is False
|
||||
)
|
||||
or (
|
||||
managed_env_exists is True
|
||||
and managed_env_is_regular_file is True
|
||||
and managed_env_is_private is True
|
||||
)
|
||||
)
|
||||
)
|
||||
legacy_appdata_clean = (
|
||||
appdata.get("root_is_safe_directory") is True
|
||||
and appdata.get("deploy_env_is_regular_file") is True
|
||||
and sources_safe
|
||||
and managed_env_safe
|
||||
and _is_nonnegative_int(appdata.get("unexpected_entry_count"))
|
||||
and appdata.get("unexpected_entry_count") == 0
|
||||
)
|
||||
add_check(legacy_checks, "legacy_compatibility", "legacy_appdata", legacy_appdata_clean, "The legacy appdata root must retain its regular deploy.env and contain no unexpected entries; optional sources and a private managed environment file are allowed.")
|
||||
data_target_clear = appdata.get("target_data_exists") is False and appdata.get("target_data_is_symlink") is False
|
||||
add_check(acceptance_checks, "target_acceptance", "managed_data_target", data_target_clear, "The managed appdata/data target must be absent before the reviewed storage migration; an existing target is reported as a collision.")
|
||||
add_check(
|
||||
acceptance_checks,
|
||||
"target_acceptance",
|
||||
"managed_environment_file",
|
||||
managed_env_exists is True and managed_env_is_regular_file is True and managed_env_is_private is True,
|
||||
"The fixed deploy.managed.env target file must be a private regular non-symlink file; no environment values are read or returned.",
|
||||
)
|
||||
|
||||
scan_access = facts.get("scan_access") if isinstance(facts.get("scan_access"), dict) else {}
|
||||
scan_access_ready = (
|
||||
scan_access.get("runtime_uid") == 10001
|
||||
and scan_access.get("runtime_gid") == 10001
|
||||
and scan_access.get("root_readable") is True
|
||||
and scan_access.get("repopulse_source_readable") is True
|
||||
and scan_access.get("privatemuse_source_readable") is True
|
||||
)
|
||||
add_check(
|
||||
acceptance_checks,
|
||||
"target_acceptance",
|
||||
"scan_access",
|
||||
scan_access_ready,
|
||||
"The fixed RepoPulse and PrivateMuse scan directories must be readable and traversable by runtime UID/GID 10001, with the permission policy retained after atomic source synchronization.",
|
||||
)
|
||||
|
||||
health = facts.get("health") if isinstance(facts.get("health"), dict) else {}
|
||||
meta = facts.get("meta") if isinstance(facts.get("meta"), dict) else {}
|
||||
health_matches = health.get("http_ok") is True and health.get("status") == "ok" and meta.get("http_ok") is True and meta.get("product_matches") is True
|
||||
add_check(legacy_checks, "legacy_compatibility", "product_api", health_matches, "The loopback product health and metadata endpoints must return healthy RepoPulse facts.")
|
||||
|
||||
recovery_status = meta.get("database_recovery_status")
|
||||
add_check(legacy_checks, "legacy_compatibility", "database_health", recovery_status in ("healthy", "acknowledged"), "The live product must report a healthy or acknowledged database recovery state.")
|
||||
queue = meta.get("queue") if isinstance(meta.get("queue"), dict) else {}
|
||||
queue_facts_present = all(_is_nonnegative_int(queue.get(key)) for key in ("queued", "active", "completed", "failed"))
|
||||
add_check(legacy_checks, "legacy_compatibility", "queue_quiescence", queue_facts_present and queue.get("queued") == 0 and queue.get("active") == 0, "Queue and active-job counts must be available and both zero for this legacy compatibility snapshot.")
|
||||
storage = meta.get("storage") if isinstance(meta.get("storage"), dict) else {}
|
||||
db_facts_present = all(_is_nonnegative_int(storage.get(key)) for key in ("database_bytes", "journal_bytes", "scan_count", "compacted_scan_count"))
|
||||
add_check(legacy_checks, "legacy_compatibility", "database_baseline", db_facts_present and storage.get("database_bytes", 0) > 0 and storage.get("scan_count", -1) >= storage.get("compacted_scan_count", 0), "The live product must provide safe aggregate SQLite size and scan-history counts.")
|
||||
|
||||
auth_mode = meta.get("mutation_authentication")
|
||||
operator_token_configured = auth_mode == "operator_token"
|
||||
add_check(acceptance_checks, "target_acceptance", "mutation_boundary", operator_token_configured, "Managed mutation access requires an operator token or a separately verified authenticated NPM perimeter; this preflight does not inspect NPM.")
|
||||
|
||||
legacy_status = "passed" if legacy_checks and all(item["status"] == "passed" for item in legacy_checks) else "blocked"
|
||||
acceptance_status = "blocked" if any(item["status"] != "passed" for item in acceptance_checks) else "external_verification_required"
|
||||
status = "preflight_passed" if legacy_status == "passed" else "blocked"
|
||||
evidence = {
|
||||
"main_revision": main_revision if isinstance(main_revision, str) and _SHA.fullmatch(main_revision) else None,
|
||||
"live_revision": live_revision if isinstance(live_revision, str) and _SHA.fullmatch(live_revision) else None,
|
||||
"container_state": runtime.get("state") if runtime.get("state") in ("running", "exited", "created", "restarting", "paused", "dead", "unknown") else "unknown",
|
||||
"container_health": runtime.get("health") if runtime.get("health") in ("healthy", "unhealthy", "starting", "missing", "unknown") else "unknown",
|
||||
"container_id_sha256": runtime.get("container_id_sha256") if isinstance(runtime.get("container_id_sha256"), str) and _SHA256.fullmatch(runtime["container_id_sha256"]) else None,
|
||||
"container_image_sha256": runtime.get("image_sha256") if isinstance(runtime.get("image_sha256"), str) and _SHA256.fullmatch(runtime["image_sha256"]) else None,
|
||||
"host_ports": [item for item in host_ports if isinstance(item, dict) and isinstance(item.get("port"), int) and not isinstance(item.get("port"), bool) and isinstance(item.get("ip"), str)][:16],
|
||||
"database": {
|
||||
key: storage.get(key) for key in ("database_bytes", "journal_bytes", "scan_count", "compacted_scan_count", "mirror_count", "mirror_cache_bytes", "worktree_count") if _is_nonnegative_int(storage.get(key))
|
||||
},
|
||||
"queue": {key: queue.get(key) for key in ("queued", "active", "completed", "failed") if _is_nonnegative_int(queue.get(key))},
|
||||
"appdata_target_exists": appdata.get("target_data_exists") if isinstance(appdata.get("target_data_exists"), bool) else None,
|
||||
"appdata_sources_exists": appdata.get("sources_exists") if isinstance(appdata.get("sources_exists"), bool) else None,
|
||||
"appdata_sources_safe_directory": appdata.get("sources_is_safe_directory") if isinstance(appdata.get("sources_is_safe_directory"), bool) else None,
|
||||
"managed_environment_file_ready": managed_env_exists is True and managed_env_is_regular_file is True and managed_env_is_private is True,
|
||||
"runtime_hardening": {
|
||||
key: runtime.get(key) if isinstance(runtime.get(key), bool) else None
|
||||
for key in ("read_only_root", "cap_drop_all", "no_new_privileges")
|
||||
},
|
||||
"network_mode": runtime.get("network_mode") if isinstance(runtime.get("network_mode"), str) else "unknown",
|
||||
"operator_token_configured": operator_token_configured,
|
||||
"scan_access_uid_10001": scan_access_ready,
|
||||
}
|
||||
checks = legacy_checks + acceptance_checks
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption-preflight/v1",
|
||||
"repository": _REPOSITORY,
|
||||
"status": status,
|
||||
"legacyCompatibility": {"status": legacy_status, "checks": legacy_checks},
|
||||
"targetAcceptance": {
|
||||
"status": acceptance_status,
|
||||
"checks": acceptance_checks,
|
||||
"externalGates": ["Verify the exact authenticated NPM HTTPS route and access list separately."],
|
||||
},
|
||||
"readOnly": True,
|
||||
"adoptionAuthorized": False,
|
||||
"checks": checks,
|
||||
"evidence": evidence,
|
||||
}
|
||||
|
||||
|
||||
class RepoPulseAdoptionPreflight:
|
||||
"""Collect fixed live facts through the controller and evaluate read-only."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return _unavailable_report("controller", "The fixed deployment controller is unavailable.")
|
||||
controller = status.get("container") if isinstance(status.get("container"), dict) else {}
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return _unavailable_report("controller", "The fixed deployment controller is not running.")
|
||||
controller_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not controller_id or not config:
|
||||
return _unavailable_report("controller", "Portainer deployment control is unavailable.")
|
||||
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(controller_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": True,
|
||||
"AttachStderr": False,
|
||||
"Tty": True,
|
||||
"Cmd": ["python3", "-c", _PREFLIGHT_SCRIPT],
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
raise ValueError("exec unavailable")
|
||||
raw_response = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{quote(exec_id, safe='')}/start",
|
||||
json={"Detach": False, "Tty": True},
|
||||
timeout=60.0,
|
||||
)
|
||||
raw = (raw_response.text or "").strip()
|
||||
if not raw or len(raw) > 65536:
|
||||
raise ValueError("preflight output unavailable")
|
||||
facts = json.loads(raw)
|
||||
except (RuntimeError, ValueError, TypeError, json.JSONDecodeError):
|
||||
return _unavailable_report("collector", "The fixed read-only collector failed.")
|
||||
return evaluate_facts(facts)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point for an operator-run, read-only preflight report."""
|
||||
result = RepoPulseAdoptionPreflight().run()
|
||||
print(json.dumps(result, sort_keys=True, separators=(",", ":")))
|
||||
return 0 if result.get("status") == "preflight_passed" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,984 @@
|
||||
"""Pure state and evidence rules for the fixed RepoPulse legacy adoption.
|
||||
|
||||
This module deliberately has no Portainer, filesystem, Docker, database, or
|
||||
controller client. The integrated operator runner uses these rules only after
|
||||
the fixed managed profile and controller-owned durable intent are verified.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
from backend.app.services.enrollment_profiles import PROFILE_VERSION
|
||||
|
||||
PROTOCOL_VERSION = "itworx.repopulse-legacy-adoption/v1"
|
||||
INTENT_VERSION = "itworx.repopulse-legacy-adoption-intent/v1"
|
||||
CONTROLLER_RECEIPT_VERSION = "itworx.repopulse-legacy-adoption-receipt/v1"
|
||||
REPOSITORY = "jens/repopulse"
|
||||
APPLICATION_ID = "repopulse"
|
||||
LEGACY_VOLUME = "repopulse_data"
|
||||
EXECUTION_ENABLED = True
|
||||
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_SHA1 = re.compile(r"^[0-9a-f]{40}$")
|
||||
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
_PROFILE_ID = "repopulse-authenticated"
|
||||
_ENROLLMENT_ID = re.compile(r"^enr-[0-9a-f]{20}$")
|
||||
_INTENT_ID = re.compile(r"^rpa-[0-9a-f]{20}$")
|
||||
|
||||
_LEGACY_CHECKS = {
|
||||
"legacy_registry",
|
||||
"controller_state",
|
||||
"gitea_ancestry",
|
||||
"runtime_identity_and_health",
|
||||
"legacy_storage_topology",
|
||||
"legacy_appdata",
|
||||
"product_api",
|
||||
"database_health",
|
||||
"queue_quiescence",
|
||||
"database_baseline",
|
||||
}
|
||||
_PHASES = (
|
||||
"reserved",
|
||||
"target_gates_ready",
|
||||
"quiesced",
|
||||
"snapshot_verified",
|
||||
"target_registered",
|
||||
"appops_owned",
|
||||
"completed",
|
||||
)
|
||||
_STEP_CONTRACTS: dict[str, tuple[str, str]] = {
|
||||
"target_gates": ("reserved", "target_gates_ready"),
|
||||
"quiescence": ("target_gates_ready", "quiesced"),
|
||||
"snapshot": ("quiesced", "snapshot_verified"),
|
||||
"controller_registration": ("snapshot_verified", "target_registered"),
|
||||
"appops_ownership": ("target_registered", "appops_owned"),
|
||||
}
|
||||
_RECOVERY_REASONS = {
|
||||
"controller_unavailable",
|
||||
"database_backup_failed",
|
||||
"registry_receipt_uncertain",
|
||||
"target_health_failed",
|
||||
"appops_commit_uncertain",
|
||||
"preflight_drift",
|
||||
"operator_interruption",
|
||||
"unknown_external_effect",
|
||||
}
|
||||
_STEP_EVIDENCE_KEYS = {
|
||||
"target_gates": "targetGates",
|
||||
"quiescence": "quiescence",
|
||||
"snapshot": "snapshot",
|
||||
"controller_registration": "controllerRegistration",
|
||||
"appops_ownership": "appopsOwnership",
|
||||
"completion": "completion",
|
||||
}
|
||||
|
||||
|
||||
class AdoptionProtocolError(ValueError):
|
||||
"""A fail-closed protocol or evidence validation error."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _canonical(value: object) -> bytes:
|
||||
try:
|
||||
return json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
allow_nan=False,
|
||||
).encode("ascii")
|
||||
except (TypeError, ValueError, UnicodeEncodeError) as exc:
|
||||
raise AdoptionProtocolError("invalid_evidence_encoding") from exc
|
||||
|
||||
|
||||
def _digest(value: object) -> str:
|
||||
return hashlib.sha256(_canonical(value)).hexdigest()
|
||||
|
||||
|
||||
def _is_count(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||
|
||||
|
||||
def _validate_preflight_projection(value: object) -> dict[str, Any]:
|
||||
required = {
|
||||
"repository",
|
||||
"main_revision",
|
||||
"live_revision",
|
||||
"container_id_sha256",
|
||||
"container_image_sha256",
|
||||
"queue",
|
||||
"database",
|
||||
"container_state",
|
||||
"container_health",
|
||||
}
|
||||
_need(isinstance(value, dict) and set(value) == required, "intent_preflight_invalid")
|
||||
projection = value
|
||||
_need(
|
||||
projection.get("repository") == REPOSITORY
|
||||
and isinstance(projection.get("main_revision"), str)
|
||||
and _SHA1.fullmatch(projection["main_revision"]) is not None
|
||||
and isinstance(projection.get("live_revision"), str)
|
||||
and _SHA1.fullmatch(projection["live_revision"]) is not None
|
||||
and isinstance(projection.get("container_id_sha256"), str)
|
||||
and _SHA256.fullmatch(projection["container_id_sha256"]) is not None
|
||||
and isinstance(projection.get("container_image_sha256"), str)
|
||||
and _SHA256.fullmatch(projection["container_image_sha256"]) is not None
|
||||
and projection.get("container_state") == "running"
|
||||
and projection.get("container_health") == "healthy",
|
||||
"intent_preflight_invalid",
|
||||
)
|
||||
queue = projection.get("queue")
|
||||
database = projection.get("database")
|
||||
_need(
|
||||
isinstance(queue, dict)
|
||||
and set(queue) == {"queued", "active", "completed", "failed"}
|
||||
and all(_is_count(queue.get(key)) for key in queue)
|
||||
and queue.get("queued") == 0
|
||||
and queue.get("active") == 0,
|
||||
"intent_preflight_queue_invalid",
|
||||
)
|
||||
_need(
|
||||
isinstance(database, dict)
|
||||
and set(database)
|
||||
== {"database_bytes", "journal_bytes", "scan_count", "compacted_scan_count"}
|
||||
and all(_is_count(database.get(key)) for key in database)
|
||||
and database.get("database_bytes", 0) > 0
|
||||
and database.get("scan_count", -1) >= database.get("compacted_scan_count", 0),
|
||||
"intent_preflight_database_invalid",
|
||||
)
|
||||
return projection
|
||||
|
||||
|
||||
def _need(condition: bool, code: str) -> None:
|
||||
if not condition:
|
||||
raise AdoptionProtocolError(code)
|
||||
|
||||
|
||||
def _safe_preflight_projection(report: object) -> dict[str, Any]:
|
||||
"""Accept only the existing read-only preflight's allowlisted identity facts."""
|
||||
_need(isinstance(report, dict), "legacy_preflight_missing")
|
||||
source = report
|
||||
_need(
|
||||
source.get("apiVersion") == "itworx.repopulse-legacy-adoption-preflight/v1"
|
||||
and source.get("repository") == REPOSITORY
|
||||
and source.get("status") == "preflight_passed"
|
||||
and source.get("readOnly") is True
|
||||
and source.get("adoptionAuthorized") is False,
|
||||
"legacy_preflight_not_compatible",
|
||||
)
|
||||
compatibility = source.get("legacyCompatibility")
|
||||
_need(
|
||||
isinstance(compatibility, dict)
|
||||
and compatibility.get("status") == "passed",
|
||||
"legacy_preflight_not_compatible",
|
||||
)
|
||||
checks = compatibility.get("checks")
|
||||
_need(isinstance(checks, list), "legacy_preflight_checks_missing")
|
||||
observed: dict[str, str] = {}
|
||||
for check in checks:
|
||||
_need(isinstance(check, dict), "legacy_preflight_check_invalid")
|
||||
code = check.get("code")
|
||||
_need(
|
||||
isinstance(code, str)
|
||||
and code in _LEGACY_CHECKS
|
||||
and check.get("gate") == "legacy_compatibility"
|
||||
and check.get("status") == "passed"
|
||||
and code not in observed,
|
||||
"legacy_preflight_check_failed",
|
||||
)
|
||||
observed[code] = "passed"
|
||||
_need(set(observed) == _LEGACY_CHECKS, "legacy_preflight_check_incomplete")
|
||||
|
||||
evidence = source.get("evidence")
|
||||
_need(isinstance(evidence, dict), "legacy_preflight_evidence_missing")
|
||||
main_revision = evidence.get("main_revision")
|
||||
live_revision = evidence.get("live_revision")
|
||||
queue = evidence.get("queue")
|
||||
database = evidence.get("database")
|
||||
_need(
|
||||
isinstance(main_revision, str)
|
||||
and _SHA1.fullmatch(main_revision) is not None
|
||||
and isinstance(live_revision, str)
|
||||
and _SHA1.fullmatch(live_revision) is not None,
|
||||
"legacy_preflight_revision_invalid",
|
||||
)
|
||||
_need(
|
||||
isinstance(queue, dict)
|
||||
and set(queue) == {"queued", "active", "completed", "failed"}
|
||||
and all(_is_count(queue.get(key)) for key in queue)
|
||||
and queue.get("queued") == 0
|
||||
and queue.get("active") == 0,
|
||||
"legacy_preflight_queue_not_quiescent",
|
||||
)
|
||||
_need(
|
||||
isinstance(database, dict)
|
||||
and all(
|
||||
_is_count(database.get(key))
|
||||
for key in (
|
||||
"database_bytes",
|
||||
"journal_bytes",
|
||||
"scan_count",
|
||||
"compacted_scan_count",
|
||||
)
|
||||
)
|
||||
and database.get("database_bytes", 0) > 0
|
||||
and database.get("scan_count", -1)
|
||||
>= database.get("compacted_scan_count", 0),
|
||||
"legacy_preflight_database_facts_invalid",
|
||||
)
|
||||
_need(
|
||||
evidence.get("container_state") == "running"
|
||||
and evidence.get("container_health") == "healthy",
|
||||
"legacy_preflight_runtime_invalid",
|
||||
)
|
||||
return {
|
||||
"repository": REPOSITORY,
|
||||
"main_revision": main_revision,
|
||||
"live_revision": live_revision,
|
||||
"container_id_sha256": evidence.get("container_id_sha256"),
|
||||
"container_image_sha256": evidence.get("container_image_sha256"),
|
||||
"queue": {
|
||||
key: queue[key]
|
||||
for key in ("queued", "active", "completed", "failed")
|
||||
},
|
||||
"database": {
|
||||
key: database[key]
|
||||
for key in (
|
||||
"database_bytes",
|
||||
"journal_bytes",
|
||||
"scan_count",
|
||||
"compacted_scan_count",
|
||||
)
|
||||
},
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
}
|
||||
|
||||
|
||||
def create_intent(report: object, idempotency_key: str) -> dict[str, Any]:
|
||||
"""Create deterministic fixed-identity intent data from a passing preflight.
|
||||
|
||||
Persistence is deliberately left to the future AppOps/controller adapters;
|
||||
this pure constructor never writes a database or controller state file.
|
||||
"""
|
||||
_need(
|
||||
isinstance(idempotency_key, str)
|
||||
and _IDEMPOTENCY_KEY.fullmatch(idempotency_key) is not None,
|
||||
"idempotency_key_invalid",
|
||||
)
|
||||
projection = _safe_preflight_projection(report)
|
||||
idempotency_sha256 = hashlib.sha256(
|
||||
f"{PROTOCOL_VERSION}:{REPOSITORY}:{idempotency_key}".encode("ascii")
|
||||
).hexdigest()
|
||||
intent_id = f"rpa-{idempotency_sha256[:20]}"
|
||||
intent = {
|
||||
"apiVersion": INTENT_VERSION,
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"intentId": intent_id,
|
||||
"idempotencySha256": idempotency_sha256,
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"phase": "reserved",
|
||||
"status": "in_progress",
|
||||
"preflightSha256": _digest(projection),
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumePreserved": True,
|
||||
"evidence": {"preflight": projection},
|
||||
"receipts": {},
|
||||
"recovery": None,
|
||||
}
|
||||
return _validate_intent(intent)
|
||||
|
||||
|
||||
def reserve_or_replay(
|
||||
existing_intent: object | None, report: object, idempotency_key: str
|
||||
) -> dict[str, Any]:
|
||||
"""Enforce the single fixed adoption slot and exact retry identity."""
|
||||
candidate = create_intent(report, idempotency_key)
|
||||
if existing_intent is None:
|
||||
return candidate
|
||||
existing = _validate_intent(existing_intent)
|
||||
_need(
|
||||
existing["idempotencySha256"] == candidate["idempotencySha256"],
|
||||
"one_time_adoption_already_reserved",
|
||||
)
|
||||
_need(
|
||||
existing["preflightSha256"] == candidate["preflightSha256"],
|
||||
"idempotency_conflict",
|
||||
)
|
||||
return deepcopy(existing)
|
||||
|
||||
|
||||
def _validate_intent(intent: object) -> dict[str, Any]:
|
||||
required = {
|
||||
"apiVersion",
|
||||
"protocolVersion",
|
||||
"intentId",
|
||||
"idempotencySha256",
|
||||
"repository",
|
||||
"applicationId",
|
||||
"phase",
|
||||
"status",
|
||||
"preflightSha256",
|
||||
"legacyVolume",
|
||||
"legacyVolumePreserved",
|
||||
"evidence",
|
||||
"receipts",
|
||||
"recovery",
|
||||
}
|
||||
_need(isinstance(intent, dict) and set(intent) == required, "intent_shape_invalid")
|
||||
value = intent
|
||||
_need(
|
||||
value.get("apiVersion") == INTENT_VERSION
|
||||
and value.get("protocolVersion") == PROTOCOL_VERSION
|
||||
and value.get("repository") == REPOSITORY
|
||||
and value.get("applicationId") == APPLICATION_ID
|
||||
and value.get("legacyVolume") == LEGACY_VOLUME
|
||||
and value.get("legacyVolumePreserved") is True
|
||||
and isinstance(value.get("intentId"), str)
|
||||
and _INTENT_ID.fullmatch(value["intentId"]) is not None
|
||||
and isinstance(value.get("idempotencySha256"), str)
|
||||
and _SHA256.fullmatch(value["idempotencySha256"]) is not None
|
||||
and value["intentId"] == f"rpa-{value['idempotencySha256'][:20]}"
|
||||
and isinstance(value.get("preflightSha256"), str)
|
||||
and _SHA256.fullmatch(value["preflightSha256"]) is not None
|
||||
and value.get("phase") in _PHASES
|
||||
and value.get("status") in {"in_progress", "recovery_required", "completed"}
|
||||
and isinstance(value.get("evidence"), dict)
|
||||
and isinstance(value.get("receipts"), dict),
|
||||
"intent_identity_invalid",
|
||||
)
|
||||
_need(
|
||||
value.get("status") != "completed" or value.get("phase") == "completed",
|
||||
"intent_terminal_state_invalid",
|
||||
)
|
||||
evidence = value["evidence"]
|
||||
_need(
|
||||
set(evidence) <= {"preflight", *_STEP_EVIDENCE_KEYS.values()}
|
||||
and isinstance(evidence.get("preflight"), dict)
|
||||
and _digest(evidence["preflight"]) == value["preflightSha256"],
|
||||
"intent_evidence_invalid",
|
||||
)
|
||||
_validate_preflight_projection(evidence["preflight"])
|
||||
receipts = value["receipts"]
|
||||
phase = value["phase"]
|
||||
expected_steps = {
|
||||
"reserved": (),
|
||||
"target_gates_ready": ("target_gates",),
|
||||
"quiesced": ("target_gates", "quiescence"),
|
||||
"snapshot_verified": ("target_gates", "quiescence", "snapshot"),
|
||||
"target_registered": ("target_gates", "quiescence", "snapshot", "controller_registration"),
|
||||
"appops_owned": tuple(key for key in _STEP_EVIDENCE_KEYS if key != "completion"),
|
||||
"completed": tuple(_STEP_EVIDENCE_KEYS),
|
||||
}[phase]
|
||||
_need(set(receipts) == set(expected_steps), "intent_receipts_incomplete")
|
||||
for step, record in receipts.items():
|
||||
_need(
|
||||
isinstance(record, dict)
|
||||
and set(record) == {"evidenceSha256"}
|
||||
and isinstance(record.get("evidenceSha256"), str)
|
||||
and _SHA256.fullmatch(record["evidenceSha256"]) is not None,
|
||||
"intent_receipt_invalid",
|
||||
)
|
||||
evidence_key = _STEP_EVIDENCE_KEYS[step]
|
||||
_need(
|
||||
evidence_key in evidence
|
||||
and _digest(evidence[evidence_key]) == record["evidenceSha256"],
|
||||
"intent_receipt_evidence_mismatch",
|
||||
)
|
||||
if value["status"] == "recovery_required":
|
||||
recovery = value.get("recovery")
|
||||
_need(
|
||||
isinstance(recovery, dict)
|
||||
and set(recovery)
|
||||
== {
|
||||
"reason",
|
||||
"afterPhase",
|
||||
"failureEvidenceSha256",
|
||||
"controllerReconciliationRequired",
|
||||
"legacyVolumePreserved",
|
||||
}
|
||||
and recovery.get("reason") in _RECOVERY_REASONS
|
||||
and recovery.get("afterPhase") == phase
|
||||
and isinstance(recovery.get("failureEvidenceSha256"), str)
|
||||
and _SHA256.fullmatch(recovery["failureEvidenceSha256"]) is not None
|
||||
and recovery.get("controllerReconciliationRequired") is True
|
||||
and recovery.get("legacyVolumePreserved") is True,
|
||||
"intent_recovery_marker_invalid",
|
||||
)
|
||||
else:
|
||||
_need(value.get("recovery") is None, "intent_recovery_marker_unexpected")
|
||||
return value
|
||||
|
||||
|
||||
def _intent_binding(intent: dict[str, Any]) -> str:
|
||||
return _digest(
|
||||
{
|
||||
"intentId": intent["intentId"],
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"preflightSha256": intent["preflightSha256"],
|
||||
"idempotencySha256": intent["idempotencySha256"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _target_gates(evidence: object) -> dict[str, Any]:
|
||||
required = {
|
||||
"profileId",
|
||||
"profileVersion",
|
||||
"planSha256",
|
||||
"profileReady",
|
||||
"controllerTrustReady",
|
||||
"controllerBootstrapSecretImported",
|
||||
"typedEnrollmentTrustReady",
|
||||
"targetSecretReady",
|
||||
"exactPolicyReady",
|
||||
"authenticatedRouteReady",
|
||||
"accessListReady",
|
||||
"mutationBoundaryReady",
|
||||
"managedRuntimeReady",
|
||||
"scanSourceMatches",
|
||||
"scanMountReadOnly",
|
||||
"scanPermissionPolicyPersistent",
|
||||
"scanAccessForRuntimeIdentityReady",
|
||||
"ownedDataTargetReady",
|
||||
"managedEnvironmentReady",
|
||||
"legacyEnvironmentRetained",
|
||||
"maintenanceAuthReady",
|
||||
"controllerAdoptionCapabilityReady",
|
||||
"controllerRegistryWriterReady",
|
||||
"controllerSecretWriterReady",
|
||||
}
|
||||
_need(isinstance(evidence, dict) and set(evidence) == required, "target_gates_invalid")
|
||||
value = evidence
|
||||
_need(
|
||||
value.get("profileId") == _PROFILE_ID
|
||||
and value.get("profileVersion") == PROFILE_VERSION
|
||||
and isinstance(value.get("planSha256"), str)
|
||||
and _SHA256.fullmatch(value["planSha256"]) is not None
|
||||
and all(value.get(key) is True for key in required - {"profileId", "profileVersion", "planSha256"}),
|
||||
"target_gates_not_ready",
|
||||
)
|
||||
return {key: value[key] for key in sorted(required)}
|
||||
|
||||
|
||||
def _quiescence_evidence(evidence: object, intent: dict[str, Any]) -> dict[str, Any]:
|
||||
required = {
|
||||
"intentId",
|
||||
"globalDeployLockHeld",
|
||||
"projectLockHeld",
|
||||
"maintenanceModeActive",
|
||||
"queueFrozen",
|
||||
"queuedJobs",
|
||||
"activeJobs",
|
||||
"maintenanceModeAuthenticated",
|
||||
"containerName",
|
||||
"containerIdSha256",
|
||||
"containerImageSha256",
|
||||
"containerState",
|
||||
"legacyVolume",
|
||||
"controllerIntentDurable",
|
||||
"controllerIntentBindingSha256",
|
||||
}
|
||||
_need(isinstance(evidence, dict) and set(evidence) == required, "quiescence_evidence_invalid")
|
||||
value = evidence
|
||||
_need(
|
||||
value.get("intentId") == intent["intentId"]
|
||||
and value.get("globalDeployLockHeld") is True
|
||||
and value.get("projectLockHeld") is True
|
||||
and value.get("maintenanceModeActive") is True
|
||||
and value.get("queueFrozen") is True
|
||||
and value.get("maintenanceModeAuthenticated") is True
|
||||
and _is_count(value.get("queuedJobs"))
|
||||
and value.get("queuedJobs") == 0
|
||||
and _is_count(value.get("activeJobs"))
|
||||
and value.get("activeJobs") == 0
|
||||
and value.get("containerName") == "repopulse"
|
||||
and isinstance(value.get("containerIdSha256"), str)
|
||||
and _SHA256.fullmatch(value["containerIdSha256"]) is not None
|
||||
and value.get("containerIdSha256") == intent["evidence"]["preflight"]["container_id_sha256"]
|
||||
and isinstance(value.get("containerImageSha256"), str)
|
||||
and _SHA256.fullmatch(value["containerImageSha256"]) is not None
|
||||
and value.get("containerImageSha256") == intent["evidence"]["preflight"]["container_image_sha256"]
|
||||
and value.get("containerState") == "exited"
|
||||
and value.get("legacyVolume") == LEGACY_VOLUME
|
||||
and value.get("controllerIntentDurable") is True,
|
||||
"quiescence_not_proven",
|
||||
)
|
||||
_need(
|
||||
isinstance(value.get("controllerIntentBindingSha256"), str)
|
||||
and _SHA256.fullmatch(value["controllerIntentBindingSha256"]) is not None
|
||||
and value["controllerIntentBindingSha256"] == _intent_binding(intent),
|
||||
"controller_intent_binding_mismatch",
|
||||
)
|
||||
return {key: value[key] for key in sorted(required)}
|
||||
|
||||
|
||||
def _snapshot_evidence(evidence: object, intent: dict[str, Any]) -> dict[str, Any]:
|
||||
required = {
|
||||
"intentId",
|
||||
"globalDeployLockHeld",
|
||||
"projectLockHeld",
|
||||
"legacyContainerStopped",
|
||||
"legacyVolume",
|
||||
"legacyVolumeRetained",
|
||||
"sqliteBackupMethod",
|
||||
"quickCheck",
|
||||
"integrityCheck",
|
||||
"sourceLogicalSha256",
|
||||
"snapshotLogicalSha256",
|
||||
"snapshotFileSha256",
|
||||
"sourceScanCount",
|
||||
"snapshotScanCount",
|
||||
"sourceCompactedScanCount",
|
||||
"snapshotCompactedScanCount",
|
||||
"sourceQueueSha256",
|
||||
"snapshotQueueSha256",
|
||||
"sourceAuxiliaryTreeSha256",
|
||||
"sourceAuxiliaryFileCount",
|
||||
"targetDataPathWasAbsent",
|
||||
"snapshotReceiptDurable",
|
||||
"legacyEnvironmentRetained",
|
||||
"managedEnvironmentFilePrivate",
|
||||
"controllerIntentBindingSha256",
|
||||
}
|
||||
_need(isinstance(evidence, dict) and set(evidence) == required, "snapshot_evidence_invalid")
|
||||
value = evidence
|
||||
_need(
|
||||
value.get("intentId") == intent["intentId"]
|
||||
and value.get("globalDeployLockHeld") is True
|
||||
and value.get("projectLockHeld") is True
|
||||
and value.get("legacyContainerStopped") is True
|
||||
and value.get("legacyVolume") == LEGACY_VOLUME
|
||||
and value.get("legacyVolumeRetained") is True
|
||||
and value.get("sqliteBackupMethod") == "sqlite_online_backup"
|
||||
and value.get("quickCheck") == "ok"
|
||||
and value.get("integrityCheck") == "ok"
|
||||
and isinstance(value.get("sourceLogicalSha256"), str)
|
||||
and _SHA256.fullmatch(value["sourceLogicalSha256"]) is not None
|
||||
and value.get("sourceLogicalSha256") == value.get("snapshotLogicalSha256")
|
||||
and isinstance(value.get("snapshotFileSha256"), str)
|
||||
and _SHA256.fullmatch(value["snapshotFileSha256"]) is not None
|
||||
and _is_count(value.get("sourceScanCount"))
|
||||
and value.get("sourceScanCount") == value.get("snapshotScanCount")
|
||||
and _is_count(value.get("sourceCompactedScanCount"))
|
||||
and value.get("sourceCompactedScanCount") == value.get("snapshotCompactedScanCount")
|
||||
and isinstance(value.get("sourceQueueSha256"), str)
|
||||
and _SHA256.fullmatch(value["sourceQueueSha256"]) is not None
|
||||
and value.get("sourceQueueSha256") == value.get("snapshotQueueSha256")
|
||||
and isinstance(value.get("sourceAuxiliaryTreeSha256"), str)
|
||||
and _SHA256.fullmatch(value["sourceAuxiliaryTreeSha256"]) is not None
|
||||
and _is_count(value.get("sourceAuxiliaryFileCount"))
|
||||
and value.get("targetDataPathWasAbsent") is True
|
||||
and value.get("snapshotReceiptDurable") is True
|
||||
and value.get("legacyEnvironmentRetained") is True
|
||||
and value.get("managedEnvironmentFilePrivate") is True,
|
||||
"sqlite_snapshot_not_verified",
|
||||
)
|
||||
_need(
|
||||
isinstance(value.get("controllerIntentBindingSha256"), str)
|
||||
and _SHA256.fullmatch(value["controllerIntentBindingSha256"]) is not None
|
||||
and value["controllerIntentBindingSha256"] == _intent_binding(intent),
|
||||
"controller_intent_binding_mismatch",
|
||||
)
|
||||
return {key: value[key] for key in sorted(required)}
|
||||
|
||||
|
||||
def _controller_registration_evidence(
|
||||
evidence: object, intent: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
required = {
|
||||
"receiptVersion",
|
||||
"intentId",
|
||||
"receiptSha256",
|
||||
"repository",
|
||||
"applicationId",
|
||||
"enrollmentId",
|
||||
"profileId",
|
||||
"planSha256",
|
||||
"snapshotFileSha256",
|
||||
"targetLogicalSha256",
|
||||
"targetScanCount",
|
||||
"targetCompactedScanCount",
|
||||
"targetQueueSha256",
|
||||
"targetAuxiliaryTreeSha256",
|
||||
"targetAuxiliaryFileCount",
|
||||
"targetQueuedJobs",
|
||||
"targetActiveJobs",
|
||||
"targetQuickCheck",
|
||||
"targetIntegrityCheck",
|
||||
"targetRecoveryStatus",
|
||||
"targetDataUid",
|
||||
"targetDataGid",
|
||||
"managedDataMountMatches",
|
||||
"copySourceIsVerifiedSnapshot",
|
||||
"registryTargetMatches",
|
||||
"controllerStateReceiptVerified",
|
||||
"deploymentReceiptSchema",
|
||||
"deploymentReceiptSha256",
|
||||
"deploymentReceiptVerified",
|
||||
"runtimeHealthy",
|
||||
"publishedHostPorts",
|
||||
"globalDeployLockHeld",
|
||||
"projectLockHeld",
|
||||
"legacyVolume",
|
||||
"legacyVolumeRetained",
|
||||
"legacyEnvironmentRetained",
|
||||
"managedEnvironmentFilePrivate",
|
||||
"scanSourceMatches",
|
||||
"scanMountReadOnly",
|
||||
"scanPermissionPolicyPersistent",
|
||||
"scanAccessForRuntimeIdentityReady",
|
||||
"controllerIntentDurable",
|
||||
"controllerIntentBindingSha256",
|
||||
}
|
||||
_need(
|
||||
isinstance(evidence, dict) and set(evidence) == required,
|
||||
"controller_receipt_evidence_invalid",
|
||||
)
|
||||
value = evidence
|
||||
gates = intent["evidence"].get("targetGates")
|
||||
snapshot = intent["evidence"].get("snapshot")
|
||||
_need(isinstance(gates, dict) and isinstance(snapshot, dict), "adoption_prerequisite_missing")
|
||||
_need(
|
||||
value.get("receiptVersion") == CONTROLLER_RECEIPT_VERSION
|
||||
and value.get("intentId") == intent["intentId"]
|
||||
and isinstance(value.get("receiptSha256"), str)
|
||||
and _SHA256.fullmatch(value["receiptSha256"]) is not None
|
||||
and value.get("repository") == REPOSITORY
|
||||
and value.get("applicationId") == APPLICATION_ID
|
||||
and isinstance(value.get("enrollmentId"), str)
|
||||
and _ENROLLMENT_ID.fullmatch(value["enrollmentId"]) is not None
|
||||
and value.get("profileId") == gates["profileId"]
|
||||
and value.get("planSha256") == gates["planSha256"]
|
||||
and value.get("snapshotFileSha256") == snapshot["snapshotFileSha256"]
|
||||
and isinstance(value.get("targetLogicalSha256"), str)
|
||||
and value.get("targetLogicalSha256") == snapshot["snapshotLogicalSha256"]
|
||||
and _is_count(value.get("targetScanCount"))
|
||||
and value.get("targetScanCount") == snapshot["snapshotScanCount"]
|
||||
and _is_count(value.get("targetCompactedScanCount"))
|
||||
and value.get("targetCompactedScanCount") == snapshot["snapshotCompactedScanCount"]
|
||||
and value.get("targetQueueSha256") == snapshot["snapshotQueueSha256"]
|
||||
and value.get("targetAuxiliaryTreeSha256") == snapshot["sourceAuxiliaryTreeSha256"]
|
||||
and value.get("targetAuxiliaryFileCount") == snapshot["sourceAuxiliaryFileCount"]
|
||||
and _is_count(value.get("targetQueuedJobs"))
|
||||
and value.get("targetQueuedJobs") == 0
|
||||
and _is_count(value.get("targetActiveJobs"))
|
||||
and value.get("targetActiveJobs") == 0
|
||||
and value.get("targetQuickCheck") == "ok"
|
||||
and value.get("targetIntegrityCheck") == "ok"
|
||||
and value.get("targetRecoveryStatus") in {"healthy", "acknowledged"}
|
||||
and value.get("targetDataUid") == 10001
|
||||
and value.get("targetDataGid") == 10001
|
||||
and value.get("managedDataMountMatches") is True
|
||||
and value.get("copySourceIsVerifiedSnapshot") is True
|
||||
and value.get("registryTargetMatches") is True
|
||||
and value.get("controllerStateReceiptVerified") is True
|
||||
and value.get("deploymentReceiptSchema") == "gitea-deploy-receipt.v1"
|
||||
and isinstance(value.get("deploymentReceiptSha256"), str)
|
||||
and _SHA256.fullmatch(value["deploymentReceiptSha256"]) is not None
|
||||
and value.get("deploymentReceiptVerified") is True
|
||||
and value.get("runtimeHealthy") is True
|
||||
and value.get("publishedHostPorts") == []
|
||||
and value.get("globalDeployLockHeld") is True
|
||||
and value.get("projectLockHeld") is True
|
||||
and value.get("legacyVolume") == LEGACY_VOLUME
|
||||
and value.get("legacyVolumeRetained") is True
|
||||
and value.get("legacyEnvironmentRetained") is True
|
||||
and value.get("managedEnvironmentFilePrivate") is True
|
||||
and value.get("scanSourceMatches") is True
|
||||
and value.get("scanMountReadOnly") is True
|
||||
and value.get("scanPermissionPolicyPersistent") is True
|
||||
and value.get("scanAccessForRuntimeIdentityReady") is True
|
||||
and value.get("controllerIntentDurable") is True
|
||||
and value.get("controllerIntentBindingSha256") == _intent_binding(intent),
|
||||
"controller_adoption_receipt_not_verified",
|
||||
)
|
||||
return {key: value[key] for key in sorted(required)}
|
||||
|
||||
|
||||
def validate_controller_registration_evidence(
|
||||
intent: object, evidence: object
|
||||
) -> dict[str, Any]:
|
||||
"""Validate fresh controller registration evidence against the intent.
|
||||
|
||||
This pure projection can validate receipt replays after AppOps reaches its
|
||||
terminal `completed` phase. Controller response authentication remains the
|
||||
caller's responsibility.
|
||||
"""
|
||||
current = _validate_intent(intent)
|
||||
return _controller_registration_evidence(evidence, current)
|
||||
|
||||
|
||||
def _appops_ownership_evidence(
|
||||
evidence: object, intent: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
required = {
|
||||
"repository",
|
||||
"applicationId",
|
||||
"enrollmentId",
|
||||
"profileId",
|
||||
"planSha256",
|
||||
"controllerReceiptSha256",
|
||||
"enrollmentPersisted",
|
||||
"exactApplyPolicyPersisted",
|
||||
"exactRollbackPolicyPersisted",
|
||||
"secretReferenceReady",
|
||||
}
|
||||
_need(isinstance(evidence, dict) and set(evidence) == required, "appops_ownership_evidence_invalid")
|
||||
value = evidence
|
||||
registration = intent["evidence"].get("controllerRegistration")
|
||||
_need(isinstance(registration, dict), "controller_registration_missing")
|
||||
gates = intent["evidence"].get("targetGates")
|
||||
_need(isinstance(gates, dict), "target_gates_missing")
|
||||
_need(
|
||||
value.get("repository") == REPOSITORY
|
||||
and value.get("applicationId") == APPLICATION_ID
|
||||
and value.get("enrollmentId") == registration["enrollmentId"]
|
||||
and value.get("profileId") == registration["profileId"]
|
||||
and value.get("planSha256") == gates["planSha256"]
|
||||
and value.get("controllerReceiptSha256") == registration["receiptSha256"]
|
||||
and all(
|
||||
value.get(key) is True
|
||||
for key in (
|
||||
"enrollmentPersisted",
|
||||
"exactApplyPolicyPersisted",
|
||||
"exactRollbackPolicyPersisted",
|
||||
"secretReferenceReady",
|
||||
)
|
||||
),
|
||||
"appops_ownership_not_verified",
|
||||
)
|
||||
return {key: value[key] for key in sorted(required)}
|
||||
|
||||
|
||||
def validate_appops_ownership_evidence(
|
||||
intent: object, evidence: object
|
||||
) -> dict[str, Any]:
|
||||
"""Validate fixed ownership evidence for an idempotent completed replay."""
|
||||
current = _validate_intent(intent)
|
||||
return _appops_ownership_evidence(evidence, current)
|
||||
|
||||
|
||||
def advance_intent(
|
||||
intent: object, step: str, evidence: object
|
||||
) -> dict[str, Any]:
|
||||
"""Purely validate and record one forward protocol checkpoint.
|
||||
|
||||
The exact evidence digest makes retries idempotent. Replaying a step with
|
||||
different evidence is a conflict and must enter recovery review.
|
||||
"""
|
||||
current = _validate_intent(intent)
|
||||
_need(current.get("status") == "in_progress", "intent_requires_recovery")
|
||||
_need(step in _STEP_CONTRACTS, "adoption_step_unknown")
|
||||
source_phase, destination_phase = _STEP_CONTRACTS[step]
|
||||
checked = {
|
||||
"target_gates": _target_gates,
|
||||
"quiescence": lambda value: _quiescence_evidence(value, current),
|
||||
"snapshot": lambda value: _snapshot_evidence(value, current),
|
||||
"controller_registration": lambda value: _controller_registration_evidence(value, current),
|
||||
"appops_ownership": lambda value: _appops_ownership_evidence(value, current),
|
||||
}[step](evidence)
|
||||
digest = _digest(checked)
|
||||
receipts = current["receipts"]
|
||||
previous = receipts.get(step)
|
||||
if previous is not None:
|
||||
_need(
|
||||
current.get("phase") in _PHASES[_PHASES.index(destination_phase):]
|
||||
and isinstance(previous, dict)
|
||||
and previous.get("evidenceSha256") == digest,
|
||||
"idempotency_conflict",
|
||||
)
|
||||
return deepcopy(current)
|
||||
_need(current.get("phase") == source_phase, "adoption_step_out_of_order")
|
||||
|
||||
updated = deepcopy(current)
|
||||
updated["phase"] = destination_phase
|
||||
updated["recovery"] = None
|
||||
updated["receipts"][step] = {"evidenceSha256": digest}
|
||||
evidence_key = _STEP_EVIDENCE_KEYS[step]
|
||||
updated["evidence"][evidence_key] = checked
|
||||
_validate_intent(updated)
|
||||
return updated
|
||||
|
||||
|
||||
def complete_intent(intent: object, evidence: object) -> dict[str, Any]:
|
||||
"""Finalize only after re-reading both AppOps ownership and controller receipt."""
|
||||
current = _validate_intent(intent)
|
||||
_need(
|
||||
current.get("status") == "in_progress"
|
||||
and current.get("phase") == "appops_owned",
|
||||
"adoption_not_ready_to_complete",
|
||||
)
|
||||
required = {
|
||||
"intentId",
|
||||
"repository",
|
||||
"applicationId",
|
||||
"enrollmentId",
|
||||
"controllerReceiptSha256",
|
||||
"enrollmentStillPersisted",
|
||||
"exactApplyPolicyStillPersisted",
|
||||
"exactRollbackPolicyStillPersisted",
|
||||
"controllerReceiptStillMatches",
|
||||
"legacyVolumeRetained",
|
||||
}
|
||||
_need(isinstance(evidence, dict) and set(evidence) == required, "completion_evidence_invalid")
|
||||
value = evidence
|
||||
ownership = current["evidence"].get("appopsOwnership")
|
||||
registration = current["evidence"].get("controllerRegistration")
|
||||
_need(isinstance(ownership, dict) and isinstance(registration, dict), "adoption_prerequisite_missing")
|
||||
_need(
|
||||
value.get("intentId") == current["intentId"]
|
||||
and value.get("repository") == REPOSITORY
|
||||
and value.get("applicationId") == APPLICATION_ID
|
||||
and value.get("enrollmentId") == registration["enrollmentId"]
|
||||
and value.get("controllerReceiptSha256") == registration["receiptSha256"]
|
||||
and value.get("controllerReceiptSha256") == ownership["controllerReceiptSha256"]
|
||||
and all(
|
||||
value.get(key) is True
|
||||
for key in (
|
||||
"enrollmentStillPersisted",
|
||||
"exactApplyPolicyStillPersisted",
|
||||
"exactRollbackPolicyStillPersisted",
|
||||
"controllerReceiptStillMatches",
|
||||
"legacyVolumeRetained",
|
||||
)
|
||||
),
|
||||
"completion_not_verified",
|
||||
)
|
||||
checked = {key: value[key] for key in sorted(required)}
|
||||
updated = deepcopy(current)
|
||||
updated["phase"] = "completed"
|
||||
updated["status"] = "completed"
|
||||
updated["evidence"]["completion"] = checked
|
||||
updated["receipts"]["completion"] = {"evidenceSha256": _digest(checked)}
|
||||
return _validate_intent(updated)
|
||||
|
||||
|
||||
def mark_recovery_required(
|
||||
intent: object, reason: str, failure_evidence: object
|
||||
) -> dict[str, Any]:
|
||||
"""Persist a sanitized recovery marker without compensating external state."""
|
||||
current = _validate_intent(intent)
|
||||
_need(current.get("status") != "completed", "intent_already_completed")
|
||||
_need(reason in _RECOVERY_REASONS, "recovery_reason_invalid")
|
||||
_need(isinstance(failure_evidence, dict), "recovery_evidence_invalid")
|
||||
# Only bind the evidence; arbitrary external messages/paths/secret values
|
||||
# are never copied into the journal by this pure protocol helper.
|
||||
updated = deepcopy(current)
|
||||
updated["status"] = "recovery_required"
|
||||
updated["recovery"] = {
|
||||
"reason": reason,
|
||||
"afterPhase": current["phase"],
|
||||
"failureEvidenceSha256": _digest(failure_evidence),
|
||||
"controllerReconciliationRequired": True,
|
||||
"legacyVolumePreserved": True,
|
||||
}
|
||||
return updated
|
||||
|
||||
|
||||
def resume_intent(intent: object, controller_receipt: object) -> dict[str, Any]:
|
||||
"""Resume only from an authenticated controller receipt bound to this intent.
|
||||
|
||||
The adapter that calls this function must authenticate the controller
|
||||
response. This pure helper validates its exact content and only reconciles
|
||||
the single ambiguous boundary: controller registration committed while the
|
||||
AppOps intent checkpoint was interrupted.
|
||||
"""
|
||||
current = _validate_intent(intent)
|
||||
_need(current.get("status") == "recovery_required", "intent_not_in_recovery")
|
||||
required = {
|
||||
"apiVersion",
|
||||
"intentId",
|
||||
"phase",
|
||||
"receiptSha256",
|
||||
"repository",
|
||||
"applicationId",
|
||||
"legacyVolumeRetained",
|
||||
"intentDurable",
|
||||
"intentBindingSha256",
|
||||
"registrationEvidence",
|
||||
}
|
||||
_need(
|
||||
isinstance(controller_receipt, dict)
|
||||
and set(controller_receipt) == required,
|
||||
"controller_recovery_receipt_invalid",
|
||||
)
|
||||
receipt = controller_receipt
|
||||
_need(
|
||||
receipt.get("apiVersion") == CONTROLLER_RECEIPT_VERSION
|
||||
and receipt.get("intentId") == current["intentId"]
|
||||
and receipt.get("repository") == REPOSITORY
|
||||
and receipt.get("applicationId") == APPLICATION_ID
|
||||
and receipt.get("legacyVolumeRetained") is True
|
||||
and receipt.get("intentDurable") is True
|
||||
and receipt.get("intentBindingSha256") == _intent_binding(current)
|
||||
and isinstance(receipt.get("receiptSha256"), str)
|
||||
and _SHA256.fullmatch(receipt["receiptSha256"]) is not None,
|
||||
"controller_recovery_receipt_unbound",
|
||||
)
|
||||
phase = receipt.get("phase")
|
||||
if phase == current["phase"]:
|
||||
_need(
|
||||
current["phase"] in {"reserved", "target_gates_ready", "quiesced", "snapshot_verified", "target_registered", "appops_owned"},
|
||||
"controller_recovery_phase_invalid",
|
||||
)
|
||||
if phase in {"target_registered", "appops_owned"}:
|
||||
registration = _controller_registration_evidence(
|
||||
receipt.get("registrationEvidence"), current
|
||||
)
|
||||
_need(
|
||||
receipt.get("receiptSha256") == registration["receiptSha256"]
|
||||
and registration == current["evidence"].get("controllerRegistration"),
|
||||
"controller_recovery_receipt_conflict",
|
||||
)
|
||||
else:
|
||||
_need(
|
||||
receipt.get("registrationEvidence") is None,
|
||||
"controller_recovery_receipt_conflict",
|
||||
)
|
||||
updated = deepcopy(current)
|
||||
updated["status"] = "in_progress"
|
||||
updated["recovery"] = None
|
||||
return updated
|
||||
_need(
|
||||
current.get("phase") == "snapshot_verified" and phase == "target_registered",
|
||||
"controller_recovery_phase_conflict",
|
||||
)
|
||||
registration = _controller_registration_evidence(receipt.get("registrationEvidence"), current)
|
||||
registration_digest = _digest(registration)
|
||||
_need(receipt.get("receiptSha256") == registration["receiptSha256"], "controller_recovery_receipt_digest_mismatch")
|
||||
updated = deepcopy(current)
|
||||
updated["phase"] = "target_registered"
|
||||
updated["status"] = "in_progress"
|
||||
updated["recovery"] = None
|
||||
updated["receipts"]["controller_registration"] = {"evidenceSha256": registration_digest}
|
||||
updated["evidence"]["controllerRegistration"] = registration
|
||||
_validate_intent(updated)
|
||||
return updated
|
||||
|
||||
|
||||
def intent_binding_sha256(intent: object) -> str:
|
||||
"""Return the immutable root-controller journal binding for an intent."""
|
||||
current = _validate_intent(intent)
|
||||
return _intent_binding(current)
|
||||
|
||||
|
||||
def execution_status() -> dict[str, Any]:
|
||||
"""Report the code gate without claiming live target readiness."""
|
||||
return {
|
||||
"apiVersion": PROTOCOL_VERSION,
|
||||
"repository": REPOSITORY,
|
||||
"status": "enabled" if EXECUTION_ENABLED is True else "disabled",
|
||||
"reason": None if EXECUTION_ENABLED is True else "controller_adoption_execution_disabled",
|
||||
"executionEnabled": EXECUTION_ENABLED,
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Operator-only staging of the single authenticated RepoPulse NPM route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from ..core.config import settings
|
||||
from ..db.session import engine
|
||||
from . import repopulse_adoption_journal as journal
|
||||
from .enrollment_plane import EnrollmentPlane
|
||||
from .setup_service import apply_stored_connector_settings
|
||||
|
||||
|
||||
class RepoPulseProxyStageError(ValueError):
|
||||
"""Fixed, non-secret staging failure."""
|
||||
|
||||
|
||||
def _need(condition: bool, reason: str) -> None:
|
||||
if not condition:
|
||||
raise RepoPulseProxyStageError(reason)
|
||||
|
||||
|
||||
def stage_repopulse_proxy_host(
|
||||
session: Session, *, connector: NginxProxyManagerConnector | None = None
|
||||
) -> dict[str, object]:
|
||||
"""Create or verify only the reserved RepoPulse host and ACL1 contract."""
|
||||
row, intent = journal._load(session)
|
||||
_need(row is not None and isinstance(intent, dict), "adoption_intent_unavailable")
|
||||
intent_id = intent.get("intentId")
|
||||
_need(
|
||||
isinstance(intent_id, str)
|
||||
and re.fullmatch(r"rpa-[0-9a-f]{20}", intent_id) is not None
|
||||
and intent.get("status") == "in_progress",
|
||||
"adoption_intent_unavailable",
|
||||
)
|
||||
trust = EnrollmentPlane.get_trust_config(session)
|
||||
_need(
|
||||
trust is not None
|
||||
and trust.controller_identity == "unraid-controller"
|
||||
and trust.repopulse_proxy_access_list_id == 1
|
||||
and "repopulse-authenticated" in trust.autonomous_profiles
|
||||
and "itworx-mcp-hub-npm" in trust.approved_networks
|
||||
and "example.com" in trust.allowed_domain_suffixes,
|
||||
"repopulse_trust_boundary_mismatch",
|
||||
)
|
||||
apply_stored_connector_settings(session, {"nginx_proxy_manager"})
|
||||
identity = settings.apis.get("NPM_IDENTITY")
|
||||
_need(
|
||||
isinstance(identity, str)
|
||||
and re.fullmatch(r"[^\s@]+@[^\s@]+\.[^\s@]+", identity) is not None,
|
||||
"certificate_automation_unavailable",
|
||||
)
|
||||
settings.apis["NPM_LETSENCRYPT_EMAIL"] = identity
|
||||
npm = connector or NginxProxyManagerConnector()
|
||||
_need(npm.is_configured(), "npm_connector_unavailable")
|
||||
acl = npm.validate_managed_access_list(1)
|
||||
_need(
|
||||
isinstance(acl, dict)
|
||||
and acl.get("success") is True
|
||||
and acl.get("status") == "verified"
|
||||
and acl.get("accessListId") == 1,
|
||||
"repopulse_access_list_unavailable",
|
||||
)
|
||||
ownership_id = f"enr-{intent_id[4:]}"
|
||||
result = npm.ensure_managed_proxy_host(
|
||||
hostname="repopulse.example.com",
|
||||
forward_host="repopulse",
|
||||
forward_port=8000,
|
||||
certificate_mode="managed",
|
||||
force_ssl=True,
|
||||
http2=True,
|
||||
upload_limit_mb=128,
|
||||
read_timeout_seconds=120,
|
||||
ownership_id=ownership_id,
|
||||
access_list_id=1,
|
||||
)
|
||||
_need(
|
||||
isinstance(result, dict)
|
||||
and result.get("success") is True
|
||||
and result.get("status") in {"created", "already_managed"}
|
||||
and result.get("hostname") == "repopulse.example.com",
|
||||
"repopulse_proxy_host_unavailable",
|
||||
)
|
||||
return {
|
||||
"status": "ready",
|
||||
"intentId": intent_id,
|
||||
"hostname": "repopulse.example.com",
|
||||
"accessListId": 1,
|
||||
"hostId": result.get("hostId"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 1:
|
||||
print('{"status":"blocked","reason":"invalid_invocation"}')
|
||||
return 2
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
result = stage_repopulse_proxy_host(session)
|
||||
print(json.dumps(result, separators=(",", ":")))
|
||||
return 0
|
||||
except Exception: # noqa: BLE001 - connector failures can include sensitive response text.
|
||||
print('{"status":"blocked","reason":"repopulse_proxy_stage_unavailable"}')
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
"""One-time, write-only import of RepoPulse's existing Gitea credential.
|
||||
|
||||
Run inside the AppOps container as its service account. The credential is read
|
||||
from stdin so it never appears in argv, environment, logs, or command output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import json
|
||||
import sys
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..db.session import engine
|
||||
from .secret_store import get_secret, has_secret, secret_storage_status, set_secret
|
||||
|
||||
SECRET_REF = "REPOPULSE_GITEA_TOKEN"
|
||||
BOOTSTRAP_SECRET_REF = "ENROLLMENT_BOOTSTRAP"
|
||||
MAX_SECRET_BYTES = 8192
|
||||
|
||||
|
||||
def _import_fixed_secret(
|
||||
session: Session, token: str, *, secret_ref: str, used_by: str, purpose: str
|
||||
) -> str:
|
||||
if not secret_storage_status()["productionReady"]:
|
||||
raise ValueError("production secret storage is not ready")
|
||||
if not token or token != token.strip() or "\n" in token or "\r" in token:
|
||||
raise ValueError("invalid credential input")
|
||||
if len(token.encode("utf-8")) > MAX_SECRET_BYTES:
|
||||
raise ValueError("credential input too large")
|
||||
|
||||
existing = get_secret(session, secret_ref, touch=False)
|
||||
if existing is None and has_secret(session, secret_ref):
|
||||
raise ValueError("credential reference cannot be decrypted")
|
||||
if existing is not None:
|
||||
if hmac.compare_digest(existing, token):
|
||||
return "already_present"
|
||||
raise ValueError("credential reference already contains a different value")
|
||||
|
||||
set_secret(
|
||||
session,
|
||||
secret_ref,
|
||||
token,
|
||||
used_by=[used_by],
|
||||
metadata={"purpose": purpose},
|
||||
)
|
||||
return "imported"
|
||||
|
||||
|
||||
def import_repopulse_gitea_token(session: Session, token: str) -> str:
|
||||
return _import_fixed_secret(
|
||||
session,
|
||||
token,
|
||||
secret_ref=SECRET_REF,
|
||||
used_by="deployment:repopulse",
|
||||
purpose="repopulse_gitea_access",
|
||||
)
|
||||
|
||||
|
||||
def import_controller_bootstrap(session: Session, token: str) -> str:
|
||||
return _import_fixed_secret(
|
||||
session,
|
||||
token,
|
||||
secret_ref=BOOTSTRAP_SECRET_REF,
|
||||
used_by="deployment:enrollment_controller",
|
||||
purpose="controller_enrollment_bootstrap",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
raw = sys.stdin.buffer.read(MAX_SECRET_BYTES + 1)
|
||||
if len(raw) > MAX_SECRET_BYTES:
|
||||
print(json.dumps({"status": "blocked", "reason": "credential input too large"}))
|
||||
return 2
|
||||
try:
|
||||
token = raw.decode("utf-8")
|
||||
with Session(engine) as session:
|
||||
outcome = import_repopulse_gitea_token(session, token)
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
print(json.dumps({"status": "blocked", "reason": "credential import rejected"}))
|
||||
return 2
|
||||
print(json.dumps({"status": outcome, "secretRef": SECRET_REF}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,96 @@
|
||||
"""One durable credential for the fixed RepoPulse legacy adoption target."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import Secret
|
||||
from . import secret_store
|
||||
|
||||
_INTENT_ID = re.compile(r"^rpa-[0-9a-f]{20}$")
|
||||
_REF = "ENROLL_REPOPULSE_TARGET_HMAC"
|
||||
|
||||
|
||||
class RepoPulseTargetCredentialError(RuntimeError):
|
||||
"""A sanitized fixed-target credential error."""
|
||||
|
||||
|
||||
def get_or_create_repopulse_target_credential(
|
||||
session: Session, *, intent_id: str
|
||||
) -> tuple[str, bytes]:
|
||||
"""Insert once, then reuse the encrypted credential on every retry.
|
||||
|
||||
The caller must have durably reserved the same adoption intent before this
|
||||
method is called. The plaintext is returned only to the in-process private
|
||||
TLS client and must never be added to a receipt, response, or log.
|
||||
"""
|
||||
if not isinstance(intent_id, str) or _INTENT_ID.fullmatch(intent_id) is None:
|
||||
raise RepoPulseTargetCredentialError("adoption_intent_invalid")
|
||||
if secret_store.secret_storage_status().get("productionReady") is not True:
|
||||
raise RepoPulseTargetCredentialError("secret_store_not_ready")
|
||||
|
||||
enrollment_id = f"enr-{intent_id.removeprefix('rpa-')}"
|
||||
candidate = secrets.token_urlsafe(48)
|
||||
metadata = {
|
||||
"intentId": intent_id,
|
||||
"enrollmentId": enrollment_id,
|
||||
"rotationState": "active",
|
||||
"credentialFingerprint": hashlib.sha256(candidate.encode("ascii")).hexdigest()[
|
||||
:16
|
||||
],
|
||||
}
|
||||
now = utc_now()
|
||||
try:
|
||||
session.execute(
|
||||
sqlite_insert(Secret)
|
||||
.values(
|
||||
id=_REF,
|
||||
name=_REF,
|
||||
encrypted_value=secret_store._fernet()
|
||||
.encrypt(candidate.encode("ascii"))
|
||||
.decode("ascii"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
used_by_json=[f"enrollment:{enrollment_id}", "deployment-controller"],
|
||||
metadata_json=metadata,
|
||||
)
|
||||
.on_conflict_do_nothing(index_elements=[Secret.id])
|
||||
)
|
||||
session.commit()
|
||||
row = session.get(Secret, _REF, populate_existing=True)
|
||||
if (
|
||||
row is None
|
||||
or row.name != _REF
|
||||
or not isinstance(row.metadata_json, dict)
|
||||
or row.metadata_json.get("intentId") != intent_id
|
||||
or row.metadata_json.get("enrollmentId") != enrollment_id
|
||||
):
|
||||
raise RepoPulseTargetCredentialError("target_credential_collision")
|
||||
value = secret_store.get_secret(session, _REF, touch=False)
|
||||
if (
|
||||
not value
|
||||
or not 32 <= len(value) <= 256
|
||||
or not value.isascii()
|
||||
or not value.isprintable()
|
||||
):
|
||||
raise RepoPulseTargetCredentialError("target_credential_unavailable")
|
||||
if (
|
||||
row.metadata_json.get("credentialFingerprint")
|
||||
!= hashlib.sha256(value.encode("utf-8")).hexdigest()[:16]
|
||||
):
|
||||
raise RepoPulseTargetCredentialError(
|
||||
"target_credential_fingerprint_mismatch"
|
||||
)
|
||||
return enrollment_id, value.encode("utf-8")
|
||||
except SQLAlchemyError:
|
||||
session.rollback()
|
||||
raise RepoPulseTargetCredentialError(
|
||||
"target_credential_storage_unavailable"
|
||||
) from None
|
||||
@@ -0,0 +1,92 @@
|
||||
"""One-time fixed AppOps trust staging for the RepoPulse controller route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..db.session import engine
|
||||
from ..schemas.enrollment import EnrollmentTrustConfigV1
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
from .enrollment_plane import EnrollmentPlane
|
||||
from .secret_store import get_secret, has_secret, secret_storage_status
|
||||
|
||||
_BROKER_URL = "https://203.0.113.1:1300"
|
||||
_CA_FILE = "/data/controller-ca.pem"
|
||||
_BOOTSTRAP_REF = "ENROLLMENT_BOOTSTRAP"
|
||||
|
||||
|
||||
def _need(condition: bool) -> None:
|
||||
if not condition:
|
||||
raise ValueError("repopulse_trust_not_ready")
|
||||
|
||||
|
||||
def stage(
|
||||
session: Session, *, broker: DeploymentBrokerClient | None = None
|
||||
) -> dict[str, str]:
|
||||
"""Write only the exact reviewed trust document after TLS and secrets pass."""
|
||||
client = broker or DeploymentBrokerClient()
|
||||
_need(
|
||||
client.base_url == _BROKER_URL
|
||||
and client.verify_tls is True
|
||||
and client.ca_file == _CA_FILE
|
||||
and client.configuration_error() is None
|
||||
)
|
||||
_need(secret_storage_status().get("productionReady") is True)
|
||||
bootstrap = get_secret(session, _BOOTSTRAP_REF, touch=False)
|
||||
_need(isinstance(bootstrap, str) and 32 <= len(bootstrap) <= 256)
|
||||
_need(has_secret(session, "REPOPULSE_GITEA_TOKEN"))
|
||||
context = ssl.create_default_context(cafile=_CA_FILE)
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context)
|
||||
)
|
||||
with opener.open(_BROKER_URL + "/health", timeout=8) as response:
|
||||
_need(response.status == 200)
|
||||
body = response.read(4097)
|
||||
_need(len(body) <= 4096)
|
||||
health = json.loads(body)
|
||||
_need(isinstance(health, dict) and health.get("status") == "ok")
|
||||
|
||||
expected = EnrollmentTrustConfigV1(
|
||||
controller_identity="unraid-controller",
|
||||
bootstrap_secret_ref=_BOOTSTRAP_REF,
|
||||
approved_networks=("itworx-mcp-hub-npm",),
|
||||
allowed_domain_suffixes=("example.com",),
|
||||
autonomous_profiles=("repopulse-authenticated",),
|
||||
repopulse_proxy_access_list_id=1,
|
||||
)
|
||||
existing = EnrollmentPlane.get_trust_config(session)
|
||||
if existing is not None:
|
||||
_need(existing == expected)
|
||||
return {
|
||||
"status": "already_configured",
|
||||
"controllerIdentity": "unraid-controller",
|
||||
}
|
||||
result = EnrollmentPlane.set_trust_config(session, expected)
|
||||
_need(
|
||||
result.get("status") == "ready"
|
||||
and result.get("bootstrapSecretConfigured") is True
|
||||
)
|
||||
return {"status": "configured", "controllerIdentity": "unraid-controller"}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 1:
|
||||
print('{"status":"blocked","reason":"invalid_invocation"}')
|
||||
return 2
|
||||
try:
|
||||
with Session(engine) as session:
|
||||
result = stage(session)
|
||||
print(json.dumps(result, separators=(",", ":")))
|
||||
return 0
|
||||
except Exception: # noqa: BLE001 - never expose TLS or secret-store details.
|
||||
print('{"status":"blocked","reason":"repopulse_trust_unavailable"}')
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,130 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from datetime import UTC, datetime
|
||||
import math
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting
|
||||
|
||||
|
||||
_MAX_SAMPLES = 2_000
|
||||
_PERSISTED_SAMPLES = 500
|
||||
_SETTING_KEY = "observability:request_samples"
|
||||
_samples: deque[dict[str, Any]] = deque(maxlen=_MAX_SAMPLES)
|
||||
_lock = Lock()
|
||||
_source_identity: str | None = None
|
||||
|
||||
|
||||
def record_request(*, method: str, path: str, status: int, duration_ms: float) -> None:
|
||||
sample = {
|
||||
"method": method,
|
||||
"path": path,
|
||||
"status": status,
|
||||
"durationMs": round(max(0.0, duration_ms), 2),
|
||||
"observedAt": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
with _lock:
|
||||
_samples.append(sample)
|
||||
|
||||
|
||||
def _percentile(values: list[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
rank = max(1, math.ceil(max(0.0, min(percentile, 1.0)) * len(values)))
|
||||
return values[min(len(values) - 1, rank - 1)]
|
||||
|
||||
|
||||
def request_metrics_snapshot() -> dict[str, Any]:
|
||||
with _lock:
|
||||
samples = list(_samples)
|
||||
|
||||
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for sample in samples:
|
||||
grouped[(sample["method"], sample["path"])].append(sample)
|
||||
|
||||
endpoints = []
|
||||
for (method, path), items in grouped.items():
|
||||
durations = sorted(float(item["durationMs"]) for item in items)
|
||||
endpoints.append({
|
||||
"method": method,
|
||||
"path": path,
|
||||
"requests": len(items),
|
||||
"errors": sum(1 for item in items if int(item["status"]) >= 400),
|
||||
"averageMs": round(sum(durations) / len(durations), 2),
|
||||
"p95Ms": round(_percentile(durations, 0.95), 2),
|
||||
"maxMs": round(durations[-1], 2),
|
||||
})
|
||||
|
||||
endpoints.sort(key=lambda item: (item["p95Ms"], item["requests"]), reverse=True)
|
||||
return {
|
||||
"window": "process_and_persisted",
|
||||
"sampleLimit": _MAX_SAMPLES,
|
||||
"sampleCount": len(samples),
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"slowEndpoints": endpoints[:30],
|
||||
"recentErrors": [sample for sample in samples if int(sample["status"]) >= 400][-20:],
|
||||
}
|
||||
|
||||
|
||||
def restore_request_metrics(session: Session) -> int:
|
||||
global _source_identity
|
||||
source_identity = str(session.get_bind().url)
|
||||
row = session.get(AppSetting, _SETTING_KEY)
|
||||
persisted = (row.value_json or {}).get("samples", []) if row else []
|
||||
restored = 0
|
||||
with _lock:
|
||||
if _source_identity != source_identity:
|
||||
_samples.clear()
|
||||
_source_identity = source_identity
|
||||
if _samples:
|
||||
return 0
|
||||
for sample in persisted[-_PERSISTED_SAMPLES:]:
|
||||
if not isinstance(sample, dict):
|
||||
continue
|
||||
if not {"method", "path", "status", "durationMs", "observedAt"}.issubset(sample):
|
||||
continue
|
||||
_samples.append({
|
||||
"method": str(sample["method"])[:12],
|
||||
"path": str(sample["path"])[:240],
|
||||
"status": int(sample["status"]),
|
||||
"durationMs": round(max(0.0, float(sample["durationMs"])), 2),
|
||||
"observedAt": str(sample["observedAt"])[:64],
|
||||
})
|
||||
restored += 1
|
||||
return restored
|
||||
|
||||
|
||||
def persist_request_metrics(session: Session) -> int:
|
||||
with _lock:
|
||||
samples = list(_samples)[-_PERSISTED_SAMPLES:]
|
||||
row = session.get(AppSetting, _SETTING_KEY) or AppSetting(key=_SETTING_KEY)
|
||||
row.value_json = {"samples": samples, "persistedAt": datetime.now(UTC).isoformat()}
|
||||
row.is_secret = False
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return len(samples)
|
||||
|
||||
|
||||
def request_metrics_openmetrics() -> str:
|
||||
snapshot = request_metrics_snapshot()
|
||||
lines = [
|
||||
"# HELP appops_http_requests_total Requests observed by method and route template.",
|
||||
"# TYPE appops_http_requests_total counter",
|
||||
"# HELP appops_http_request_duration_p95_milliseconds Nearest-rank p95 request duration.",
|
||||
"# TYPE appops_http_request_duration_p95_milliseconds gauge",
|
||||
]
|
||||
for endpoint in snapshot["slowEndpoints"]:
|
||||
method = str(endpoint["method"]).replace('"', '')
|
||||
path = str(endpoint["path"]).replace('\\', '\\\\').replace('"', '\\"')
|
||||
labels = f'method="{method}",path="{path}"'
|
||||
lines.append(f'appops_http_requests_total{{{labels}}} {endpoint["requests"]}')
|
||||
lines.append(f'appops_http_request_errors_total{{{labels}}} {endpoint["errors"]}')
|
||||
lines.append(f'appops_http_request_duration_p95_milliseconds{{{labels}}} {endpoint["p95Ms"]}')
|
||||
lines.append(f'appops_http_request_samples {snapshot["sampleCount"]}')
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Compose bounded Gitea and controller runner-fleet telemetry.
|
||||
|
||||
AppOps is intentionally a consumer of this projection. The root-owned
|
||||
controller remains the only component that may perform an allowlisted recovery
|
||||
or cleanup inside the isolated CI Docker daemon.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..connectors.gitea_actions import GiteaActionsConnector
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
|
||||
|
||||
_CAPABILITIES = (
|
||||
"linux-validation",
|
||||
"docker-validation",
|
||||
"browser-validation",
|
||||
"security-validation",
|
||||
"windows-native",
|
||||
)
|
||||
_SEVERITY_RANK = {"healthy": 0, "warning": 1, "critical": 2, "unknown": 3}
|
||||
|
||||
|
||||
def _severity(*states: object) -> str:
|
||||
normalized = [str(state) for state in states]
|
||||
return max(normalized, key=lambda value: _SEVERITY_RANK.get(value, 3), default="unknown")
|
||||
|
||||
|
||||
def _capabilities(runners: list[dict[str, Any]]) -> list[dict[str, object]]:
|
||||
rows = []
|
||||
for capability in _CAPABILITIES:
|
||||
compatible = [runner for runner in runners if capability in runner.get("labels", [])]
|
||||
online = [
|
||||
runner
|
||||
for runner in compatible
|
||||
if runner.get("status") == "online" and not runner.get("disabled")
|
||||
]
|
||||
rows.append(
|
||||
{
|
||||
"name": capability,
|
||||
"total": len(compatible),
|
||||
"online": len(online),
|
||||
"busy": sum(bool(runner.get("busy")) for runner in online),
|
||||
"status": "healthy" if online else "critical",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
class RunnerFleetService:
|
||||
"""Provide a read-only, explainable CI fleet projection."""
|
||||
|
||||
def snapshot(self) -> dict[str, object]:
|
||||
actions = GiteaActionsConnector().snapshot()
|
||||
controller = DeploymentBrokerClient().runner_fleet_status()
|
||||
runners = actions.get("runners") if isinstance(actions.get("runners"), list) else []
|
||||
typed_runners = [runner for runner in runners if isinstance(runner, dict)]
|
||||
capabilities = _capabilities(typed_runners)
|
||||
controller_status = str(controller.get("status") or "unavailable")
|
||||
controller_severity = (
|
||||
controller_status
|
||||
if controller_status in _SEVERITY_RANK
|
||||
else "warning" if controller_status != "unavailable" else "unknown"
|
||||
)
|
||||
action_severity = str(actions.get("severity") or "unknown")
|
||||
capability_severity = _severity(*(row["status"] for row in capabilities))
|
||||
return {
|
||||
"status": _severity(action_severity, controller_severity, capability_severity),
|
||||
"generatedAt": actions.get("generatedAt"),
|
||||
"runners": typed_runners,
|
||||
"capabilities": capabilities,
|
||||
"queue": actions.get("queue") if isinstance(actions.get("queue"), dict) else {},
|
||||
"controller": controller,
|
||||
"actions": [
|
||||
{"intent": "runner.inspect", "mode": "read_only"},
|
||||
{"intent": "runner.cleanup.preview", "mode": "read_only"},
|
||||
{"intent": "runner.restart", "mode": "root_allowlisted_and_bounded"},
|
||||
{"intent": "runner.drain", "mode": "operator_controlled"},
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,531 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = re.compile(r"^jens/[a-z0-9][a-z0-9_.-]{0,96}$")
|
||||
_BRANCH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
|
||||
_CONTROLLER_ROOT = "http://198.19.0.1:1299"
|
||||
|
||||
_PROVISION_SCRIPT = r"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
repository, branch, controller_root = sys.argv[1:4]
|
||||
repository_pattern = re.compile(r"^jens/[a-z0-9][a-z0-9_.-]{0,96}$")
|
||||
branch_pattern = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
|
||||
if (
|
||||
not repository_pattern.fullmatch(repository)
|
||||
or not branch_pattern.fullmatch(branch)
|
||||
or ".." in branch
|
||||
or "//" in branch
|
||||
or branch.endswith("/")
|
||||
or controller_root != "http://198.19.0.1:1299"
|
||||
):
|
||||
raise SystemExit(40)
|
||||
|
||||
projects_path = Path("/opt/gitea-deploy/projects.json")
|
||||
secret_root = Path("/opt/gitea-deploy/webhook-secrets")
|
||||
admin_token_path = Path(
|
||||
"/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token"
|
||||
)
|
||||
secret_name = repository.replace("/", "__")
|
||||
secret_path = secret_root / secret_name
|
||||
endpoint = f"{controller_root}/v1/gitea/push/{repository}"
|
||||
expected_validation = {
|
||||
"validation_gate_version": 1,
|
||||
"required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
|
||||
|
||||
def safe_regular(path: Path, *, private: bool = False) -> None:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise SystemExit(20)
|
||||
info = path.stat()
|
||||
if os.name != "nt" and info.st_uid != os.geteuid():
|
||||
raise SystemExit(21)
|
||||
if private and os.name != "nt" and stat.S_IMODE(info.st_mode) & 0o077:
|
||||
raise SystemExit(22)
|
||||
|
||||
|
||||
def safe_directory(path: Path) -> None:
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise SystemExit(20)
|
||||
info = path.stat()
|
||||
if os.name != "nt" and info.st_uid != os.geteuid():
|
||||
raise SystemExit(21)
|
||||
if os.name != "nt" and stat.S_IMODE(info.st_mode) & 0o077:
|
||||
raise SystemExit(22)
|
||||
|
||||
|
||||
safe_regular(projects_path)
|
||||
safe_regular(admin_token_path, private=True)
|
||||
safe_directory(secret_root)
|
||||
|
||||
try:
|
||||
original_projects_bytes = projects_path.read_bytes()
|
||||
projects = json.loads(original_projects_bytes)
|
||||
except Exception:
|
||||
raise SystemExit(23)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
if (
|
||||
not isinstance(project, dict)
|
||||
or project.get("branch") != branch
|
||||
or project.get("exact_source") is not True
|
||||
or not isinstance(project.get("compose_files"), list)
|
||||
or not project.get("compose_files")
|
||||
or not isinstance(project.get("containers"), list)
|
||||
or not project.get("containers")
|
||||
):
|
||||
raise SystemExit(24)
|
||||
|
||||
triplet = {
|
||||
"gitea_repository_id",
|
||||
"gitea_webhook_secret_file",
|
||||
"validation",
|
||||
}
|
||||
present = triplet.intersection(project)
|
||||
if present and present != triplet:
|
||||
raise SystemExit(25)
|
||||
already_configured = present == triplet
|
||||
if already_configured:
|
||||
repository_id = project.get("gitea_repository_id")
|
||||
if (
|
||||
isinstance(repository_id, bool)
|
||||
or not isinstance(repository_id, int)
|
||||
or repository_id <= 0
|
||||
or project.get("gitea_webhook_secret_file") != secret_name
|
||||
or project.get("validation") != expected_validation
|
||||
):
|
||||
raise SystemExit(26)
|
||||
safe_regular(secret_path, private=True)
|
||||
|
||||
token = admin_token_path.read_text(encoding="utf-8").strip()
|
||||
if not 20 <= len(token) <= 512 or any(ch in token for ch in "\r\n"):
|
||||
raise SystemExit(27)
|
||||
|
||||
health = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"Gitea",
|
||||
"curl",
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--max-time",
|
||||
"5",
|
||||
f"{controller_root}/health",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if health.returncode != 0:
|
||||
raise SystemExit(28)
|
||||
try:
|
||||
health_payload = json.loads(health.stdout)
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(29)
|
||||
if (
|
||||
not isinstance(health_payload, dict)
|
||||
or health_payload.get("status") != "ok"
|
||||
or health_payload.get("protocolVersion") != "itworx-controller-request/v2"
|
||||
):
|
||||
raise SystemExit(29)
|
||||
|
||||
|
||||
def request(method: str, path: str, payload=None):
|
||||
cmd = [
|
||||
"docker",
|
||||
"exec",
|
||||
"-i",
|
||||
"Gitea",
|
||||
"curl",
|
||||
"--fail",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--max-time",
|
||||
"15",
|
||||
"--request",
|
||||
method,
|
||||
"--header",
|
||||
f"Authorization: token {token}",
|
||||
"--header",
|
||||
"Accept: application/json",
|
||||
]
|
||||
raw = None
|
||||
if payload is not None:
|
||||
cmd.extend(
|
||||
[
|
||||
"--header",
|
||||
"Content-Type: application/json",
|
||||
"--data-binary",
|
||||
"@-",
|
||||
]
|
||||
)
|
||||
raw = json.dumps(payload, separators=(",", ":"))
|
||||
cmd.append(f"http://127.0.0.1:3000/api/v1{path}")
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=raw,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(30)
|
||||
if not result.stdout.strip():
|
||||
return None
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(31)
|
||||
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
repo = request("GET", f"/repos/{owner}/{name}")
|
||||
if (
|
||||
not isinstance(repo, dict)
|
||||
or not isinstance(repo.get("id"), int)
|
||||
or isinstance(repo.get("id"), bool)
|
||||
or str(repo.get("full_name") or "").lower() != repository
|
||||
or repo.get("default_branch") != branch
|
||||
):
|
||||
raise SystemExit(32)
|
||||
repository_id = int(repo["id"])
|
||||
if already_configured and project.get("gitea_repository_id") != repository_id:
|
||||
raise SystemExit(33)
|
||||
|
||||
hooks = request("GET", f"/repos/{owner}/{name}/hooks")
|
||||
if not isinstance(hooks, list):
|
||||
raise SystemExit(34)
|
||||
exact = []
|
||||
semantic_other = []
|
||||
for hook in hooks:
|
||||
if not isinstance(hook, dict):
|
||||
continue
|
||||
config = hook.get("config")
|
||||
url = str(config.get("url") or "") if isinstance(config, dict) else ""
|
||||
if url == endpoint:
|
||||
exact.append(hook)
|
||||
elif url.endswith(f"/v1/gitea/push/{repository}"):
|
||||
semantic_other.append(hook)
|
||||
if len(exact) > 1 or semantic_other:
|
||||
raise SystemExit(35)
|
||||
if exact and not already_configured:
|
||||
# Never adopt an unknown existing hook/secret relationship.
|
||||
raise SystemExit(36)
|
||||
|
||||
created_secret = False
|
||||
created_hook_id = None
|
||||
project_replaced = False
|
||||
|
||||
try:
|
||||
if already_configured:
|
||||
secret = secret_path.read_text(encoding="utf-8").strip()
|
||||
else:
|
||||
secret = secrets.token_hex(32)
|
||||
temporary_secret = secret_root / f".{secret_name}.{os.getpid()}.tmp"
|
||||
descriptor = os.open(
|
||||
temporary_secret,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o600,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(secret + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_secret, secret_path)
|
||||
created_secret = True
|
||||
finally:
|
||||
try:
|
||||
temporary_secret.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
project["gitea_repository_id"] = repository_id
|
||||
project["gitea_webhook_secret_file"] = secret_name
|
||||
project["validation"] = expected_validation
|
||||
temporary_projects = projects_path.with_name(
|
||||
f".{projects_path.name}.runnerless-{os.getpid()}.tmp"
|
||||
)
|
||||
payload = (
|
||||
json.dumps(projects, sort_keys=True, separators=(",", ":")) + "\n"
|
||||
)
|
||||
descriptor = os.open(
|
||||
temporary_projects,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o600,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_projects, projects_path)
|
||||
project_replaced = True
|
||||
directory_fd = os.open(str(projects_path.parent), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
finally:
|
||||
try:
|
||||
temporary_projects.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if not 32 <= len(secret) <= 256 or any(
|
||||
ord(ch) < 33 or ord(ch) > 126 for ch in secret
|
||||
):
|
||||
raise SystemExit(37)
|
||||
|
||||
hook_payload = {
|
||||
"active": True,
|
||||
"branch_filter": branch,
|
||||
"events": ["push"],
|
||||
"config": {
|
||||
"url": endpoint,
|
||||
"content_type": "json",
|
||||
"secret": secret,
|
||||
"insecure_ssl": "0",
|
||||
},
|
||||
}
|
||||
if exact:
|
||||
hook_id = exact[0].get("id")
|
||||
if isinstance(hook_id, bool) or not isinstance(hook_id, int) or hook_id <= 0:
|
||||
raise SystemExit(38)
|
||||
updated = request(
|
||||
"PATCH",
|
||||
f"/repos/{owner}/{name}/hooks/{hook_id}",
|
||||
hook_payload,
|
||||
)
|
||||
else:
|
||||
created = request(
|
||||
"POST",
|
||||
f"/repos/{owner}/{name}/hooks",
|
||||
{"type": "gitea", **hook_payload},
|
||||
)
|
||||
hook_id = created.get("id") if isinstance(created, dict) else None
|
||||
if isinstance(hook_id, bool) or not isinstance(hook_id, int) or hook_id <= 0:
|
||||
raise SystemExit(39)
|
||||
created_hook_id = hook_id
|
||||
updated = created
|
||||
|
||||
if not isinstance(updated, dict):
|
||||
updated = request(
|
||||
"GET",
|
||||
f"/repos/{owner}/{name}/hooks/{hook_id}",
|
||||
)
|
||||
config_out = updated.get("config") if isinstance(updated, dict) else None
|
||||
if (
|
||||
not isinstance(updated, dict)
|
||||
or updated.get("id") != hook_id
|
||||
or updated.get("active") is not True
|
||||
or str(updated.get("branch_filter") or "") != branch
|
||||
or not isinstance(config_out, dict)
|
||||
or config_out.get("url") != endpoint
|
||||
or str(config_out.get("content_type") or "").lower() != "json"
|
||||
or updated.get("events") != ["push"]
|
||||
):
|
||||
raise SystemExit(41)
|
||||
|
||||
except BaseException:
|
||||
rollback_failed = False
|
||||
if created_hook_id is not None:
|
||||
try:
|
||||
request(
|
||||
"DELETE",
|
||||
f"/repos/{owner}/{name}/hooks/{created_hook_id}",
|
||||
)
|
||||
except BaseException:
|
||||
rollback_failed = True
|
||||
if project_replaced:
|
||||
try:
|
||||
temporary_restore = projects_path.with_name(
|
||||
f".{projects_path.name}.restore-{os.getpid()}.tmp"
|
||||
)
|
||||
descriptor = os.open(
|
||||
temporary_restore,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o600,
|
||||
)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(original_projects_bytes)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_restore, projects_path)
|
||||
except BaseException:
|
||||
rollback_failed = True
|
||||
if created_secret:
|
||||
try:
|
||||
secret_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
rollback_failed = True
|
||||
if rollback_failed:
|
||||
raise SystemExit(42)
|
||||
raise
|
||||
|
||||
print(
|
||||
f"runnerless_target_ready repository={repository} branch={branch} "
|
||||
f"hook_id={hook_id} state={'existing' if already_configured else 'provisioned'}",
|
||||
flush=True,
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
class RunnerlessTargetProvisioning:
|
||||
"""Provision runnerless delivery only for an already registered controller target."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self, *, repository: str, branch: str) -> dict[str, Any]:
|
||||
repository = str(repository or "").strip().lower()
|
||||
branch = str(branch or "").strip()
|
||||
if (
|
||||
not _REPOSITORY.fullmatch(repository)
|
||||
or not _BRANCH.fullmatch(branch)
|
||||
or ".." in branch
|
||||
or "//" in branch
|
||||
or branch.endswith("/")
|
||||
):
|
||||
return {
|
||||
"status": "invalid",
|
||||
"reason": "runnerless target identity is outside the bounded contract",
|
||||
}
|
||||
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
controller = (
|
||||
dict(status.get("container") or {})
|
||||
if status.get("status") == "ok"
|
||||
else {}
|
||||
)
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if (
|
||||
controller.get("name") != _CONTROLLER
|
||||
or controller.get("state") != "running"
|
||||
or not container_id
|
||||
or not config
|
||||
):
|
||||
return {
|
||||
"status": "controller_unavailable",
|
||||
"reason": "deployment controller is unavailable",
|
||||
}
|
||||
|
||||
endpoint_id = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
command = [
|
||||
"python3",
|
||||
"-c",
|
||||
_PROVISION_SCRIPT,
|
||||
repository,
|
||||
branch,
|
||||
_CONTROLLER_ROOT,
|
||||
]
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint_id}/docker/containers/{safe_container}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": command,
|
||||
},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {
|
||||
"status": "error",
|
||||
"reason": "runnerless target provisioning could not start",
|
||||
}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint_id}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
deadline = time.monotonic() + 90
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint_id}/docker/exec/{safe_exec}/json",
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "runnerless provisioning timed out; inspect before retrying",
|
||||
}
|
||||
if exit_code == 0:
|
||||
return {
|
||||
"status": "ready",
|
||||
"repository": repository,
|
||||
"branch": branch,
|
||||
"endpoint": f"{_CONTROLLER_ROOT}/v1/gitea/push/{repository}",
|
||||
}
|
||||
|
||||
reasons = {
|
||||
20: "required controller/Gitea path is unavailable or unsafe",
|
||||
21: "required controller/Gitea path ownership is invalid",
|
||||
22: "required private path permissions are too broad",
|
||||
23: "deployment projects registry is invalid",
|
||||
24: "repository is not an already registered exact-source controller target",
|
||||
25: "runnerless target has partial controller metadata",
|
||||
26: "existing runnerless target metadata differs from the fixed contract",
|
||||
27: "Gitea admin credential is unavailable or invalid",
|
||||
28: "Gitea cannot reach the private deployment controller",
|
||||
29: "private deployment controller identity is invalid",
|
||||
30: "bounded Gitea API request failed",
|
||||
31: "Gitea returned invalid JSON",
|
||||
32: "Gitea repository identity or default branch differs from the target",
|
||||
33: "Gitea repository id differs from existing controller metadata",
|
||||
34: "Gitea webhook inventory is invalid",
|
||||
35: "runnerless target has ambiguous webhook endpoints",
|
||||
36: "an unbound runnerless webhook already exists and cannot be adopted",
|
||||
37: "runnerless secret material failed validation",
|
||||
38: "existing runnerless hook id is invalid",
|
||||
39: "Gitea did not return a valid hook after creation",
|
||||
41: "runnerless webhook verification failed after provisioning",
|
||||
42: "runnerless provisioning compensation failed; operator inspection required",
|
||||
40: "runnerless provisioning contract was altered",
|
||||
}
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": reasons.get(
|
||||
exit_code,
|
||||
"runnerless target provisioning failed closed",
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.version import APP_VERSION
|
||||
from ..db.models import (
|
||||
AppContainerMapping,
|
||||
AppSetting,
|
||||
AuditEvent,
|
||||
DiscoveredContainer,
|
||||
NotificationChannel,
|
||||
NotificationRule,
|
||||
Policy,
|
||||
)
|
||||
from .audit_logger import log_event
|
||||
from .secret_store import secret_storage_status
|
||||
from .setup_service import get_setup_status, list_connector_settings
|
||||
|
||||
_BACKUP_LOCK = threading.Lock()
|
||||
_OPERATOR_AUTH_SETTING_KEY = "security.operator_auth"
|
||||
|
||||
|
||||
def apply_stored_operator_auth_setting(session: Session) -> None:
|
||||
stored = session.get(AppSetting, _OPERATOR_AUTH_SETTING_KEY)
|
||||
if stored and isinstance(stored.value_json, dict) and isinstance(stored.value_json.get("required"), bool):
|
||||
settings.operator_auth_required = stored.value_json["required"]
|
||||
|
||||
|
||||
def operator_auth_policy() -> dict[str, bool]:
|
||||
return {"required": settings.operator_auth_required}
|
||||
|
||||
|
||||
def update_operator_auth_policy(session: Session, required: bool) -> dict[str, bool]:
|
||||
stored = session.get(AppSetting, _OPERATOR_AUTH_SETTING_KEY)
|
||||
if stored is None:
|
||||
stored = AppSetting(key=_OPERATOR_AUTH_SETTING_KEY, is_secret=False)
|
||||
stored.value_json = {"required": required}
|
||||
stored.updated_at = datetime.now(timezone.utc)
|
||||
session.add(stored)
|
||||
settings.operator_auth_required = required
|
||||
log_event(
|
||||
session,
|
||||
source="manual",
|
||||
identity="operator",
|
||||
action_id="security.operator_auth.update",
|
||||
target="operator_session",
|
||||
target_type="security_setting",
|
||||
risk="medium",
|
||||
policy_decision="allowed",
|
||||
reason="Operator session requirement changed from Settings.",
|
||||
result="success",
|
||||
redacted_metadata={"required": required},
|
||||
)
|
||||
session.commit()
|
||||
return operator_auth_policy()
|
||||
|
||||
|
||||
def _data_dir() -> Path:
|
||||
return Path(settings.data_dir)
|
||||
|
||||
|
||||
def _static_dir() -> Path:
|
||||
return Path(settings.static_dir)
|
||||
|
||||
|
||||
def _backup_dir() -> Path:
|
||||
return _data_dir() / "backups"
|
||||
|
||||
|
||||
def is_data_dir_writable() -> bool:
|
||||
try:
|
||||
_data_dir().mkdir(parents=True, exist_ok=True)
|
||||
probe = _data_dir() / ".appops-write-test"
|
||||
probe.write_text("ok", encoding="utf-8")
|
||||
probe.unlink(missing_ok=True)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def db_status(session: Session) -> str:
|
||||
try:
|
||||
session.exec(select(Policy).limit(1)).all()
|
||||
return "ok"
|
||||
except Exception:
|
||||
return "error"
|
||||
|
||||
|
||||
def frontend_assets_available() -> bool:
|
||||
return (_static_dir() / "index.html").exists()
|
||||
|
||||
|
||||
def production_warnings() -> list[str]:
|
||||
warnings: list[str] = []
|
||||
if settings.environment == "production":
|
||||
proxy_operator_ready = bool(
|
||||
settings.operator_proxy_auth_enabled
|
||||
and settings.operator_proxy_trusted_networks
|
||||
and (settings.operator_proxy_users or settings.operator_proxy_groups)
|
||||
)
|
||||
if not settings.operator_auth_required:
|
||||
warnings.append("APPOPS_REQUIRE_OPERATOR_AUTH is disabled; private LAN operator routes are unlocked.")
|
||||
elif not proxy_operator_ready and (not settings.operator_token or settings.operator_token in {"change-me", "changeme", "dev", "test"}):
|
||||
warnings.append("APPOPS_OPERATOR_TOKEN is missing or unsafe; protected operator routes fail closed.")
|
||||
if settings.operator_proxy_auth_enabled and not proxy_operator_ready:
|
||||
warnings.append("Account-bound operator auth is enabled but trusted proxy networks and an allowed user or group are not fully configured.")
|
||||
if not settings.voice_token or settings.voice_token in {"change-me", "changeme", "dev", "test"}:
|
||||
warnings.append("APP_OPS_VOICE_TOKEN is missing or unsafe for production.")
|
||||
warnings.extend(secret_storage_status()["warnings"])
|
||||
return warnings
|
||||
|
||||
|
||||
def connector_config_summary(session: Session) -> dict[str, Any]:
|
||||
return {
|
||||
item["connectorId"]: {
|
||||
"configured": item["configured"],
|
||||
"hasSecret": item["hasSecret"],
|
||||
"missing": item["missing"],
|
||||
}
|
||||
for item in list_connector_settings(session)
|
||||
}
|
||||
|
||||
|
||||
def runtime_info(session: Session) -> dict[str, Any]:
|
||||
setup_status = get_setup_status(session)
|
||||
storage_status = secret_storage_status()
|
||||
notification_rules = session.exec(select(NotificationRule)).all()
|
||||
notification_channels = session.exec(select(NotificationChannel)).all()
|
||||
source_metadata = Path("/etc/appops/source-metadata")
|
||||
source_lines = source_metadata.read_text(encoding="utf-8").splitlines() if source_metadata.is_file() else []
|
||||
source_repository, source_revision, source_tree = source_lines if len(source_lines) == 3 else ("unknown", "unknown", "unknown")
|
||||
return {
|
||||
"version": APP_VERSION,
|
||||
"sourceRepository": source_repository,
|
||||
"sourceRevision": source_revision,
|
||||
"sourceTree": source_tree,
|
||||
"environment": settings.environment,
|
||||
"dataDirConfigured": bool(settings.data_dir),
|
||||
"dataDir": str(_data_dir()),
|
||||
"dbPath": settings.db_path,
|
||||
"dbStatus": db_status(session),
|
||||
"frontendMode": "static" if frontend_assets_available() else "api_only",
|
||||
"frontendAssetsAvailable": frontend_assets_available(),
|
||||
"setupComplete": setup_status["setupComplete"],
|
||||
"liveReady": setup_status["liveReady"],
|
||||
"firstRunCompleted": setup_status["firstRunCompleted"],
|
||||
"coverageComplete": setup_status["coverageComplete"],
|
||||
"missingOptionalSources": setup_status["missingOptionalSources"],
|
||||
"canonicalPort": 1216,
|
||||
"canonicalInstance": settings.port == 1216,
|
||||
"secretStorageStatus": storage_status,
|
||||
"notificationRulesStatus": {
|
||||
"rules": len(notification_rules),
|
||||
"channels": len(notification_channels),
|
||||
"homeAssistantConfigured": any(channel.id == "home_assistant" and channel.enabled for channel in notification_channels),
|
||||
},
|
||||
"setupStatus": setup_status,
|
||||
"connectorConfigSummary": connector_config_summary(session),
|
||||
"secretStorage": storage_status,
|
||||
"productionWarnings": production_warnings(),
|
||||
}
|
||||
|
||||
|
||||
def deep_health(session: Session) -> dict[str, Any]:
|
||||
checks = {
|
||||
"dbReachable": db_status(session) == "ok",
|
||||
"dataDirWritable": is_data_dir_writable(),
|
||||
"frontendAssetsAvailable": frontend_assets_available(),
|
||||
"secretStorageReady": secret_storage_status()["status"] in {"ready"},
|
||||
}
|
||||
return {
|
||||
"status": "ok" if all(checks.values()) or (checks["dbReachable"] and checks["dataDirWritable"]) else "degraded",
|
||||
"checks": checks,
|
||||
"setup": get_setup_status(session),
|
||||
"connectors": connector_config_summary(session),
|
||||
"secretStorage": secret_storage_status(),
|
||||
"warnings": production_warnings(),
|
||||
}
|
||||
|
||||
|
||||
def _backup_metadata(path: Path) -> dict[str, Any]:
|
||||
stat = path.stat()
|
||||
created = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
manifest_path = path.with_suffix('.manifest.json')
|
||||
return {
|
||||
"name": path.name,
|
||||
"path": str(path),
|
||||
"sizeBytes": stat.st_size,
|
||||
"createdAt": created,
|
||||
"sha256": digest.hexdigest(),
|
||||
"manifestAvailable": manifest_path.is_file(),
|
||||
}
|
||||
|
||||
|
||||
def _backup_manifest_path(path: Path) -> Path:
|
||||
return path.with_suffix('.manifest.json')
|
||||
|
||||
|
||||
def list_sqlite_backups(limit: int = 20) -> dict[str, Any]:
|
||||
backup_dir = _backup_dir()
|
||||
backups = []
|
||||
if backup_dir.exists():
|
||||
backups = sorted(
|
||||
(item for item in backup_dir.glob("appops-db-*.db") if item.is_file()),
|
||||
key=lambda item: item.stat().st_mtime,
|
||||
reverse=True,
|
||||
)[:limit]
|
||||
return {
|
||||
"dbPath": settings.db_path,
|
||||
"backupDir": str(backup_dir),
|
||||
"backups": [_backup_metadata(item) for item in backups],
|
||||
"restoreMode": "manual_stop_container",
|
||||
}
|
||||
|
||||
|
||||
def sqlite_maintenance_status() -> dict[str, Any]:
|
||||
db_path = Path(settings.db_path)
|
||||
wal_path = Path(f"{settings.db_path}-wal")
|
||||
result: dict[str, Any] = {
|
||||
"status": "missing" if not db_path.is_file() else "ready",
|
||||
"databaseBytes": db_path.stat().st_size if db_path.is_file() else 0,
|
||||
"walBytes": wal_path.stat().st_size if wal_path.is_file() else 0,
|
||||
"journalMode": "unknown",
|
||||
"pageCount": 0,
|
||||
"freePages": 0,
|
||||
"reclaimableBytes": 0,
|
||||
"recommendation": "none",
|
||||
}
|
||||
if not db_path.is_file():
|
||||
return result
|
||||
try:
|
||||
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5)
|
||||
try:
|
||||
page_size = int(connection.execute("PRAGMA page_size").fetchone()[0])
|
||||
result["pageCount"] = int(connection.execute("PRAGMA page_count").fetchone()[0])
|
||||
result["freePages"] = int(connection.execute("PRAGMA freelist_count").fetchone()[0])
|
||||
result["journalMode"] = str(connection.execute("PRAGMA journal_mode").fetchone()[0])
|
||||
result["reclaimableBytes"] = result["freePages"] * page_size
|
||||
finally:
|
||||
connection.close()
|
||||
except (OSError, sqlite3.Error, TypeError, ValueError):
|
||||
result["status"] = "error"
|
||||
return result
|
||||
|
||||
if result["walBytes"] >= 16 * 1024 * 1024:
|
||||
result["recommendation"] = "checkpoint"
|
||||
elif result["reclaimableBytes"] >= max(32 * 1024 * 1024, result["databaseBytes"] // 4):
|
||||
result["recommendation"] = "offline_vacuum_after_backup"
|
||||
return result
|
||||
|
||||
|
||||
def checkpoint_sqlite(session: Session) -> dict[str, Any]:
|
||||
before = sqlite_maintenance_status()
|
||||
backup = create_sqlite_backup(session)
|
||||
if not backup.get("success"):
|
||||
return {"success": False, "status": "backup_failed", "before": before, "backup": backup}
|
||||
try:
|
||||
connection = sqlite3.connect(settings.db_path, timeout=30)
|
||||
try:
|
||||
checkpoint = connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
|
||||
finally:
|
||||
connection.close()
|
||||
after = sqlite_maintenance_status()
|
||||
success = bool(checkpoint) and int(checkpoint[0]) == 0
|
||||
except (OSError, sqlite3.Error, TypeError, ValueError):
|
||||
checkpoint = None
|
||||
after = sqlite_maintenance_status()
|
||||
success = False
|
||||
audit = log_event(
|
||||
session,
|
||||
source="manual",
|
||||
identity="operator",
|
||||
action_id="runtime.sqlite.checkpoint",
|
||||
target="appops.db",
|
||||
target_type="runtime_database",
|
||||
risk="low",
|
||||
policy_decision="allowed",
|
||||
reason="Operator requested a backup-first SQLite WAL checkpoint.",
|
||||
result="success" if success else "failed",
|
||||
redacted_metadata={"beforeWalBytes": before["walBytes"], "afterWalBytes": after["walBytes"]},
|
||||
)
|
||||
return {
|
||||
"success": success,
|
||||
"status": "checkpointed" if success else "failed",
|
||||
"before": before,
|
||||
"after": after,
|
||||
"backup": backup.get("backup"),
|
||||
"auditId": audit.id,
|
||||
}
|
||||
|
||||
|
||||
def _sqlite_backup_path(name: str) -> Path | None:
|
||||
normalized_name = Path(str(name)).name
|
||||
if normalized_name != name or not normalized_name.startswith('appops-db-') or not normalized_name.endswith('.db'):
|
||||
return None
|
||||
backup_dir = _backup_dir().resolve()
|
||||
candidate = (backup_dir / normalized_name).resolve()
|
||||
if candidate.parent != backup_dir or not candidate.is_file():
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def get_sqlite_backup_path(name: str) -> Path | None:
|
||||
return _sqlite_backup_path(name)
|
||||
|
||||
|
||||
def verify_sqlite_backup(
|
||||
session: Session,
|
||||
name: str,
|
||||
*,
|
||||
source: str = 'manual',
|
||||
identity: str = 'operator',
|
||||
reason: str = 'Existing SQLite backup integrity verification requested from Settings.',
|
||||
) -> dict[str, Any]:
|
||||
path = _sqlite_backup_path(name)
|
||||
if not path:
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'not_found',
|
||||
'message': 'SQLite backup not found.',
|
||||
'restoreMode': 'manual_stop_container',
|
||||
}
|
||||
integrity_check = 'unknown'
|
||||
restore_check = 'unknown'
|
||||
manifest_check = 'legacy_missing'
|
||||
try:
|
||||
metadata = _backup_metadata(path)
|
||||
manifest_path = _backup_manifest_path(path)
|
||||
if manifest_path.is_file():
|
||||
manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
|
||||
manifest_check = 'ok' if (
|
||||
manifest.get('databaseFile') == path.name
|
||||
and manifest.get('sha256') == metadata['sha256']
|
||||
and manifest.get('sizeBytes') == metadata['sizeBytes']
|
||||
) else 'mismatch'
|
||||
connection = sqlite3.connect(f'file:{path}?mode=ro', uri=True)
|
||||
try:
|
||||
result = connection.execute('PRAGMA integrity_check').fetchone()
|
||||
finally:
|
||||
connection.close()
|
||||
integrity_check = str(result[0]) if result else 'unknown'
|
||||
with tempfile.TemporaryDirectory(prefix="appops-restore-check-") as temp_dir:
|
||||
restored = Path(temp_dir) / "restored.db"
|
||||
shutil.copy2(path, restored)
|
||||
restored_connection = sqlite3.connect(restored)
|
||||
try:
|
||||
restored_integrity = restored_connection.execute("PRAGMA integrity_check").fetchone()
|
||||
restored_connection.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
|
||||
finally:
|
||||
restored_connection.close()
|
||||
restore_check = str(restored_integrity[0]) if restored_integrity else 'unknown'
|
||||
success = integrity_check == 'ok' and restore_check == 'ok' and manifest_check != 'mismatch'
|
||||
except (OSError, ValueError, json.JSONDecodeError, sqlite3.Error):
|
||||
success = False
|
||||
|
||||
audit = log_event(
|
||||
session,
|
||||
source=source,
|
||||
identity=identity,
|
||||
action_id='runtime.sqlite_backup.verify',
|
||||
target=path.name,
|
||||
target_type='runtime_backup',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason=reason,
|
||||
result='success' if success else 'failed',
|
||||
execution_plan={'steps': ['open backup read-only', 'PRAGMA integrity_check', 'write audit']},
|
||||
post_checks={'integrityCheck': integrity_check, 'restoreCheck': restore_check, 'manifestCheck': manifest_check, 'backupUnchanged': True},
|
||||
redacted_metadata={'backupName': path.name, 'sizeBytes': path.stat().st_size},
|
||||
)
|
||||
return {
|
||||
'success': success,
|
||||
'status': 'verified' if success else 'failed',
|
||||
'message': 'SQLite backup integrity is valid.' if success else 'SQLite backup integrity check failed.',
|
||||
'backup': _backup_metadata(path),
|
||||
'integrityCheck': integrity_check,
|
||||
'restoreCheck': restore_check,
|
||||
'manifestCheck': manifest_check,
|
||||
'auditId': audit.id,
|
||||
'restoreMode': 'manual_stop_container',
|
||||
}
|
||||
|
||||
|
||||
def verify_latest_backup_drill(session: Session) -> dict[str, Any]:
|
||||
backups = list_sqlite_backups(limit=1).get('backups', [])
|
||||
if not backups:
|
||||
return {'success': False, 'status': 'no_backup', 'restoreMode': 'manual_stop_container'}
|
||||
return verify_sqlite_backup(
|
||||
session,
|
||||
str(backups[0]['name']),
|
||||
source='system',
|
||||
identity='restore-drill',
|
||||
reason='Scheduled read-only backup integrity and disposable restore-copy drill.',
|
||||
)
|
||||
|
||||
|
||||
def create_sqlite_backup(session: Session) -> dict[str, Any]:
|
||||
source = Path(settings.db_path)
|
||||
destination_dir = _backup_dir()
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
|
||||
destination = destination_dir / f"appops-db-{timestamp}.db"
|
||||
|
||||
if not _BACKUP_LOCK.acquire(blocking=False):
|
||||
return {"success": False, "status": "busy", "message": "A SQLite backup is already running.", "restoreMode": "manual_stop_container"}
|
||||
try:
|
||||
if not source.exists():
|
||||
raise FileNotFoundError("configured SQLite database does not exist")
|
||||
destination_dir.mkdir(parents=True, exist_ok=True)
|
||||
free_bytes = shutil.disk_usage(destination_dir).free
|
||||
required_bytes = max(source.stat().st_size * 2, 16 * 1024 * 1024)
|
||||
if free_bytes < required_bytes:
|
||||
raise OSError("insufficient_space_for_backup")
|
||||
with sqlite3.connect(source) as source_conn, sqlite3.connect(destination) as destination_conn:
|
||||
source_conn.backup(destination_conn)
|
||||
integrity = destination_conn.execute("PRAGMA integrity_check").fetchone()
|
||||
integrity_check = str(integrity[0]) if integrity else "unknown"
|
||||
success = integrity_check == "ok"
|
||||
metadata = _backup_metadata(destination)
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"appVersion": APP_VERSION,
|
||||
"databaseFile": destination.name,
|
||||
"createdAt": metadata["createdAt"],
|
||||
"sizeBytes": metadata["sizeBytes"],
|
||||
"sha256": metadata["sha256"],
|
||||
"integrityCheck": integrity_check,
|
||||
"restoreMode": "manual_stop_container",
|
||||
"secretKeySource": settings.secret_key_source,
|
||||
}
|
||||
_backup_manifest_path(destination).write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
metadata["manifestAvailable"] = True
|
||||
retained = sorted(
|
||||
(item for item in destination_dir.glob("appops-db-*.db") if item.is_file()),
|
||||
key=lambda item: item.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
for stale in retained[settings.backup_retention:]:
|
||||
stale.unlink()
|
||||
_backup_manifest_path(stale).unlink(missing_ok=True)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="manual",
|
||||
identity="operator",
|
||||
action_id="runtime.sqlite_backup",
|
||||
target="appops-db",
|
||||
target_type="runtime",
|
||||
risk="low",
|
||||
policy_decision="allowed",
|
||||
reason="SQLite online backup requested from Settings.",
|
||||
result="success" if success else "failed",
|
||||
execution_plan={"steps": ["sqlite backup api", "integrity_check", "write audit"]},
|
||||
post_checks={"integrityCheck": integrity_check, "backupCreated": destination.exists()},
|
||||
redacted_metadata={"backupPath": str(destination), "sizeBytes": metadata["sizeBytes"]},
|
||||
)
|
||||
return {
|
||||
"success": success,
|
||||
"status": "created" if success else "failed",
|
||||
"message": "SQLite backup created." if success else "SQLite backup integrity check failed.",
|
||||
"backup": metadata,
|
||||
"integrityCheck": integrity_check,
|
||||
"auditId": audit.id,
|
||||
"restoreMode": "manual_stop_container",
|
||||
}
|
||||
except (OSError, sqlite3.Error) as exc:
|
||||
audit = log_event(
|
||||
session,
|
||||
source="manual",
|
||||
identity="operator",
|
||||
action_id="runtime.sqlite_backup",
|
||||
target="appops-db",
|
||||
target_type="runtime",
|
||||
risk="low",
|
||||
policy_decision="allowed",
|
||||
reason="SQLite online backup requested from Settings.",
|
||||
result="failed",
|
||||
execution_plan={"steps": ["sqlite backup api", "integrity_check", "write audit"]},
|
||||
post_checks={"backupCreated": False},
|
||||
redacted_metadata={"errorType": type(exc).__name__},
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"status": "failed",
|
||||
"message": "SQLite backup failed.",
|
||||
"reason": type(exc).__name__,
|
||||
"auditId": audit.id,
|
||||
"restoreMode": "manual_stop_container",
|
||||
}
|
||||
finally:
|
||||
_BACKUP_LOCK.release()
|
||||
|
||||
|
||||
def export_config(session: Session) -> dict[str, Any]:
|
||||
return {
|
||||
"runtime": {
|
||||
"environment": settings.environment,
|
||||
"dataDir": str(_data_dir()),
|
||||
"dbPath": settings.db_path,
|
||||
"frontendOrigin": settings.frontend_origin,
|
||||
"secretStorage": secret_storage_status(),
|
||||
},
|
||||
"connectors": list_connector_settings(session),
|
||||
"setup": get_setup_status(session),
|
||||
"note": "Secret values are intentionally excluded.",
|
||||
}
|
||||
|
||||
|
||||
def export_policies(session: Session) -> dict[str, Any]:
|
||||
policies = session.exec(select(Policy).order_by(Policy.id)).all()
|
||||
return {
|
||||
"policies": [
|
||||
{
|
||||
"id": policy.id,
|
||||
"action": policy.action,
|
||||
"target": policy.target,
|
||||
"voice": policy.voice,
|
||||
"manual": policy.manual,
|
||||
"risk": policy.risk,
|
||||
"reason": policy.reason,
|
||||
}
|
||||
for policy in policies
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def export_inventory(session: Session) -> dict[str, Any]:
|
||||
discovered = session.exec(select(DiscoveredContainer).order_by(DiscoveredContainer.name)).all()
|
||||
mappings = session.exec(select(AppContainerMapping).order_by(AppContainerMapping.container_name)).all()
|
||||
return {
|
||||
"discovered": [
|
||||
{
|
||||
"id": item.id,
|
||||
"containerId": item.container_id,
|
||||
"name": item.name,
|
||||
"image": item.image,
|
||||
"state": item.state,
|
||||
"status": item.status,
|
||||
"suggestedAppType": item.suggested_app_type,
|
||||
"suggestedCategory": item.suggested_category,
|
||||
"suggestedCriticality": item.suggested_criticality,
|
||||
"suggestedConnector": item.suggested_connector,
|
||||
"importStatus": item.import_status,
|
||||
"matchedAppId": item.matched_app_id,
|
||||
}
|
||||
for item in discovered
|
||||
],
|
||||
"mappings": [
|
||||
{
|
||||
"id": mapping.id,
|
||||
"appId": mapping.app_id,
|
||||
"containerId": mapping.container_id,
|
||||
"containerName": mapping.container_name,
|
||||
"role": mapping.role,
|
||||
"source": mapping.source,
|
||||
"isPrimary": mapping.is_primary,
|
||||
}
|
||||
for mapping in mappings
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def export_audit_summary(session: Session) -> dict[str, Any]:
|
||||
events = session.exec(select(AuditEvent).order_by(AuditEvent.timestamp.desc())).all()
|
||||
by_result: dict[str, int] = {}
|
||||
by_action: dict[str, int] = {}
|
||||
for event in events:
|
||||
by_result[event.result] = by_result.get(event.result, 0) + 1
|
||||
by_action[event.action_id] = by_action.get(event.action_id, 0) + 1
|
||||
return {
|
||||
"total": len(events),
|
||||
"byResult": by_result,
|
||||
"byAction": by_action,
|
||||
"recent": [
|
||||
{
|
||||
"id": event.id,
|
||||
"timestamp": event.timestamp.isoformat(),
|
||||
"source": event.source,
|
||||
"action": event.action_id,
|
||||
"target": event.target,
|
||||
"risk": event.risk,
|
||||
"policyDecision": event.policy_decision,
|
||||
"result": event.result,
|
||||
}
|
||||
for event in events[:25]
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,850 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from threading import Lock, Thread
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import App, AppSetting, AuditEvent, Connector, DiscoveryRun, JobRun, Policy, Secret
|
||||
from . import ai_remediation_planner, ai_security_reviewer
|
||||
from .audit_logger import log_event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_CACHE_TTL_SECONDS = 60
|
||||
CACHE_KEY = 'safety-dashboard-v1'
|
||||
CACHE_SCHEMA_VERSION = 2
|
||||
CACHE_SETTING_KEY = 'safety_center:dashboard_cache'
|
||||
DEFAULT_PERSISTENT_CACHE_MAX_STALE_SECONDS = 900
|
||||
|
||||
_dashboard_cache: dict[str, Any] | None = None
|
||||
_skip_persistent_cache_once = False
|
||||
_background_refresh_lock = Lock()
|
||||
_background_refresh_running = False
|
||||
|
||||
|
||||
SECTION_DEFINITIONS = [
|
||||
('setup-coverage', 'Setup & Coverage', '/setup', {'coverage', 'connector_config', 'setup'}),
|
||||
('external-exposure', 'External Exposure', '/network', {'exposure'}),
|
||||
('authentication-coverage', 'Authentication Coverage', '/apps/authentik', {'auth'}),
|
||||
('secret-hygiene', 'Secret Hygiene', '/secrets', {'secrets'}),
|
||||
('policy-posture', 'Policy Posture', '/policies', {'policies'}),
|
||||
('inventory-app-review', 'Inventory & App Review', '/inventory', {'inventory', 'connectors'}),
|
||||
('media-pipeline', 'Media Pipeline', '/media-pipeline', {'media'}),
|
||||
('unraid-host-safety', 'Unraid & Host Safety', '/unraid', {'unraid'}),
|
||||
('audit-activity', 'Audit & Activity', '/audit', {'audit'}),
|
||||
]
|
||||
|
||||
SEVERITY_PENALTY = {'critical': 20, 'high': 12, 'medium': 6, 'low': 2, 'info': 0}
|
||||
CATEGORY_MAP = {
|
||||
'auth_coverage': 'auth',
|
||||
'coverage': 'setup',
|
||||
'connector_config': 'connectors',
|
||||
}
|
||||
SOURCE_LABELS = {
|
||||
'inventory': 'Inventory',
|
||||
'portainer': 'Portainer',
|
||||
'unraid': 'Unraid',
|
||||
'npm': 'NPM',
|
||||
'authentik': 'Authentik',
|
||||
'secret_storage': 'Secret Storage',
|
||||
'policies': 'Policies',
|
||||
'audit': 'Audit',
|
||||
'media_pipeline': 'Media Pipeline',
|
||||
}
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def _parse_utc(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
|
||||
|
||||
def _ttl_seconds() -> int:
|
||||
raw = os.getenv('SAFETY_CACHE_TTL_SECONDS', str(DEFAULT_CACHE_TTL_SECONDS))
|
||||
try:
|
||||
return max(1, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_CACHE_TTL_SECONDS
|
||||
|
||||
|
||||
def _persistent_cache_max_stale_seconds() -> int:
|
||||
raw = os.getenv('SAFETY_PERSISTENT_CACHE_MAX_STALE_SECONDS', str(DEFAULT_PERSISTENT_CACHE_MAX_STALE_SECONDS))
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_PERSISTENT_CACHE_MAX_STALE_SECONDS
|
||||
|
||||
|
||||
def invalidate_cache() -> None:
|
||||
global _dashboard_cache, _skip_persistent_cache_once
|
||||
_dashboard_cache = None
|
||||
_skip_persistent_cache_once = True
|
||||
|
||||
|
||||
def _redact(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return redact_message(value)
|
||||
if isinstance(value, list):
|
||||
return [_redact(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
str(key): _redact(item)
|
||||
for key, item in value.items()
|
||||
if 'secret' not in str(key).lower() and 'token' not in str(key).lower()
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def _state_key(kind: str, item_id: str) -> str:
|
||||
return f'safety_center:{kind}:{item_id}:status'
|
||||
|
||||
|
||||
def _get_state(session: Session, kind: str, item_id: str, default: str = 'open') -> str:
|
||||
row = session.get(AppSetting, _state_key(kind, item_id))
|
||||
if not row:
|
||||
return default
|
||||
return str(row.value_json.get('status') or default)
|
||||
|
||||
|
||||
def _set_state(session: Session, kind: str, item_id: str, status: str) -> dict[str, Any]:
|
||||
row = session.get(AppSetting, _state_key(kind, item_id))
|
||||
if not row:
|
||||
row = AppSetting(key=_state_key(kind, item_id), is_secret=False)
|
||||
row.value_json = {'status': status, 'updatedAt': _now()}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='safety_center',
|
||||
action_id=f'safety.{kind}.{status}',
|
||||
target=item_id,
|
||||
target_type=kind,
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason=f'Safety Center {kind} marked {status}.',
|
||||
result='success',
|
||||
redacted_metadata={'kind': kind, 'id': item_id, 'status': status},
|
||||
)
|
||||
invalidate_cache()
|
||||
return {'success': True, 'id': item_id, 'status': status}
|
||||
|
||||
|
||||
def _category(category: str) -> str:
|
||||
return CATEGORY_MAP.get(category, category)
|
||||
|
||||
|
||||
def _action_type(raw: str | None) -> str:
|
||||
allowed = {
|
||||
'open_page',
|
||||
'configure_connector',
|
||||
'configure_secret',
|
||||
'run_review',
|
||||
'create_remediation_plan',
|
||||
'run_readonly_check',
|
||||
'review_policy',
|
||||
}
|
||||
if raw == 'run_readonly_check':
|
||||
return raw
|
||||
if raw in {'policy_draft', 'review_policy'}:
|
||||
return 'review_policy'
|
||||
if raw == 'configure_secret':
|
||||
return raw
|
||||
if raw == 'configure_connector':
|
||||
return raw
|
||||
return 'open_page' if raw in {None, 'link'} else (raw if raw in allowed else 'open_page')
|
||||
|
||||
|
||||
def _suggested_actions(actions: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
result: list[dict[str, Any]] = []
|
||||
for action in actions:
|
||||
label = str(action.get('label') or action.get('type') or 'Open details')
|
||||
href = action.get('href') or action.get('link')
|
||||
result.append({
|
||||
'label': _redact(label),
|
||||
'actionType': _action_type(action.get('type')),
|
||||
'href': href or '/safety',
|
||||
'policyDecision': action.get('policyDecision'),
|
||||
'executable': False,
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def _findings(session: Session, review: Any) -> list[dict[str, Any]]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
for finding in review.findings:
|
||||
source_status = _get_state(session, 'finding', finding.id, finding.status)
|
||||
category = _category(finding.category)
|
||||
findings.append({
|
||||
'id': finding.id,
|
||||
'title': _redact(finding.title),
|
||||
'description': _redact(finding.description),
|
||||
'severity': finding.severity,
|
||||
'category': category,
|
||||
'affectedResources': _redact(finding.affectedResources),
|
||||
'evidence': _redact([item.model_dump() for item in finding.evidence]),
|
||||
'suggestedActions': _suggested_actions([item.model_dump() for item in finding.suggestedActions]),
|
||||
'source': 'AI Security Reviewer',
|
||||
'status': source_status,
|
||||
})
|
||||
return findings
|
||||
|
||||
|
||||
def _coverage(session: Session, summary: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], list[str], list[str], dict[str, Any]]:
|
||||
breakdown = summary.get('scoreBreakdown') if isinstance(summary.get('scoreBreakdown'), dict) else {}
|
||||
sources = breakdown.get('coverageSources') if isinstance(breakdown.get('coverageSources'), dict) else {}
|
||||
if not sources:
|
||||
sources = {
|
||||
'inventory': {'status': 'connected'},
|
||||
'portainer': {'status': _connector_status(session, 'portainer')},
|
||||
'unraid': {'status': _connector_status(session, 'unraid')},
|
||||
'npm': {'status': _connector_status(session, 'nginx_proxy_manager')},
|
||||
'authentik': {'status': _connector_status(session, 'authentik')},
|
||||
'secret_storage': {'status': 'connected'},
|
||||
'policies': {'status': 'connected'},
|
||||
'audit': {'status': 'partial'},
|
||||
'media_pipeline': {'status': 'partial'},
|
||||
}
|
||||
normalized = {
|
||||
key: {'label': SOURCE_LABELS.get(key, key), 'status': str(value.get('status', 'unknown')), 'effectOnConfidence': value.get('effectOnConfidence', 'medium')}
|
||||
for key, value in sources.items()
|
||||
if isinstance(value, dict)
|
||||
}
|
||||
connected = [key for key, value in normalized.items() if value['status'] == 'connected']
|
||||
missing = [key for key, value in normalized.items() if value['status'] in {'missing_config', 'unavailable', 'unsupported', 'error', 'unknown'}]
|
||||
completeness_weight = sum(_coverage_weight(value['status']) for value in normalized.values())
|
||||
completeness = {'percentage': round((completeness_weight / max(len(normalized), 1)) * 100), 'connectedSources': connected, 'missingSources': missing}
|
||||
return normalized, connected, missing, completeness
|
||||
|
||||
|
||||
def _connector_status(session: Session, connector_id: str) -> str:
|
||||
connector = session.get(Connector, connector_id)
|
||||
return str(connector.status if connector else 'unknown')
|
||||
|
||||
|
||||
def _coverage_weight(status: str) -> float:
|
||||
if status == 'connected':
|
||||
return 1.0
|
||||
if status == 'partial':
|
||||
return 0.5
|
||||
return 0.0
|
||||
|
||||
|
||||
def _counts(findings: list[dict[str, Any]]) -> Counter:
|
||||
return Counter(finding['severity'] for finding in findings if finding['status'] == 'open')
|
||||
|
||||
|
||||
def _has_real_high_or_critical(findings: list[dict[str, Any]]) -> bool:
|
||||
proven_high_risk_categories = {'auth', 'auth_coverage', 'exposure', 'policies', 'secrets', 'unraid'}
|
||||
return any(
|
||||
finding['status'] == 'open'
|
||||
and finding['severity'] in {'high', 'critical'}
|
||||
and finding['category'] in proven_high_risk_categories
|
||||
for finding in findings
|
||||
)
|
||||
|
||||
|
||||
def _status(score: int, counts: Counter, missing: list[str], findings: list[dict[str, Any]]) -> str:
|
||||
if counts['critical'] > 0:
|
||||
return 'critical'
|
||||
if missing:
|
||||
return 'warning' if counts['high'] > 0 and _has_real_high_or_critical(findings) else 'partial'
|
||||
if score < 85 or counts['medium'] or counts['high']:
|
||||
return 'warning'
|
||||
return 'healthy'
|
||||
|
||||
|
||||
def _section_status(score: int, counts: Counter, missing_only: bool) -> str:
|
||||
if counts['critical'] > 0:
|
||||
return 'critical'
|
||||
if missing_only and counts['high'] == 0:
|
||||
return 'partial'
|
||||
if score < 85 or counts['medium'] or counts['high']:
|
||||
return 'warning'
|
||||
return 'healthy'
|
||||
|
||||
|
||||
def _section_for_category(category: str) -> str:
|
||||
for section_id, _, _, categories in SECTION_DEFINITIONS:
|
||||
if category in categories:
|
||||
return section_id
|
||||
return 'setup-coverage'
|
||||
|
||||
|
||||
def _section_summary(section_id: str, section_findings: list[dict[str, Any]]) -> str:
|
||||
if not section_findings:
|
||||
return 'Geen open rule-based findings voor deze sectie.'
|
||||
top = section_findings[0]
|
||||
return str(top['title'])
|
||||
|
||||
|
||||
def _sections(findings: list[dict[str, Any]], recommendations: list[dict[str, Any]], coverage: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for section_id, title, href, categories in SECTION_DEFINITIONS:
|
||||
section_findings = [
|
||||
finding
|
||||
for finding in findings
|
||||
if finding['status'] == 'open' and _section_for_category(finding['category']) == section_id
|
||||
]
|
||||
section_recommendations = [item for item in recommendations if item['sectionId'] == section_id and item['status'] == 'open']
|
||||
counts = _counts(section_findings)
|
||||
score = max(0, 100 - sum(SEVERITY_PENALTY.get(finding['severity'], 0) for finding in section_findings))
|
||||
related_sources = _related_sources(section_id)
|
||||
source_weight = sum(_coverage_weight(str(coverage.get(source, {}).get('status', 'unknown'))) for source in related_sources)
|
||||
missing_only = bool(related_sources) and source_weight < len(related_sources) and not any(finding['severity'] in {'high', 'critical'} for finding in section_findings)
|
||||
completeness = round((source_weight / max(len(related_sources), 1)) * 100) if related_sources else 100
|
||||
result.append({
|
||||
'id': section_id,
|
||||
'title': title,
|
||||
'status': _section_status(score, counts, missing_only),
|
||||
'score': score,
|
||||
'summary': _section_summary(section_id, section_findings),
|
||||
'findingsCount': len(section_findings),
|
||||
'recommendationsCount': len(section_recommendations),
|
||||
'dataCompleteness': completeness,
|
||||
'href': href,
|
||||
'categories': sorted(categories),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def _related_sources(section_id: str) -> list[str]:
|
||||
return {
|
||||
'setup-coverage': ['inventory', 'portainer', 'secret_storage', 'policies'],
|
||||
'external-exposure': ['npm'],
|
||||
'authentication-coverage': ['authentik', 'npm'],
|
||||
'secret-hygiene': ['secret_storage'],
|
||||
'policy-posture': ['policies'],
|
||||
'inventory-app-review': ['inventory', 'portainer'],
|
||||
'media-pipeline': ['media_pipeline'],
|
||||
'unraid-host-safety': ['unraid', 'portainer'],
|
||||
'audit-activity': ['audit'],
|
||||
}.get(section_id, [])
|
||||
|
||||
|
||||
def _base_recommendations(session: Session, findings: list[dict[str, Any]], coverage: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
recommendations: list[dict[str, Any]] = []
|
||||
for source, detail in coverage.items():
|
||||
if detail['status'] not in {'missing_config', 'unavailable', 'unsupported', 'error', 'unknown'}:
|
||||
continue
|
||||
if source in {'npm', 'authentik', 'unraid'}:
|
||||
recommendations.append(_recommendation(
|
||||
rec_id=f'safety-rec-configure-{source}',
|
||||
title=f'Configureer {SOURCE_LABELS.get(source, source)} connector',
|
||||
description='Vul de connectorconfiguratie en secret in en draai daarna een read-only test.',
|
||||
severity='medium',
|
||||
confidence='high',
|
||||
source='Setup & Coverage',
|
||||
affected=[source],
|
||||
action_type='configure_connector',
|
||||
href='/setup',
|
||||
section_id='setup-coverage',
|
||||
))
|
||||
for finding in findings:
|
||||
if finding['severity'] not in {'critical', 'high', 'medium'}:
|
||||
continue
|
||||
section_id = _section_for_category(finding['category'])
|
||||
action = finding['suggestedActions'][0] if finding['suggestedActions'] else {}
|
||||
recommendations.append(_recommendation(
|
||||
rec_id=f'safety-rec-{finding["id"]}',
|
||||
title=f'Review: {finding["title"]}',
|
||||
description=finding['description'],
|
||||
severity=finding['severity'],
|
||||
confidence='high',
|
||||
source=section_id,
|
||||
affected=finding['affectedResources'],
|
||||
action_type=action.get('actionType') or 'open_page',
|
||||
href=action.get('href') or '/safety',
|
||||
section_id=section_id,
|
||||
source_finding_id=finding['id'],
|
||||
policy_decision=action.get('policyDecision'),
|
||||
))
|
||||
return list({item['id']: item for item in recommendations}.values())
|
||||
|
||||
|
||||
def _recommendation(
|
||||
*,
|
||||
rec_id: str,
|
||||
title: str,
|
||||
description: str,
|
||||
severity: str,
|
||||
confidence: str,
|
||||
source: str,
|
||||
affected: list[str],
|
||||
action_type: str,
|
||||
href: str,
|
||||
section_id: str,
|
||||
source_finding_id: str | None = None,
|
||||
policy_decision: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
'id': rec_id,
|
||||
'title': _redact(title),
|
||||
'description': _redact(description),
|
||||
'severity': severity,
|
||||
'confidence': confidence,
|
||||
'source': source,
|
||||
'affectedResources': _redact(affected),
|
||||
'actionType': _action_type(action_type),
|
||||
'href': href,
|
||||
'policyDecision': policy_decision,
|
||||
'executable': False,
|
||||
'canCreateRemediationPlan': bool(source_finding_id),
|
||||
'sourceFindingId': source_finding_id,
|
||||
'sectionId': section_id,
|
||||
'status': 'open',
|
||||
}
|
||||
|
||||
|
||||
def _add_local_findings(session: Session, findings: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
result = list(findings)
|
||||
latest_run = session.exec(select(DiscoveryRun).order_by(DiscoveryRun.started_at.desc())).first()
|
||||
if not latest_run:
|
||||
result.append(_local_finding('safety-discovery-not-run', 'Container discovery is nog niet gedraaid', 'Discovery coverage ontbreekt; inventory review is daardoor partial.', 'medium', 'setup', ['discovery'], ['No discovery run found'], '/inventory'))
|
||||
apps_need_review = session.exec(select(App).where(App.needs_review == True)).all() # noqa: E712
|
||||
if apps_need_review:
|
||||
href = f'/apps/{apps_need_review[0].id}' if len(apps_need_review) == 1 else '/inventory'
|
||||
result.append(_local_finding('safety-apps-need-review', 'Apps wachten op review', f'{len(apps_need_review)} app(s) vragen operator review voordat bredere automation veilig is.', 'medium', 'inventory', [app.id for app in apps_need_review[:6]], ['needs_review=true'], href))
|
||||
jobs = session.exec(select(JobRun).where(JobRun.status.in_(['queued', 'running']))).all()
|
||||
if len(jobs) >= 5:
|
||||
result.append(_local_finding('safety-many-open-jobs', 'Veel open jobs actief', f'{len(jobs)} jobs staan queued/running.', 'low', 'audit', ['jobs'], ['open jobs >= 5'], '/jobs'))
|
||||
return list({item['id']: item for item in result}.values())
|
||||
|
||||
|
||||
def _local_finding(finding_id: str, title: str, description: str, severity: str, category: str, affected: list[str], evidence: list[str], href: str) -> dict[str, Any]:
|
||||
return {
|
||||
'id': finding_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'severity': severity,
|
||||
'category': category,
|
||||
'affectedResources': affected,
|
||||
'evidence': [{'source': 'safety_center', 'label': title, 'value': evidence, 'redacted': False}],
|
||||
'suggestedActions': [{'label': 'Open details', 'actionType': 'open_page', 'href': href, 'policyDecision': 'allowed', 'executable': False}],
|
||||
'source': 'Safety Center',
|
||||
'status': 'open',
|
||||
}
|
||||
|
||||
|
||||
def _source_signature(session: Session) -> str:
|
||||
from .app_registry import list_active_apps
|
||||
from .inventory_discovery import latest_discovered_by_container
|
||||
|
||||
policies = [
|
||||
{
|
||||
'id': item.id,
|
||||
'action': item.action,
|
||||
'target': item.target,
|
||||
'voice': item.voice,
|
||||
'manual': item.manual,
|
||||
'risk': item.risk,
|
||||
}
|
||||
for item in session.exec(select(Policy)).all()
|
||||
]
|
||||
connectors = [
|
||||
{
|
||||
'id': item.id,
|
||||
'status': item.status,
|
||||
'lastTest': item.last_test,
|
||||
'secretRef': item.secret_ref,
|
||||
'capabilities': item.capabilities,
|
||||
}
|
||||
for item in session.exec(select(Connector)).all()
|
||||
]
|
||||
apps = [
|
||||
{
|
||||
'id': item.id,
|
||||
'status': item.status,
|
||||
'connectorStatus': item.connector_status,
|
||||
'containerStatus': item.container_status,
|
||||
'needsReview': item.needs_review,
|
||||
'discoveryStatus': item.discovery_status,
|
||||
'protected': item.protected,
|
||||
}
|
||||
for item in list_active_apps(session)
|
||||
]
|
||||
settings = [
|
||||
{
|
||||
'key': item.key,
|
||||
'updatedAt': item.updated_at.isoformat() if item.updated_at else None,
|
||||
'isSecret': item.is_secret,
|
||||
'status': item.value_json.get('status') if isinstance(item.value_json, dict) else None,
|
||||
}
|
||||
for item in session.exec(select(AppSetting)).all()
|
||||
if item.key != CACHE_SETTING_KEY
|
||||
and not item.key.startswith('ai_security:review_cache')
|
||||
and item.key not in {'media_pipeline:summary_cache', 'network_exposure:cache'}
|
||||
and (not item.is_secret or item.key.startswith('safety_center:'))
|
||||
]
|
||||
secret_rows = [
|
||||
{
|
||||
'id': item.id,
|
||||
'name': item.name,
|
||||
'updatedAt': item.updated_at.isoformat() if item.updated_at else None,
|
||||
'usedBy': item.used_by_json,
|
||||
}
|
||||
for item in session.exec(select(Secret)).all()
|
||||
]
|
||||
discovered = [
|
||||
{
|
||||
'containerId': item.container_id,
|
||||
'name': item.name,
|
||||
'image': item.image,
|
||||
'state': item.state,
|
||||
'status': item.status,
|
||||
'importStatus': item.import_status,
|
||||
'matchedAppId': item.matched_app_id,
|
||||
}
|
||||
for item in latest_discovered_by_container(session).values()
|
||||
]
|
||||
job_count = session.exec(select(func.count(JobRun.id))).one()
|
||||
latest_job_started = session.exec(select(func.max(JobRun.started_at))).one()
|
||||
non_safety_audit = AuditEvent.action_id.not_like('safety.%')
|
||||
audit_count = session.exec(select(func.count(AuditEvent.id)).where(non_safety_audit)).one()
|
||||
latest_audit_timestamp = session.exec(select(func.max(AuditEvent.timestamp)).where(non_safety_audit)).one()
|
||||
signature_payload = {
|
||||
'cacheSchemaVersion': CACHE_SCHEMA_VERSION,
|
||||
'policies': sorted(policies, key=lambda item: item['id']),
|
||||
'connectors': sorted(connectors, key=lambda item: item['id']),
|
||||
'apps': sorted(apps, key=lambda item: item['id']),
|
||||
'settings': sorted(settings, key=lambda item: item['key']),
|
||||
'secrets': sorted(secret_rows, key=lambda item: item['id']),
|
||||
'discovered': sorted(discovered, key=lambda item: item['containerId']),
|
||||
'counts': {
|
||||
'jobs': job_count,
|
||||
'latestJobStarted': latest_job_started.isoformat() if latest_job_started else None,
|
||||
'audit': audit_count,
|
||||
'latestAuditTimestamp': latest_audit_timestamp.isoformat() if latest_audit_timestamp else None,
|
||||
},
|
||||
}
|
||||
serialized = json.dumps(signature_payload, sort_keys=True, default=str, separators=(',', ':'))
|
||||
return hashlib.sha256(serialized.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def _payload(session: Session) -> dict[str, Any]:
|
||||
review = ai_security_reviewer.generate_review(session)
|
||||
security_summary = review.summary.model_dump()
|
||||
coverage, connected, missing, completeness = _coverage(session, security_summary)
|
||||
findings = _add_local_findings(session, _findings(session, review))
|
||||
counts = _counts(findings)
|
||||
score = max(0, min(100, 100 - sum(SEVERITY_PENALTY.get(finding['severity'], 0) for finding in findings if finding['status'] == 'open')))
|
||||
recommendations = _base_recommendations(session, findings, coverage)
|
||||
sections = _sections(findings, recommendations, coverage)
|
||||
status = _status(score, counts, missing, findings)
|
||||
summary = {
|
||||
'status': status,
|
||||
'score': score,
|
||||
'generatedAt': _now(),
|
||||
'dataCompleteness': completeness,
|
||||
'coverageStatus': coverage,
|
||||
'criticalCount': counts['critical'],
|
||||
'highCount': counts['high'],
|
||||
'mediumCount': counts['medium'],
|
||||
'warningCount': counts['low'] + counts['medium'] + counts['high'],
|
||||
'recommendationCount': len([item for item in recommendations if item['status'] == 'open']),
|
||||
'connectedSources': connected,
|
||||
'missingSources': missing,
|
||||
'sections': sections,
|
||||
}
|
||||
score_breakdown = {
|
||||
'baseScore': 100,
|
||||
'finalScore': score,
|
||||
'penalties': [
|
||||
{
|
||||
'source': finding['source'],
|
||||
'reason': finding['title'],
|
||||
'severity': finding['severity'],
|
||||
'points': SEVERITY_PENALTY.get(finding['severity'], 0),
|
||||
'findingId': finding['id'],
|
||||
}
|
||||
for finding in findings
|
||||
if finding['status'] == 'open' and SEVERITY_PENALTY.get(finding['severity'], 0) > 0
|
||||
],
|
||||
'bonuses': [],
|
||||
'missingCoverageImpact': 'Missing coverage lowers confidence and data completeness; it is not treated as critical without evidence.',
|
||||
'criticalEvidence': [finding['id'] for finding in findings if finding['status'] == 'open' and finding['severity'] == 'critical'],
|
||||
'explanation': f'Startscore 100 minus open finding penalties. Current status is {status}.',
|
||||
'missingSources': missing,
|
||||
'dataCompleteness': completeness,
|
||||
'coverageStatus': coverage,
|
||||
}
|
||||
return {'summary': _redact(summary), 'sections': _redact(sections), 'findings': _redact(findings), 'recommendations': _redact(recommendations), 'scoreBreakdown': _redact(score_breakdown)}
|
||||
|
||||
|
||||
def _coverage_sources(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
coverage = payload.get('summary', {}).get('coverageStatus', {})
|
||||
if not isinstance(coverage, dict):
|
||||
return []
|
||||
return [
|
||||
{
|
||||
'id': source,
|
||||
'label': detail.get('label') or SOURCE_LABELS.get(source, source),
|
||||
'status': detail.get('status', 'unknown'),
|
||||
'effectOnConfidence': detail.get('effectOnConfidence'),
|
||||
}
|
||||
for source, detail in sorted(coverage.items())
|
||||
if isinstance(detail, dict)
|
||||
]
|
||||
|
||||
|
||||
def _with_cache_metadata(payload: dict[str, Any], *, hit: bool, source: str, generated_at: str, expires_at: str, ttl_seconds: int, generation_duration_ms: int, stale_reason: str | None = None) -> dict[str, Any]:
|
||||
dashboard = dict(payload)
|
||||
dashboard['generatedAt'] = generated_at
|
||||
dashboard['coverageSources'] = _coverage_sources(payload)
|
||||
cache_meta: dict[str, Any] = {
|
||||
'hit': hit,
|
||||
'ttlSeconds': ttl_seconds,
|
||||
'expiresAt': expires_at,
|
||||
'source': source,
|
||||
'generationDurationMs': generation_duration_ms,
|
||||
}
|
||||
if stale_reason:
|
||||
cache_meta['staleReason'] = _redact(stale_reason)
|
||||
dashboard['cache'] = cache_meta
|
||||
return _redact(dashboard)
|
||||
|
||||
|
||||
def _persistent_cache(session: Session, signature: str) -> dict[str, Any] | None:
|
||||
row = session.get(AppSetting, CACHE_SETTING_KEY)
|
||||
if not row or not isinstance(row.value_json, dict):
|
||||
return None
|
||||
value = row.value_json
|
||||
if value.get('schemaVersion') != CACHE_SCHEMA_VERSION:
|
||||
return None
|
||||
if value.get('signature') != signature:
|
||||
return None
|
||||
payload = value.get('payload')
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
return {
|
||||
'key': CACHE_KEY,
|
||||
'signature': signature,
|
||||
'payload': payload,
|
||||
'generatedAt': value.get('generatedAt'),
|
||||
'expiresAt': value.get('expiresAt'),
|
||||
'generationDurationMs': value.get('generationDurationMs', 0),
|
||||
}
|
||||
|
||||
|
||||
def _store_persistent_cache(session: Session, cache: dict[str, Any]) -> None:
|
||||
row = session.get(AppSetting, CACHE_SETTING_KEY)
|
||||
if not row:
|
||||
row = AppSetting(key=CACHE_SETTING_KEY, is_secret=False)
|
||||
row.value_json = {
|
||||
'schemaVersion': CACHE_SCHEMA_VERSION,
|
||||
'signature': cache.get('signature'),
|
||||
'payload': _redact(cache.get('payload')),
|
||||
'generatedAt': cache.get('generatedAt'),
|
||||
'expiresAt': cache.get('expiresAt'),
|
||||
'generationDurationMs': cache.get('generationDurationMs', 0),
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
|
||||
def _schedule_background_refresh() -> bool:
|
||||
global _background_refresh_running
|
||||
with _background_refresh_lock:
|
||||
if _background_refresh_running:
|
||||
return False
|
||||
_background_refresh_running = True
|
||||
|
||||
def refresh_worker() -> None:
|
||||
global _background_refresh_running
|
||||
try:
|
||||
from ..db.session import engine
|
||||
|
||||
with Session(engine) as refresh_session:
|
||||
get_dashboard(refresh_session, force_refresh=True)
|
||||
except Exception: # noqa: BLE001 - stale safety data remains available on refresh failure.
|
||||
logger.exception('Background Safety dashboard refresh failed')
|
||||
finally:
|
||||
with _background_refresh_lock:
|
||||
_background_refresh_running = False
|
||||
|
||||
Thread(target=refresh_worker, name='appops-safety-refresh', daemon=True).start()
|
||||
return True
|
||||
|
||||
|
||||
def get_dashboard(session: Session, force_refresh: bool = False) -> dict[str, Any]:
|
||||
global _dashboard_cache, _skip_persistent_cache_once
|
||||
ttl_seconds = _ttl_seconds()
|
||||
now_dt = datetime.now(UTC)
|
||||
signature = _source_signature(session)
|
||||
if _dashboard_cache and not force_refresh and _dashboard_cache.get('signature') == signature:
|
||||
expires_at = _parse_utc(str(_dashboard_cache['expiresAt']))
|
||||
if now_dt < expires_at:
|
||||
return _with_cache_metadata(
|
||||
_dashboard_cache['payload'],
|
||||
hit=True,
|
||||
source='cache',
|
||||
generated_at=str(_dashboard_cache['generatedAt']),
|
||||
expires_at=str(_dashboard_cache['expiresAt']),
|
||||
ttl_seconds=ttl_seconds,
|
||||
generation_duration_ms=int(_dashboard_cache.get('generationDurationMs', 0)),
|
||||
)
|
||||
|
||||
skip_persistent = _skip_persistent_cache_once
|
||||
_skip_persistent_cache_once = False
|
||||
persisted = None if force_refresh or skip_persistent else _persistent_cache(session, signature)
|
||||
if persisted:
|
||||
expires_at = _parse_utc(str(persisted['expiresAt']))
|
||||
max_stale_seconds = _persistent_cache_max_stale_seconds()
|
||||
stale_until = expires_at.timestamp() + max_stale_seconds
|
||||
if now_dt < expires_at:
|
||||
_dashboard_cache = persisted
|
||||
return _with_cache_metadata(
|
||||
persisted['payload'],
|
||||
hit=True,
|
||||
source='cache',
|
||||
generated_at=str(persisted['generatedAt']),
|
||||
expires_at=str(persisted['expiresAt']),
|
||||
ttl_seconds=ttl_seconds,
|
||||
generation_duration_ms=int(persisted.get('generationDurationMs', 0)),
|
||||
)
|
||||
if max_stale_seconds > 0 and now_dt.timestamp() < stale_until:
|
||||
_dashboard_cache = persisted
|
||||
_schedule_background_refresh()
|
||||
return _with_cache_metadata(
|
||||
persisted['payload'],
|
||||
hit=True,
|
||||
source='stale-while-revalidate',
|
||||
generated_at=str(persisted['generatedAt']),
|
||||
expires_at=str(persisted['expiresAt']),
|
||||
ttl_seconds=ttl_seconds,
|
||||
generation_duration_ms=int(persisted.get('generationDurationMs', 0)),
|
||||
)
|
||||
|
||||
started = perf_counter()
|
||||
try:
|
||||
payload = _payload(session)
|
||||
except Exception as exc:
|
||||
if _dashboard_cache:
|
||||
return _with_cache_metadata(
|
||||
_dashboard_cache['payload'],
|
||||
hit=True,
|
||||
source='stale',
|
||||
generated_at=str(_dashboard_cache['generatedAt']),
|
||||
expires_at=str(_dashboard_cache['expiresAt']),
|
||||
ttl_seconds=ttl_seconds,
|
||||
generation_duration_ms=int(_dashboard_cache.get('generationDurationMs', 0)),
|
||||
stale_reason=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
duration_ms = int((perf_counter() - started) * 1000)
|
||||
generated_at = str(payload['summary']['generatedAt'])
|
||||
expires_at = (now_dt.replace(microsecond=0).timestamp() + ttl_seconds)
|
||||
expires_at_iso = datetime.fromtimestamp(expires_at, UTC).isoformat().replace('+00:00', 'Z')
|
||||
_dashboard_cache = {
|
||||
'key': CACHE_KEY,
|
||||
'signature': signature,
|
||||
'payload': payload,
|
||||
'generatedAt': generated_at,
|
||||
'expiresAt': expires_at_iso,
|
||||
'generationDurationMs': duration_ms,
|
||||
}
|
||||
_store_persistent_cache(session, _dashboard_cache)
|
||||
logger.info('Safety dashboard generated in %sms with score=%s status=%s', duration_ms, payload['summary'].get('score'), payload['summary'].get('status'))
|
||||
return _with_cache_metadata(
|
||||
payload,
|
||||
hit=False,
|
||||
source='fresh',
|
||||
generated_at=generated_at,
|
||||
expires_at=expires_at_iso,
|
||||
ttl_seconds=ttl_seconds,
|
||||
generation_duration_ms=duration_ms,
|
||||
)
|
||||
|
||||
|
||||
def get_summary(session: Session) -> dict[str, Any]:
|
||||
return get_dashboard(session)['summary']
|
||||
|
||||
|
||||
def get_sections(session: Session) -> list[dict[str, Any]]:
|
||||
return get_dashboard(session)['sections']
|
||||
|
||||
|
||||
def get_findings(session: Session) -> list[dict[str, Any]]:
|
||||
return get_dashboard(session)['findings']
|
||||
|
||||
|
||||
def get_recommendations(session: Session) -> list[dict[str, Any]]:
|
||||
return get_dashboard(session)['recommendations']
|
||||
|
||||
|
||||
def get_score_breakdown(session: Session) -> dict[str, Any]:
|
||||
return get_dashboard(session)['scoreBreakdown']
|
||||
|
||||
|
||||
def refresh(session: Session) -> dict[str, Any]:
|
||||
invalidate_cache()
|
||||
dashboard = get_dashboard(session, force_refresh=True)
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='safety_center',
|
||||
action_id='safety.refresh',
|
||||
target='safety-center',
|
||||
target_type='safety',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Safety Center read-only refresh.',
|
||||
result='success',
|
||||
redacted_metadata={'score': dashboard['summary']['score'], 'status': dashboard['summary']['status'], 'findings': len(dashboard['findings']), 'cache': dashboard['cache']['source']},
|
||||
)
|
||||
return dashboard
|
||||
|
||||
|
||||
def acknowledge_finding(session: Session, finding_id: str) -> dict[str, Any]:
|
||||
return _set_state(session, 'finding', finding_id, 'acknowledged')
|
||||
|
||||
|
||||
def dismiss_finding(session: Session, finding_id: str) -> dict[str, Any]:
|
||||
return _set_state(session, 'finding', finding_id, 'dismissed')
|
||||
|
||||
|
||||
def dismiss_recommendation(session: Session, recommendation_id: str) -> dict[str, Any]:
|
||||
return _set_state(session, 'recommendation', recommendation_id, 'dismissed')
|
||||
|
||||
|
||||
def create_remediation_plan(session: Session, recommendation_id: str) -> dict[str, Any]:
|
||||
recommendation = next((item for item in get_recommendations(session) if item['id'] == recommendation_id), None)
|
||||
if not recommendation:
|
||||
return {'success': False, 'status': 'not_found', 'recommendationId': recommendation_id, 'executable': False, 'href': '/ai/remediation-planner'}
|
||||
finding_id = recommendation.get('sourceFindingId')
|
||||
plan_id = None
|
||||
if finding_id:
|
||||
plan = ai_remediation_planner.generate_from_finding(session, str(finding_id))
|
||||
plan_id = plan.id
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='safety_center',
|
||||
action_id='safety.recommendation.create_remediation_plan',
|
||||
target=recommendation_id,
|
||||
target_type='safety_recommendation',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Safety Center linked a non-executing remediation plan.',
|
||||
result='success',
|
||||
redacted_metadata={'recommendationId': recommendation_id, 'planId': plan_id},
|
||||
)
|
||||
invalidate_cache()
|
||||
href = f'/ai/remediation-planner?plan={plan_id}' if plan_id else '/ai/remediation-planner'
|
||||
return {'success': True, 'status': 'plan_linked', 'recommendationId': recommendation_id, 'planId': plan_id, 'href': href, 'executable': False}
|
||||
@@ -0,0 +1,197 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import Secret
|
||||
|
||||
|
||||
SECRET_LIKE_PATTERN = re.compile(r"(secret|token|api[_-]?key|password|x-api-key)", re.IGNORECASE)
|
||||
REDACTED = "[REDACTED]"
|
||||
|
||||
|
||||
def _derive_fernet_key(value: str) -> bytes:
|
||||
value = value.strip()
|
||||
try:
|
||||
Fernet(value.encode("utf-8"))
|
||||
return value.encode("utf-8")
|
||||
except (ValueError, TypeError):
|
||||
digest = hashlib.sha256(value.encode("utf-8")).digest()
|
||||
return base64.urlsafe_b64encode(digest)
|
||||
|
||||
|
||||
def _fallback_material() -> str:
|
||||
return f"appops-development-secret-store:{settings.db_path}:{settings.data_dir}"
|
||||
|
||||
|
||||
def _active_key_material() -> str:
|
||||
if settings.secret_key:
|
||||
if settings.environment == "production" and len(settings.secret_key) < 32:
|
||||
raise RuntimeError("APPOPS_SECRET_KEY is too weak for production")
|
||||
return settings.secret_key
|
||||
return _fallback_material()
|
||||
|
||||
|
||||
def _fernet() -> Fernet:
|
||||
return Fernet(_derive_fernet_key(_active_key_material()))
|
||||
|
||||
|
||||
def secret_storage_status() -> dict[str, Any]:
|
||||
key_source = settings.secret_key_source if settings.secret_key else "missing"
|
||||
encrypted = bool(settings.secret_key) or settings.environment != "production"
|
||||
key_strong = bool(settings.secret_key and len(settings.secret_key) >= 32)
|
||||
production_ready = key_strong and key_source in {"env", "file"}
|
||||
warnings: list[str] = []
|
||||
if settings.environment == "production" and not settings.secret_key:
|
||||
warnings.append("APPOPS_SECRET_KEY is required for production encrypted secret storage.")
|
||||
elif settings.environment == "production" and not key_strong:
|
||||
warnings.append("APPOPS_SECRET_KEY must contain at least 32 characters in production.")
|
||||
if settings.secret_key_source == "file_unreadable":
|
||||
warnings.append("APPOPS_SECRET_KEY_FILE could not be read.")
|
||||
if settings.environment != "production" and not settings.secret_key:
|
||||
key_source = "development_fallback"
|
||||
warnings.append("Development fallback key is active; set APPOPS_SECRET_KEY before production.")
|
||||
return {
|
||||
"encryptedAtRest": encrypted,
|
||||
"enabled": encrypted,
|
||||
"keySource": key_source,
|
||||
"rotationEnabled": settings.secret_rotation_enabled,
|
||||
"productionReady": production_ready,
|
||||
"status": "ready" if production_ready or settings.environment != "production" else "warning",
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
def set_secret(
|
||||
session: Session,
|
||||
name: str,
|
||||
value: str,
|
||||
*,
|
||||
used_by: list[str] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
commit: bool = True,
|
||||
) -> Secret:
|
||||
clean_name = name.strip().upper()
|
||||
clean_value = value.strip()
|
||||
now = utc_now()
|
||||
encrypted = _fernet().encrypt(clean_value.encode("utf-8")).decode("utf-8")
|
||||
row = session.get(Secret, clean_name)
|
||||
if not row:
|
||||
row = Secret(id=clean_name, name=clean_name, encrypted_value=encrypted, created_at=now)
|
||||
row.name = clean_name
|
||||
row.encrypted_value = encrypted
|
||||
row.updated_at = now
|
||||
row.used_by_json = sorted(set(used_by or row.used_by_json or []))
|
||||
row.metadata_json = metadata or row.metadata_json or {}
|
||||
session.add(row)
|
||||
if commit:
|
||||
session.commit()
|
||||
return row
|
||||
|
||||
|
||||
def get_secret(session: Session, name: str, *, touch: bool = True) -> str | None:
|
||||
row = session.get(Secret, name.strip().upper())
|
||||
if not row:
|
||||
return None
|
||||
try:
|
||||
value = _fernet().decrypt(row.encrypted_value.encode("utf-8")).decode("utf-8")
|
||||
except InvalidToken:
|
||||
return None
|
||||
if touch:
|
||||
row.last_used_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return value
|
||||
|
||||
|
||||
def has_secret(session: Session, name: str) -> bool:
|
||||
row = session.get(Secret, name.strip().upper())
|
||||
return bool(row and row.encrypted_value)
|
||||
|
||||
|
||||
def delete_secret(session: Session, name: str) -> None:
|
||||
row = session.get(Secret, name.strip().upper())
|
||||
if row:
|
||||
session.delete(row)
|
||||
session.commit()
|
||||
|
||||
|
||||
def rotate_secret(session: Session, name: str, new_value: str, *, used_by: list[str] | None = None) -> Secret:
|
||||
row = session.get(Secret, name.strip().upper())
|
||||
return set_secret(session, name, new_value, used_by=used_by or (row.used_by_json if row else None))
|
||||
|
||||
|
||||
def secret_status(row: Secret) -> dict[str, Any]:
|
||||
updated_at = row.updated_at or row.created_at
|
||||
age_days = max(0, (utc_now() - updated_at).days) if updated_at else None
|
||||
return {
|
||||
"secretRef": row.id,
|
||||
"name": row.name,
|
||||
"hasSecret": bool(row.encrypted_value),
|
||||
"status": "configured" if row.encrypted_value else "missing",
|
||||
"usedBy": row.used_by_json or [],
|
||||
"lastUpdated": row.updated_at.isoformat() if row.updated_at else None,
|
||||
"lastUsed": row.last_used_at.isoformat() if row.last_used_at else None,
|
||||
"ageDays": age_days,
|
||||
"rotationRecommended": bool(age_days is not None and age_days >= 90),
|
||||
"metadata": {
|
||||
key: value
|
||||
for key, value in (row.metadata_json or {}).items()
|
||||
if not SECRET_LIKE_PATTERN.search(str(key))
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def empty_secret_status(name: str, *, used_by: list[str] | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"secretRef": name.strip().upper(),
|
||||
"name": name.strip().upper(),
|
||||
"hasSecret": False,
|
||||
"status": "missing",
|
||||
"usedBy": used_by or [],
|
||||
"lastUpdated": None,
|
||||
"lastUsed": None,
|
||||
"ageDays": None,
|
||||
"rotationRecommended": False,
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
|
||||
def list_secret_status(session: Session) -> list[dict[str, Any]]:
|
||||
rows = session.exec(select(Secret).order_by(Secret.name)).all()
|
||||
return [secret_status(row) for row in rows]
|
||||
|
||||
|
||||
def redact_secret_value(value: str | None) -> str | None:
|
||||
return REDACTED if value else value
|
||||
|
||||
|
||||
def _known_secret_values(session: Session | None) -> list[str]:
|
||||
if session is None:
|
||||
return []
|
||||
values: list[str] = []
|
||||
rows = session.exec(select(Secret)).all()
|
||||
for row in rows:
|
||||
value = get_secret(session, row.id)
|
||||
if value:
|
||||
values.append(value)
|
||||
return values
|
||||
|
||||
|
||||
def redact_text(text: str, *, session: Session | None = None, known_values: list[str] | None = None) -> str:
|
||||
redacted = redact_message(text)
|
||||
redacted = re.sub(r"(x-api-key\s*[:=]\s*)([^\s\"']+)", rf"\1{REDACTED}", redacted, flags=re.IGNORECASE)
|
||||
redacted = re.sub(r"(apiKey\s*[:=]\s*)([^\s\"']+)", rf"\1{REDACTED}", redacted, flags=re.IGNORECASE)
|
||||
redacted = re.sub(r"(token\s*[:=]\s*)([^\s\"']+)", rf"\1{REDACTED}", redacted, flags=re.IGNORECASE)
|
||||
for value in [*(known_values or []), *_known_secret_values(session)]:
|
||||
if value:
|
||||
redacted = redacted.replace(value, REDACTED)
|
||||
return redacted
|
||||
@@ -0,0 +1,646 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..connectors.unraid import UnraidConnector
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..core.network_security import validate_connector_url
|
||||
from ..db.models import AppContainerMapping, AppSetting, AuditEvent, Connector, DiscoveryRun, Secret
|
||||
from .audit_logger import log_event
|
||||
from .secret_store import empty_secret_status, get_secret, has_secret, secret_status, secret_storage_status, set_secret
|
||||
from . import safety_center_service
|
||||
|
||||
|
||||
_CONNECTOR_FIELDS: dict[str, dict[str, Any]] = {
|
||||
'portainer': {
|
||||
'required': ['PORTAINER_URL', 'PORTAINER_API_KEY', 'PORTAINER_ENDPOINT_ID'],
|
||||
'secret_env': 'PORTAINER_API_KEY',
|
||||
'field_env': {'url': 'PORTAINER_URL', 'endpointId': 'PORTAINER_ENDPOINT_ID', 'verifyTls': 'PORTAINER_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'endpointId', 'verifyTls'],
|
||||
},
|
||||
'unraid': {
|
||||
'required': ['UNRAID_URL', 'UNRAID_API_KEY'],
|
||||
'secret_env': 'UNRAID_API_KEY',
|
||||
'field_env': {'url': 'UNRAID_URL', 'verifyTls': 'UNRAID_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'sonarr': {
|
||||
'required': ['SONARR_URL', 'SONARR_API_KEY'],
|
||||
'secret_env': 'SONARR_API_KEY',
|
||||
'field_env': {'url': 'SONARR_URL'},
|
||||
'safe_fields': ['url'],
|
||||
},
|
||||
'radarr': {
|
||||
'required': ['RADARR_URL', 'RADARR_API_KEY'],
|
||||
'secret_env': 'RADARR_API_KEY',
|
||||
'field_env': {'url': 'RADARR_URL', 'verifyTls': 'RADARR_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'prowlarr': {
|
||||
'required': ['PROWLARR_URL', 'PROWLARR_API_KEY'],
|
||||
'secret_env': 'PROWLARR_API_KEY',
|
||||
'field_env': {'url': 'PROWLARR_URL', 'verifyTls': 'PROWLARR_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'deluge': {
|
||||
'required': ['DELUGE_URL', 'DELUGE_PASSWORD'],
|
||||
'secret_env': 'DELUGE_PASSWORD',
|
||||
'field_env': {'url': 'DELUGE_URL', 'verifyTls': 'DELUGE_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'plex': {
|
||||
'required': ['PLEX_URL', 'PLEX_TOKEN'],
|
||||
'secret_env': 'PLEX_TOKEN',
|
||||
'field_env': {'url': 'PLEX_URL', 'verifyTls': 'PLEX_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'tautulli': {
|
||||
'required': ['TAUTULLI_URL', 'TAUTULLI_API_KEY'],
|
||||
'secret_env': 'TAUTULLI_API_KEY',
|
||||
'field_env': {'url': 'TAUTULLI_URL', 'verifyTls': 'TAUTULLI_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'nginx_proxy_manager': {
|
||||
'required': ['NPM_URL', 'NPM_IDENTITY', 'NPM_SECRET'],
|
||||
'secret_env': 'NPM_SECRET',
|
||||
'field_env': {'url': 'NPM_URL', 'identity': 'NPM_IDENTITY', 'verifyTls': 'NPM_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'identity', 'verifyTls'],
|
||||
},
|
||||
'authentik': {
|
||||
'required': ['AUTHENTIK_URL', 'AUTHENTIK_TOKEN'],
|
||||
'secret_env': 'AUTHENTIK_TOKEN',
|
||||
'field_env': {
|
||||
'url': 'AUTHENTIK_URL',
|
||||
'verifyTls': 'AUTHENTIK_VERIFY_TLS',
|
||||
'outpostUrl': 'AUTHENTIK_OUTPOST_URL',
|
||||
'expectEmbeddedOutpost': 'AUTHENTIK_EXPECT_EMBEDDED_OUTPOST',
|
||||
},
|
||||
'safe_fields': ['url', 'verifyTls', 'outpostUrl', 'expectEmbeddedOutpost'],
|
||||
},
|
||||
'widefrog': {
|
||||
'required': ['WIDEFROG_URL'],
|
||||
'secret_env': 'WIDEFROG_API_KEY',
|
||||
'field_env': {'url': 'WIDEFROG_URL', 'verifyTls': 'WIDEFROG_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
'home_assistant': {
|
||||
'required': ['HOME_ASSISTANT_URL', 'HOME_ASSISTANT_TOKEN'],
|
||||
'secret_env': 'HOME_ASSISTANT_TOKEN',
|
||||
'field_env': {'url': 'HOME_ASSISTANT_URL', 'verifyTls': 'HOME_ASSISTANT_VERIFY_TLS'},
|
||||
'safe_fields': ['url', 'verifyTls'],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _setting_key(connector_id: str, suffix: str) -> str:
|
||||
return f'connector:{connector_id}:{suffix}'
|
||||
|
||||
|
||||
def _get_setting(session: Session, key: str) -> AppSetting | None:
|
||||
return session.get(AppSetting, key)
|
||||
|
||||
|
||||
def _set_setting(session: Session, key: str, value: dict[str, Any], is_secret: bool = False) -> AppSetting:
|
||||
row = _get_setting(session, key)
|
||||
if not row:
|
||||
row = AppSetting(key=key, is_secret=is_secret)
|
||||
row.value_json = value
|
||||
row.is_secret = is_secret
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
return row
|
||||
|
||||
|
||||
def _connector_schema(connector_id: str) -> dict[str, Any]:
|
||||
return _CONNECTOR_FIELDS.get(connector_id, {
|
||||
'required': [f'{connector_id.upper()}_URL'],
|
||||
'secret_env': f'{connector_id.upper()}_API_KEY',
|
||||
'field_env': {'url': f'{connector_id.upper()}_URL'},
|
||||
'safe_fields': ['url'],
|
||||
})
|
||||
|
||||
|
||||
def _safe_config_from_row(session: Session, connector_id: str) -> dict[str, Any]:
|
||||
row = _get_setting(session, _setting_key(connector_id, 'config'))
|
||||
config = dict(row.value_json) if row else {}
|
||||
schema = _connector_schema(connector_id)
|
||||
safe: dict[str, Any] = {}
|
||||
for field in schema['safe_fields']:
|
||||
value = config.get(field)
|
||||
if value is None:
|
||||
env_name = schema['field_env'].get(field)
|
||||
value = settings.apis.get(env_name) if env_name else None
|
||||
if value is not None:
|
||||
safe[field] = value
|
||||
if connector_id == 'portainer' and not str(safe.get('endpointId') or '').strip():
|
||||
safe['endpointId'] = '1'
|
||||
return safe
|
||||
|
||||
|
||||
def _has_secret(session: Session, connector_id: str) -> bool:
|
||||
schema = _connector_schema(connector_id)
|
||||
secret_row = _get_setting(session, _setting_key(connector_id, 'secret'))
|
||||
secret_env = schema.get('secret_env')
|
||||
return bool(secret_env and has_secret(session, secret_env)) or bool(secret_row and secret_row.value_json.get('value')) or bool(secret_env and settings.apis.get(secret_env))
|
||||
|
||||
|
||||
def _connector_secret_status(session: Session, connector_id: str) -> dict[str, Any]:
|
||||
schema = _connector_schema(connector_id)
|
||||
secret_env = schema.get('secret_env')
|
||||
if not secret_env:
|
||||
return empty_secret_status(f'{connector_id.upper()}_API_KEY', used_by=[connector_id])
|
||||
row = session.get(Secret, secret_env)
|
||||
if row:
|
||||
return secret_status(row)
|
||||
if settings.apis.get(secret_env):
|
||||
status = empty_secret_status(secret_env, used_by=[connector_id])
|
||||
status['hasSecret'] = True
|
||||
status['status'] = 'env_fallback'
|
||||
status['lastUpdated'] = None
|
||||
return status
|
||||
return empty_secret_status(secret_env, used_by=[connector_id])
|
||||
|
||||
|
||||
def _migrate_legacy_connector_secret(session: Session, connector_id: str) -> None:
|
||||
schema = _connector_schema(connector_id)
|
||||
secret_env = schema.get('secret_env')
|
||||
if not secret_env:
|
||||
return
|
||||
legacy_row = _get_setting(session, _setting_key(connector_id, 'secret'))
|
||||
legacy_value = legacy_row.value_json.get('value') if legacy_row else None
|
||||
if not isinstance(legacy_value, str) or not legacy_value.strip():
|
||||
return
|
||||
set_secret(
|
||||
session,
|
||||
secret_env,
|
||||
legacy_value,
|
||||
used_by=[connector_id],
|
||||
metadata={'connectorId': connector_id, 'migratedFrom': _setting_key(connector_id, 'secret')},
|
||||
commit=False,
|
||||
)
|
||||
legacy_row.value_json = {'secretRef': secret_env, 'hasSecret': True, 'migrated': True}
|
||||
legacy_row.updated_at = utc_now()
|
||||
session.add(legacy_row)
|
||||
log_event(
|
||||
session,
|
||||
source='system',
|
||||
identity='secret_store',
|
||||
action_id='secret_migrated',
|
||||
target=secret_env,
|
||||
target_type='secret',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Legacy plaintext connector secret migrated to encrypted secret store.',
|
||||
result='success',
|
||||
redacted_metadata={'connector': connector_id, 'secretRef': secret_env},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
def _apply_connector_config_to_runtime(session: Session, connector_id: str) -> None:
|
||||
_migrate_legacy_connector_secret(session, connector_id)
|
||||
schema = _connector_schema(connector_id)
|
||||
safe_config = _safe_config_from_row(session, connector_id)
|
||||
for field, env_name in schema['field_env'].items():
|
||||
if field in safe_config:
|
||||
settings.apis[env_name] = safe_config[field]
|
||||
if 'verifyTls' in safe_config and connector_id in {'portainer', 'unraid'}:
|
||||
settings.verify_tls = bool(safe_config['verifyTls'])
|
||||
secret_row = _get_setting(session, _setting_key(connector_id, 'secret'))
|
||||
secret_env = schema.get('secret_env')
|
||||
if secret_env:
|
||||
stored_secret = get_secret(session, secret_env, touch=False)
|
||||
if stored_secret:
|
||||
settings.apis[secret_env] = stored_secret
|
||||
elif secret_row and secret_row.value_json.get('value'):
|
||||
settings.apis[secret_env] = secret_row.value_json.get('value')
|
||||
|
||||
|
||||
def apply_stored_connector_settings(session: Session, connector_ids: set[str] | None = None) -> None:
|
||||
selected = connector_ids if connector_ids is not None else set(_CONNECTOR_FIELDS)
|
||||
for connector_id in _CONNECTOR_FIELDS:
|
||||
if connector_id not in selected:
|
||||
continue
|
||||
_apply_connector_config_to_runtime(session, connector_id)
|
||||
|
||||
|
||||
def serialize_connector_settings(session: Session, connector_id: str) -> dict[str, Any]:
|
||||
_apply_connector_config_to_runtime(session, connector_id)
|
||||
schema = _connector_schema(connector_id)
|
||||
missing = [name for name in schema['required'] if not settings.apis.get(name)]
|
||||
return {
|
||||
'connectorId': connector_id,
|
||||
'configured': not missing,
|
||||
'hasSecret': _has_secret(session, connector_id),
|
||||
'missing': missing,
|
||||
'safeConfig': _safe_config_from_row(session, connector_id),
|
||||
'secretRef': schema.get('secret_env'),
|
||||
'secretStatus': _connector_secret_status(session, connector_id),
|
||||
'secretStore': 'encrypted_local_db',
|
||||
'secretStorage': secret_storage_status(),
|
||||
}
|
||||
|
||||
|
||||
def list_connector_settings(session: Session) -> list[dict[str, Any]]:
|
||||
connector_ids = [connector.id for connector in session.exec(select(Connector).order_by(Connector.id)).all()]
|
||||
for connector_id in _CONNECTOR_FIELDS:
|
||||
if connector_id not in connector_ids:
|
||||
connector_ids.append(connector_id)
|
||||
return [serialize_connector_settings(session, connector_id) for connector_id in connector_ids]
|
||||
|
||||
|
||||
def _normalize_connector_field(field: str, value: Any) -> Any:
|
||||
if field == 'verifyTls':
|
||||
return bool(value)
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
normalized = value.strip()
|
||||
if field in {'url', 'outpostUrl'}:
|
||||
normalized = validate_connector_url(normalized)
|
||||
return normalized
|
||||
|
||||
|
||||
def save_connector_settings(session: Session, connector_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
schema = _connector_schema(connector_id)
|
||||
safe_payload = dict(_safe_config_from_row(session, connector_id))
|
||||
for field in schema['safe_fields']:
|
||||
if field in payload and payload[field] is not None:
|
||||
safe_payload[field] = _normalize_connector_field(field, payload[field])
|
||||
if connector_id == 'portainer' and not str(safe_payload.get('endpointId') or '').strip():
|
||||
safe_payload['endpointId'] = '1'
|
||||
_set_setting(session, _setting_key(connector_id, 'config'), safe_payload, is_secret=False)
|
||||
|
||||
secret_value = payload.get('apiKey') or payload.get('token') or payload.get('password') or payload.get('secret')
|
||||
if isinstance(secret_value, str) and secret_value.strip():
|
||||
secret_env = schema.get('secret_env')
|
||||
if secret_env:
|
||||
set_secret(
|
||||
session,
|
||||
secret_env,
|
||||
secret_value,
|
||||
used_by=[connector_id],
|
||||
metadata={'connectorId': connector_id},
|
||||
commit=False,
|
||||
)
|
||||
_set_setting(session, _setting_key(connector_id, 'secret'), {'secretRef': secret_env, 'hasSecret': True}, is_secret=True)
|
||||
|
||||
session.commit()
|
||||
_apply_connector_config_to_runtime(session, connector_id)
|
||||
|
||||
connector = session.get(Connector, connector_id)
|
||||
if connector:
|
||||
connector.status = 'disconnected' if serialize_connector_settings(session, connector_id)['configured'] else 'missing_config'
|
||||
session.add(connector)
|
||||
session.commit()
|
||||
safety_center_service.invalidate_cache()
|
||||
return serialize_connector_settings(session, connector_id)
|
||||
|
||||
|
||||
def _step(step_id: str, label: str, status: str, required: bool) -> dict[str, Any]:
|
||||
return {'id': step_id, 'label': label, 'status': status, 'required': required}
|
||||
|
||||
|
||||
def _successful_portainer_test_exists(session: Session) -> bool:
|
||||
event = session.exec(
|
||||
select(AuditEvent)
|
||||
.where(AuditEvent.action_id == 'setup.test.portainer')
|
||||
.where(AuditEvent.result == 'success')
|
||||
.order_by(AuditEvent.timestamp.desc())
|
||||
).first()
|
||||
return bool(event)
|
||||
|
||||
|
||||
def _setup_flag(session: Session) -> tuple[bool, str | None]:
|
||||
first_run = _get_setting(session, 'setup:first_run_completed')
|
||||
completed_at = _get_setting(session, 'setup:completed_at')
|
||||
return bool(first_run and first_run.value_json.get('value')), completed_at.value_json.get('value') if completed_at else None
|
||||
|
||||
|
||||
def _set_setup_completed(session: Session, *, completed_at: str | None = None) -> str:
|
||||
now = completed_at or utc_now().isoformat()
|
||||
_set_setting(session, 'setup:first_run_completed', {'value': True})
|
||||
_set_setting(session, 'setup:completed_at', {'value': now})
|
||||
session.commit()
|
||||
safety_center_service.invalidate_cache()
|
||||
return now
|
||||
|
||||
|
||||
def _required_blockers(
|
||||
*,
|
||||
portainer_ready: bool,
|
||||
discovery_complete: bool,
|
||||
managed_apps: int,
|
||||
review_blockers: int,
|
||||
secret_storage_ready: bool,
|
||||
) -> list[str]:
|
||||
blockers: list[str] = []
|
||||
if not portainer_ready:
|
||||
blockers.append('portainer_not_connected')
|
||||
if not discovery_complete:
|
||||
blockers.append('discovery_not_completed')
|
||||
if managed_apps <= 0:
|
||||
blockers.append('no_managed_apps')
|
||||
if review_blockers > 0:
|
||||
blockers.append('apps_need_review')
|
||||
if not secret_storage_ready:
|
||||
blockers.append('secret_storage_not_ready')
|
||||
return blockers
|
||||
|
||||
|
||||
def get_setup_status(session: Session) -> dict[str, Any]:
|
||||
from .inventory_discovery import _latest_discovered_items
|
||||
from .app_registry import list_active_apps
|
||||
|
||||
first_run_completed, setup_completed_at = _setup_flag(session)
|
||||
portainer = serialize_connector_settings(session, 'portainer')
|
||||
unraid = serialize_connector_settings(session, 'unraid')
|
||||
portainer_row = session.get(Connector, 'portainer')
|
||||
unraid_row = session.get(Connector, 'unraid')
|
||||
portainer_connected = portainer['configured'] and portainer_row is not None and portainer_row.status == 'connected'
|
||||
portainer_tested = _successful_portainer_test_exists(session)
|
||||
unraid_ready = unraid['configured'] and (unraid_row is None or unraid_row.status == 'connected')
|
||||
latest_run = session.exec(select(DiscoveryRun).order_by(DiscoveryRun.started_at.desc())).first()
|
||||
# Setup is an operator-facing current-state summary. Discovery run history
|
||||
# remains available under Inventory, but recreated Docker IDs must not make
|
||||
# the setup counters grow forever.
|
||||
discovered = _latest_discovered_items(session)
|
||||
mappings = session.exec(select(AppContainerMapping)).all()
|
||||
apps = list_active_apps(session)
|
||||
connectors = list_connector_settings(session)
|
||||
mapped_container_ids = {mapping.container_id for mapping in mappings}
|
||||
apps_need_review = len([app for app in apps if app.needs_review]) + len([
|
||||
item
|
||||
for item in discovered
|
||||
if item.import_status in {'discovered', 'unmapped'} or (item.import_status == 'changed' and item.container_id not in mapped_container_ids)
|
||||
])
|
||||
optional_connectors = [item for item in connectors if item['connectorId'] != 'portainer']
|
||||
missing_optional_sources = [item['connectorId'] for item in optional_connectors if item['missing']]
|
||||
missing_config = len([item for item in connectors if item['missing']])
|
||||
mapped_apps = len(mappings)
|
||||
managed_apps = len(apps)
|
||||
review_blockers = 0 if first_run_completed else apps_need_review
|
||||
discovery_status = 'pending'
|
||||
if latest_run:
|
||||
discovery_status = 'complete' if latest_run.status == 'success' else 'failed' if latest_run.status == 'failed' else 'warning'
|
||||
discovery_complete = bool(latest_run and latest_run.status == 'success') or bool(discovered) or mapped_apps > 0
|
||||
portainer_ready = portainer_connected or portainer_tested or discovery_complete
|
||||
|
||||
review_status = 'pending' if not latest_run else ('complete' if review_blockers == 0 else 'warning')
|
||||
connectors_status = 'warning' if missing_optional_sources else 'complete'
|
||||
home_assistant_configured = 'home_assistant' not in missing_optional_sources
|
||||
voice_status = 'complete' if settings.voice_token and home_assistant_configured else 'warning'
|
||||
storage_status = secret_storage_status()
|
||||
secret_storage_ready = storage_status['status'] == 'ready'
|
||||
required_blockers = _required_blockers(
|
||||
portainer_ready=portainer_ready,
|
||||
discovery_complete=discovery_complete,
|
||||
managed_apps=mapped_apps,
|
||||
review_blockers=review_blockers,
|
||||
secret_storage_ready=secret_storage_ready,
|
||||
)
|
||||
required_steps_complete = not required_blockers
|
||||
setup_complete = required_steps_complete and first_run_completed
|
||||
coverage_complete = not missing_optional_sources
|
||||
live_ready = setup_complete and settings.port == 1216
|
||||
warnings: list[str] = []
|
||||
if missing_optional_sources:
|
||||
warnings.append('Optional connector coverage is partial.')
|
||||
if settings.port != 1216:
|
||||
warnings.append('Runtime is not on canonical production port 1216.')
|
||||
next_actions = [
|
||||
{'id': blocker, 'required': True}
|
||||
for blocker in required_blockers
|
||||
]
|
||||
if required_steps_complete and not first_run_completed:
|
||||
next_actions.append({'id': 'reconcile_setup_state', 'required': True})
|
||||
for connector_id in missing_optional_sources:
|
||||
next_actions.append({'id': f'configure_optional_{connector_id}', 'required': False})
|
||||
|
||||
steps = [
|
||||
_step('portainer', 'Portainer verbinden', 'complete' if portainer_ready else 'pending', True),
|
||||
_step('unraid', 'Unraid API verbinden', 'complete' if unraid_ready else 'warning', False),
|
||||
_step('discovery', 'Container discovery draaien', 'complete' if discovery_complete else discovery_status, True),
|
||||
_step('review', 'Reviewbaseline vastgelegd' if first_run_completed else 'Apps reviewen', review_status, True),
|
||||
_step('connectors', 'Connectoren binden', connectors_status, False),
|
||||
_step('voice', 'Voice/Home Assistant instellen', voice_status, False),
|
||||
]
|
||||
return {
|
||||
'firstRunCompleted': first_run_completed,
|
||||
'setupComplete': setup_complete,
|
||||
'liveReady': live_ready,
|
||||
'coverageComplete': coverage_complete,
|
||||
'securityCoveragePartial': not coverage_complete,
|
||||
'setupCompletedAt': setup_completed_at,
|
||||
'requiredStepsComplete': required_steps_complete,
|
||||
'optionalStepsComplete': coverage_complete,
|
||||
'reviewBlockers': review_blockers,
|
||||
'managedApps': managed_apps,
|
||||
'needsReview': apps_need_review,
|
||||
'missingOptionalSources': missing_optional_sources,
|
||||
'steps': steps,
|
||||
'warnings': warnings,
|
||||
'nextActions': next_actions,
|
||||
'blockers': required_blockers,
|
||||
'summary': {
|
||||
'containersDiscovered': len({item.container_id for item in discovered}),
|
||||
'appsManaged': managed_apps,
|
||||
'appsNeedReview': apps_need_review,
|
||||
'connectorsMissingConfig': missing_config,
|
||||
'missingOptionalSources': missing_optional_sources,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def complete_setup(session: Session) -> dict[str, Any]:
|
||||
from .inventory_discovery import sync_app_connector_statuses
|
||||
|
||||
sync_app_connector_statuses(session)
|
||||
_set_setup_completed(session)
|
||||
return {'success': True, 'message': 'setup_completed', 'status': get_setup_status(session)}
|
||||
|
||||
|
||||
def reconcile_setup(session: Session) -> dict[str, Any]:
|
||||
from .inventory_discovery import sync_app_connector_statuses
|
||||
|
||||
sync_app_connector_statuses(session)
|
||||
before = get_setup_status(session)
|
||||
blockers = list(before.get('blockers') or [])
|
||||
if blockers:
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='setup',
|
||||
action_id='setup.reconcile',
|
||||
target='setup',
|
||||
target_type='setup',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Setup reconciliation blocked by required criteria.',
|
||||
result='blocked',
|
||||
before_state={'firstRunCompleted': before['firstRunCompleted'], 'blockers': blockers},
|
||||
after_state={'firstRunCompleted': before['firstRunCompleted']},
|
||||
redacted_metadata={'blockers': blockers},
|
||||
)
|
||||
after = get_setup_status(session)
|
||||
return {'success': False, 'message': 'setup_reconcile_blocked', 'blockers': blockers, 'before': before, 'after': after}
|
||||
|
||||
if not before['firstRunCompleted']:
|
||||
_set_setup_completed(session)
|
||||
after = get_setup_status(session)
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='setup',
|
||||
action_id='setup.reconcile',
|
||||
target='setup',
|
||||
target_type='setup',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason='Setup state reconciled from existing live data.',
|
||||
result='success',
|
||||
before_state={'firstRunCompleted': before['firstRunCompleted'], 'setupComplete': before['setupComplete']},
|
||||
after_state={'firstRunCompleted': after['firstRunCompleted'], 'setupComplete': after['setupComplete']},
|
||||
redacted_metadata={'managedApps': after['managedApps'], 'coverageComplete': after['coverageComplete']},
|
||||
)
|
||||
return {'success': True, 'message': 'setup_reconciled', 'blockers': [], 'before': before, 'after': after}
|
||||
|
||||
|
||||
def mark_setup_complete(session: Session, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if payload.get('confirm') != 'MARK_SETUP_COMPLETE':
|
||||
from fastapi import HTTPException
|
||||
|
||||
raise HTTPException(status_code=400, detail='confirm_required')
|
||||
before = get_setup_status(session)
|
||||
_set_setup_completed(session)
|
||||
after = get_setup_status(session)
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='setup',
|
||||
action_id='setup.mark_complete',
|
||||
target='setup',
|
||||
target_type='setup',
|
||||
risk='medium',
|
||||
policy_decision='allowed',
|
||||
reason='Operator explicitly marked setup complete.',
|
||||
result='success',
|
||||
before_state={'firstRunCompleted': before['firstRunCompleted'], 'setupComplete': before['setupComplete']},
|
||||
after_state={'firstRunCompleted': after['firstRunCompleted'], 'setupComplete': after['setupComplete']},
|
||||
redacted_metadata={'coverageComplete': after['coverageComplete'], 'blockers': after.get('blockers', [])},
|
||||
)
|
||||
return {'success': True, 'message': 'setup_marked_complete', 'status': after}
|
||||
|
||||
|
||||
def mark_voice_configured(session: Session) -> dict[str, Any]:
|
||||
_set_setting(session, 'setup:voice_configured', {'value': True})
|
||||
session.commit()
|
||||
return {'success': True, 'message': 'voice_configured', 'status': get_setup_status(session)}
|
||||
|
||||
|
||||
def test_portainer_setup(session: Session) -> dict[str, Any]:
|
||||
config = serialize_connector_settings(session, 'portainer')
|
||||
if config['missing']:
|
||||
return {'success': False, 'status': 'missing_config', 'connected': False, 'requiredFields': config['missing'], 'message': 'missing_config'}
|
||||
connector = PortainerConnector()
|
||||
result = connector.test_connection()
|
||||
status = str(result.get('status', 'error'))
|
||||
if status != 'connected':
|
||||
message = redact_message(str(result.get('reason', status)))
|
||||
log_setup_test(session, 'portainer', status, message)
|
||||
response_status = 'unreachable' if status == 'error' else status
|
||||
required_fields = ['PORTAINER_ENDPOINT_ID'] if response_status == 'endpoint_error' else None
|
||||
return {
|
||||
'success': False,
|
||||
'status': response_status,
|
||||
'connected': False,
|
||||
'message': message,
|
||||
'requiredFields': required_fields,
|
||||
'safeConfig': config['safeConfig'],
|
||||
}
|
||||
containers = connector.list_containers()
|
||||
stacks = connector.list_stacks()
|
||||
if containers.get('status') != 'ok':
|
||||
message = redact_message(str(containers.get('reason', 'endpoint_error')))
|
||||
log_setup_test(session, 'portainer', 'endpoint_error', message)
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'endpoint_error',
|
||||
'connected': False,
|
||||
'message': message,
|
||||
'requiredFields': ['PORTAINER_ENDPOINT_ID'],
|
||||
'summary': {'containers': 0, 'stacks': 0, 'endpointReachable': False},
|
||||
'safeConfig': config['safeConfig'],
|
||||
}
|
||||
payload = {
|
||||
'success': True,
|
||||
'status': 'connected',
|
||||
'connected': True,
|
||||
'summary': {
|
||||
'containers': len(containers.get('containers', [])),
|
||||
'stacks': len(stacks.get('stacks', [])) if stacks.get('status') == 'ok' else 0,
|
||||
'endpointReachable': True,
|
||||
},
|
||||
'safeConfig': config['safeConfig'],
|
||||
}
|
||||
connector_row = session.get(Connector, 'portainer')
|
||||
if connector_row:
|
||||
connector_row.status = 'connected'
|
||||
session.add(connector_row)
|
||||
session.commit()
|
||||
log_setup_test(session, 'portainer', 'connected', 'connected')
|
||||
return payload
|
||||
|
||||
|
||||
def test_unraid_setup(session: Session) -> dict[str, Any]:
|
||||
config = serialize_connector_settings(session, 'unraid')
|
||||
if config['missing']:
|
||||
return {'success': False, 'status': 'missing_config', 'connected': False, 'requiredFields': config['missing'], 'message': 'missing_config'}
|
||||
connector = UnraidConnector()
|
||||
result = connector.test_connection()
|
||||
status = str(result.get('status', 'error'))
|
||||
if status != 'connected':
|
||||
message = redact_message(str(result.get('reason', status)))
|
||||
log_setup_test(session, 'unraid', status, message)
|
||||
return {'success': False, 'status': 'unreachable' if status == 'error' else status, 'connected': False, 'message': message}
|
||||
info = connector.get_server_info()
|
||||
array = connector.get_array_status()
|
||||
docker = connector.get_docker_service_status()
|
||||
payload = {
|
||||
'success': True,
|
||||
'status': 'connected',
|
||||
'connected': True,
|
||||
'summary': {
|
||||
'server': info.get('info', {}),
|
||||
'arrayStatus': array.get('arrayStatus'),
|
||||
'dockerService': docker.get('serviceStatus'),
|
||||
},
|
||||
'safeConfig': config['safeConfig'],
|
||||
}
|
||||
connector_row = session.get(Connector, 'unraid')
|
||||
if connector_row:
|
||||
connector_row.status = 'connected'
|
||||
session.add(connector_row)
|
||||
session.commit()
|
||||
log_setup_test(session, 'unraid', 'connected', 'connected')
|
||||
return payload
|
||||
|
||||
|
||||
def log_setup_test(session: Session, connector_id: str, result: str, reason: str) -> None:
|
||||
log_event(
|
||||
session,
|
||||
source='manual',
|
||||
identity='setup',
|
||||
action_id=f'setup.test.{connector_id}',
|
||||
target=connector_id,
|
||||
target_type='connector',
|
||||
risk='low',
|
||||
policy_decision='allowed',
|
||||
reason=redact_message(reason),
|
||||
result='success' if result == 'connected' else 'failed',
|
||||
redacted_metadata={'connector': connector_id, 'result': result},
|
||||
)
|
||||
@@ -0,0 +1,363 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..connectors.unraid import UnraidConnector
|
||||
from .approval_service import create_approval
|
||||
from .audit_logger import log_event
|
||||
from .job_runner import complete_job, create_job, set_job_progress, start_job
|
||||
from .policy_engine import evaluate_policy
|
||||
from .setup_service import apply_stored_connector_settings
|
||||
|
||||
|
||||
VM_ACTIONS: dict[str, dict[str, Any]] = {
|
||||
'start': {
|
||||
'action_id': 'unraid.vm.start',
|
||||
'expected_before': {'SHUTOFF', 'STOPPED', 'OFFLINE'},
|
||||
'expected_after': {'RUNNING', 'STARTED', 'ONLINE'},
|
||||
'risk': 'medium',
|
||||
'label': 'Start VM',
|
||||
},
|
||||
'resume': {
|
||||
'action_id': 'unraid.vm.resume',
|
||||
'expected_before': {'PAUSED', 'PMSUSPENDED'},
|
||||
'expected_after': {'RUNNING', 'STARTED', 'ONLINE'},
|
||||
'risk': 'medium',
|
||||
'label': 'Resume VM',
|
||||
},
|
||||
'pause': {
|
||||
'action_id': 'unraid.vm.pause',
|
||||
'expected_before': {'RUNNING', 'STARTED', 'ONLINE'},
|
||||
'expected_after': {'PAUSED'},
|
||||
'risk': 'high',
|
||||
'label': 'Pause VM',
|
||||
},
|
||||
'stop': {
|
||||
'action_id': 'unraid.vm.stop',
|
||||
'risk': 'critical',
|
||||
'label': 'Stop VM',
|
||||
'blocked_reason': (
|
||||
'The current Unraid stop mutation can force-stop a VM after its graceful-shutdown timeout. '
|
||||
'AppOps keeps this destructive fallback blocked.'
|
||||
),
|
||||
},
|
||||
'reboot': {
|
||||
'action_id': 'unraid.vm.reboot',
|
||||
'risk': 'critical',
|
||||
'label': 'Reboot VM',
|
||||
'blocked_reason': 'VM reboot remains blocked until guest-shutdown and recovery semantics can be verified safely.',
|
||||
},
|
||||
'force_stop': {
|
||||
'action_id': 'unraid.vm.force_stop',
|
||||
'risk': 'critical',
|
||||
'label': 'Force-stop VM',
|
||||
'blocked_reason': 'Force-stop is destructive and blocked by default.',
|
||||
},
|
||||
'reset': {
|
||||
'action_id': 'unraid.vm.reset',
|
||||
'risk': 'critical',
|
||||
'label': 'Reset VM',
|
||||
'blocked_reason': 'VM reset is destructive and blocked by default.',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _find_vm(connector: UnraidConnector, vm_id: str) -> tuple[dict[str, Any] | None, dict[str, Any]]:
|
||||
inventory = connector.get_vm_summary()
|
||||
if inventory.get('status') != 'ok':
|
||||
return None, inventory
|
||||
vm = next((item for item in inventory.get('vms', []) if str(item.get('id')) == vm_id), None)
|
||||
return vm, inventory
|
||||
|
||||
|
||||
def vm_capabilities(session: Session) -> dict[str, Any]:
|
||||
actions = []
|
||||
for action, definition in VM_ACTIONS.items():
|
||||
action_id = str(definition['action_id'])
|
||||
policy = evaluate_policy(session, action=action_id, target='*', source='manual')
|
||||
blocked_reason = definition.get('blocked_reason')
|
||||
actions.append({
|
||||
'id': action,
|
||||
'actionId': action_id,
|
||||
'label': definition['label'],
|
||||
'risk': definition['risk'],
|
||||
'policyDecision': 'blocked' if blocked_reason else policy['decision'],
|
||||
'reason': blocked_reason or policy['reason'],
|
||||
'executionSupported': action in {'start', 'resume', 'pause'},
|
||||
})
|
||||
return {
|
||||
'source': 'unraid_graphql',
|
||||
'writePermission': 'vm:update or vm:*',
|
||||
'actions': actions,
|
||||
}
|
||||
|
||||
|
||||
def request_vm_action(
|
||||
session: Session,
|
||||
*,
|
||||
vm_id: str,
|
||||
action: str,
|
||||
source: str = 'manual',
|
||||
confirmed: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_action = str(action).strip().lower()
|
||||
definition = VM_ACTIONS.get(normalized_action)
|
||||
if not definition:
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'blocked',
|
||||
'result': 'blocked',
|
||||
'decision': 'blocked',
|
||||
'message': 'Unknown VM action.',
|
||||
}
|
||||
|
||||
action_id = str(definition['action_id'])
|
||||
risk = str(definition['risk'])
|
||||
blocked_reason = definition.get('blocked_reason')
|
||||
if blocked_reason:
|
||||
audit = log_event(
|
||||
session,
|
||||
source=source,
|
||||
identity='operator',
|
||||
action_id=action_id,
|
||||
target=vm_id,
|
||||
target_type='virtual_machine',
|
||||
risk=risk,
|
||||
policy_decision='blocked',
|
||||
reason=str(blocked_reason),
|
||||
result='blocked',
|
||||
execution_plan={'steps': []},
|
||||
redacted_metadata={'vmId': vm_id, 'requestedAction': normalized_action},
|
||||
)
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'blocked',
|
||||
'result': 'blocked',
|
||||
'decision': 'blocked',
|
||||
'policy_decision': 'blocked',
|
||||
'message': blocked_reason,
|
||||
'auditId': audit.id,
|
||||
}
|
||||
|
||||
policy = evaluate_policy(session, action=action_id, target=vm_id, source=source)
|
||||
if policy['decision'] == 'blocked':
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'blocked',
|
||||
'result': 'blocked',
|
||||
'decision': 'blocked',
|
||||
'policy_decision': 'blocked',
|
||||
'message': policy['reason'],
|
||||
}
|
||||
if policy['decision'] == 'approval_required':
|
||||
approval = create_approval(
|
||||
session,
|
||||
action=action_id,
|
||||
target=vm_id,
|
||||
source=source,
|
||||
risk=risk,
|
||||
reason=policy['reason'],
|
||||
requester=source,
|
||||
)
|
||||
audit = log_event(
|
||||
session,
|
||||
source=source,
|
||||
identity='operator',
|
||||
action_id=action_id,
|
||||
target=vm_id,
|
||||
target_type='virtual_machine',
|
||||
risk=risk,
|
||||
policy_decision='approval_required',
|
||||
reason=policy['reason'],
|
||||
result='approval_requested',
|
||||
execution_plan={'steps': ['Review VM impact', 'Approve request', 'Execute and verify state']},
|
||||
redacted_metadata={'vmId': vm_id, 'approvalId': approval.id},
|
||||
)
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'approval_required',
|
||||
'result': 'approval_required',
|
||||
'decision': 'approval_required',
|
||||
'policy_decision': 'approval_required',
|
||||
'message': 'VM action requires approval before execution.',
|
||||
'approvalId': approval.id,
|
||||
'auditId': audit.id,
|
||||
}
|
||||
if not confirmed:
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'confirm_required',
|
||||
'result': 'confirm_required',
|
||||
'decision': 'allowed',
|
||||
'policy_decision': 'allowed',
|
||||
'message': 'Explicit confirmation is required before changing VM state.',
|
||||
}
|
||||
return execute_vm_action(
|
||||
session,
|
||||
vm_id=vm_id,
|
||||
action=normalized_action,
|
||||
source=source,
|
||||
policy_decision='allowed',
|
||||
reason=policy['reason'],
|
||||
)
|
||||
|
||||
|
||||
def execute_vm_action(
|
||||
session: Session,
|
||||
*,
|
||||
vm_id: str,
|
||||
action: str,
|
||||
source: str,
|
||||
policy_decision: str,
|
||||
reason: str,
|
||||
) -> dict[str, Any]:
|
||||
definition = VM_ACTIONS.get(action)
|
||||
if not definition or action not in {'start', 'resume', 'pause'}:
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'blocked',
|
||||
'result': 'blocked',
|
||||
'message': 'No safe VM executor is registered for this action.',
|
||||
}
|
||||
|
||||
apply_stored_connector_settings(session)
|
||||
connector = UnraidConnector()
|
||||
before_vm, inventory = _find_vm(connector, vm_id)
|
||||
if not before_vm:
|
||||
message = str(inventory.get('reason') or 'VM target was not found in live Unraid inventory.')
|
||||
audit = log_event(
|
||||
session,
|
||||
source=source,
|
||||
identity='operator',
|
||||
action_id=str(definition['action_id']),
|
||||
target=vm_id,
|
||||
target_type='virtual_machine',
|
||||
risk=str(definition['risk']),
|
||||
policy_decision=policy_decision,
|
||||
reason=reason,
|
||||
result='failed',
|
||||
before_state={'inventoryStatus': inventory.get('status')},
|
||||
redacted_metadata={'vmId': vm_id},
|
||||
)
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'failed',
|
||||
'result': 'failed',
|
||||
'message': message,
|
||||
'auditId': audit.id,
|
||||
}
|
||||
|
||||
before_state = str(before_vm.get('state', 'unknown')).upper()
|
||||
if before_state not in definition['expected_before']:
|
||||
message = f'{definition["label"]} is not valid while VM state is {before_state}.'
|
||||
audit = log_event(
|
||||
session,
|
||||
source=source,
|
||||
identity='operator',
|
||||
action_id=str(definition['action_id']),
|
||||
target=vm_id,
|
||||
target_type='virtual_machine',
|
||||
risk=str(definition['risk']),
|
||||
policy_decision=policy_decision,
|
||||
reason=reason,
|
||||
result='blocked',
|
||||
before_state={'name': before_vm.get('name'), 'state': before_state},
|
||||
redacted_metadata={'vmId': vm_id},
|
||||
)
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'invalid_state',
|
||||
'result': 'blocked',
|
||||
'message': message,
|
||||
'auditId': audit.id,
|
||||
}
|
||||
|
||||
steps = [
|
||||
'Load live VM inventory',
|
||||
f'Validate {before_state} state',
|
||||
f'Run {definition["label"]} through Unraid GraphQL',
|
||||
'Reload VM inventory and verify state',
|
||||
]
|
||||
job = create_job(
|
||||
session,
|
||||
action_id=str(definition['action_id']),
|
||||
target=vm_id,
|
||||
source=source,
|
||||
risk=str(definition['risk']),
|
||||
policy_decision=policy_decision,
|
||||
steps=steps,
|
||||
)
|
||||
start_job(session, job)
|
||||
set_job_progress(session, job, 35, f'Validated VM {before_vm.get("name", vm_id)} in state {before_state}.')
|
||||
|
||||
mutation = connector.mutate_vm(vm_id, action)
|
||||
set_job_progress(session, job, 75, str(mutation.get('message') or 'VM mutation returned.'))
|
||||
after_vm, after_inventory = _find_vm(connector, vm_id)
|
||||
after_state = str(after_vm.get('state', 'unknown')).upper() if after_vm else 'UNKNOWN'
|
||||
verified = after_state in definition['expected_after']
|
||||
success = bool(mutation.get('success')) and verified
|
||||
if success:
|
||||
message = f'{before_vm.get("name", "VM")} is now {after_state.lower()}.'
|
||||
elif mutation.get('success'):
|
||||
message = f'Unraid accepted the action, but post-check reported {after_state}.'
|
||||
else:
|
||||
message = str(mutation.get('message') or 'VM action failed.')
|
||||
|
||||
completed = complete_job(session, job, status='succeeded' if success else 'failed', result_summary=message)
|
||||
audit = log_event(
|
||||
session,
|
||||
source=source,
|
||||
identity='operator',
|
||||
action_id=str(definition['action_id']),
|
||||
target=vm_id,
|
||||
target_type='virtual_machine',
|
||||
risk=str(definition['risk']),
|
||||
policy_decision=policy_decision,
|
||||
reason=reason,
|
||||
result='success' if success else 'failed',
|
||||
execution_plan={'steps': steps, 'jobId': completed.id},
|
||||
before_state={'name': before_vm.get('name'), 'state': before_state},
|
||||
after_state={
|
||||
'name': after_vm.get('name') if after_vm else before_vm.get('name'),
|
||||
'state': after_state,
|
||||
'inventoryStatus': after_inventory.get('status'),
|
||||
'verified': verified,
|
||||
},
|
||||
redacted_metadata={'vmId': vm_id, 'executor': definition['action_id']},
|
||||
)
|
||||
return {
|
||||
'success': success,
|
||||
'status': 'success' if success else 'failed',
|
||||
'result': 'success' if success else 'failed',
|
||||
'decision': policy_decision,
|
||||
'policy_decision': policy_decision,
|
||||
'message': message,
|
||||
'jobId': completed.id,
|
||||
'auditId': audit.id,
|
||||
'before_state': {'name': before_vm.get('name'), 'state': before_state},
|
||||
'after_state': {'state': after_state, 'verified': verified},
|
||||
}
|
||||
|
||||
|
||||
def execute_approved_vm_action(session: Session, approval) -> dict[str, Any]:
|
||||
action = next(
|
||||
(name for name, definition in VM_ACTIONS.items() if definition['action_id'] == approval.action),
|
||||
None,
|
||||
)
|
||||
if action not in {'start', 'resume', 'pause'}:
|
||||
return {
|
||||
'success': False,
|
||||
'status': 'failed',
|
||||
'result': 'unsupported',
|
||||
'message': 'No safe approved VM executor is registered for this action.',
|
||||
}
|
||||
return execute_vm_action(
|
||||
session,
|
||||
vm_id=approval.target,
|
||||
action=action,
|
||||
source='approved_operator',
|
||||
policy_decision='approval_required',
|
||||
reason=approval.reason or f'Approved through {approval.id}.',
|
||||
)
|
||||
@@ -0,0 +1,582 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..core.config import settings
|
||||
from .policy_engine import evaluate_policy
|
||||
|
||||
|
||||
VOICE_COMMANDS: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "unraid.health",
|
||||
"label": "Check Unraid",
|
||||
"spokenPhraseNl": "controleer unraid",
|
||||
"spokenPhraseEn": "check unraid",
|
||||
"category": "Infrastructure",
|
||||
"endpoint": "/api/voice/actions/unraid/health",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "unraid.health_summary",
|
||||
"target": "unraid",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_unraid",
|
||||
"homeAssistantScriptName": "appops_check_unraid",
|
||||
"description": "Read-only Unraid health summary through the voice policy path.",
|
||||
"safetyNotes": ["Read-only", "No mover, disk or array mutation"],
|
||||
},
|
||||
{
|
||||
"id": "appops.setup.check",
|
||||
"label": "Check AppOps setup",
|
||||
"spokenPhraseNl": "controleer appops setup",
|
||||
"spokenPhraseEn": "check appops setup",
|
||||
"category": "Gateway",
|
||||
"endpoint": "/api/voice/actions/setup/status",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "setup.status",
|
||||
"target": "appops",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_setup",
|
||||
"homeAssistantScriptName": "appops_check_setup",
|
||||
"description": "Read-only setup readiness check for the AppOps Gateway.",
|
||||
"safetyNotes": ["Read-only", "Reports setup state only"],
|
||||
},
|
||||
{
|
||||
"id": "safety.summary",
|
||||
"label": "Check Safety",
|
||||
"spokenPhraseNl": "controleer veiligheid",
|
||||
"spokenPhraseEn": "check safety",
|
||||
"category": "Safety",
|
||||
"endpoint": "/api/voice/actions/safety/summary",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "safety.summary",
|
||||
"target": "safety",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_safety",
|
||||
"homeAssistantScriptName": "appops_check_safety",
|
||||
"description": "Read-only Safety Center summary.",
|
||||
"safetyNotes": ["Read-only", "No remediation execution"],
|
||||
},
|
||||
{
|
||||
"id": "media.pipeline.check",
|
||||
"label": "Check Media Pipeline",
|
||||
"spokenPhraseNl": "controleer media pipeline",
|
||||
"spokenPhraseEn": "check media pipeline",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/media-pipeline/check",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "media_pipeline.check",
|
||||
"target": "media-pipeline",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_media_pipeline",
|
||||
"homeAssistantScriptName": "appops_check_media_pipeline",
|
||||
"description": "Runs the read-only media pipeline diagnostic check path.",
|
||||
"safetyNotes": ["Read-only connector summaries", "No media add/delete actions"],
|
||||
},
|
||||
{
|
||||
"id": "apps.widefrog.health",
|
||||
"label": "Check Widefrog",
|
||||
"spokenPhraseNl": "controleer widefrog",
|
||||
"spokenPhraseEn": "check widefrog",
|
||||
"category": "Apps",
|
||||
"endpoint": "/api/voice/actions/apps/widefrog/health",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "widefrog.health",
|
||||
"target": "widefrog",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_widefrog",
|
||||
"homeAssistantScriptName": "appops_check_widefrog",
|
||||
"description": "Read-only Widefrog connector health.",
|
||||
"safetyNotes": ["Read-only", "No retry or scan"],
|
||||
},
|
||||
{
|
||||
"id": "apps.deluge.status",
|
||||
"label": "Check Downloads",
|
||||
"spokenPhraseNl": "controleer downloads",
|
||||
"spokenPhraseEn": "check downloads",
|
||||
"category": "Downloads",
|
||||
"endpoint": "/api/voice/actions/apps/deluge/status",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "deluge.session_status",
|
||||
"target": "deluge",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_downloads",
|
||||
"homeAssistantScriptName": "appops_check_downloads",
|
||||
"description": "Read-only Deluge/download status summary.",
|
||||
"safetyNotes": ["Read-only", "No pause, resume or delete-with-data"],
|
||||
},
|
||||
{
|
||||
"id": "apps.plex.activity",
|
||||
"label": "Check Plex Activity",
|
||||
"spokenPhraseNl": "controleer plex activiteit",
|
||||
"spokenPhraseEn": "check plex activity",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/plex/activity",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "plex.active_sessions",
|
||||
"target": "plex",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_plex_activity",
|
||||
"homeAssistantScriptName": "appops_check_plex_activity",
|
||||
"description": "Read-only Plex active stream summary.",
|
||||
"safetyNotes": ["Read-only", "No library scan or session kill"],
|
||||
},
|
||||
{
|
||||
"id": "notifications.summary",
|
||||
"label": "Check Notifications",
|
||||
"spokenPhraseNl": "controleer notificaties",
|
||||
"spokenPhraseEn": "check notifications",
|
||||
"category": "Notifications",
|
||||
"endpoint": "/api/voice/actions/notifications/summary",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "notifications.summary",
|
||||
"target": "notifications",
|
||||
"risk": "low",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_notifications",
|
||||
"homeAssistantScriptName": "appops_check_notifications",
|
||||
"description": "Read-only notification rule/channel summary.",
|
||||
"safetyNotes": ["Read-only", "Does not send Home Assistant notifications"],
|
||||
},
|
||||
{
|
||||
"id": "unraid.storage",
|
||||
"label": "Check Storage",
|
||||
"spokenPhraseNl": "controleer opslag",
|
||||
"spokenPhraseEn": "check storage",
|
||||
"category": "Infrastructure",
|
||||
"endpoint": "/api/voice/actions/unraid/storage",
|
||||
"method": "POST", "payload": {}, "action": "unraid.health_summary", "target": "unraid",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_storage", "homeAssistantScriptName": "appops_check_storage",
|
||||
"description": "Returns live disk, cache and parity status without changing storage.",
|
||||
"safetyNotes": ["Read-only", "No mover, format or array action"],
|
||||
},
|
||||
{
|
||||
"id": "unraid.vms",
|
||||
"label": "Check Virtual Machines",
|
||||
"spokenPhraseNl": "controleer virtuele machines",
|
||||
"spokenPhraseEn": "check virtual machines",
|
||||
"category": "Infrastructure",
|
||||
"endpoint": "/api/voice/actions/unraid/vms",
|
||||
"method": "POST", "payload": {}, "action": "unraid.health_summary", "target": "unraid",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_vms", "homeAssistantScriptName": "appops_check_vms",
|
||||
"description": "Returns the VM inventory and power states visible to the Unraid API.",
|
||||
"safetyNotes": ["Read-only", "Does not start or stop a VM"],
|
||||
},
|
||||
{
|
||||
"id": "apps.sonarr.health",
|
||||
"label": "Check Sonarr",
|
||||
"spokenPhraseNl": "controleer sonarr",
|
||||
"spokenPhraseEn": "check sonarr",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/sonarr/health",
|
||||
"method": "POST", "payload": {}, "action": "sonarr.health", "target": "sonarr",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_sonarr", "homeAssistantScriptName": "appops_check_sonarr",
|
||||
"description": "Returns Sonarr health without modifying series.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "apps.sonarr.queue",
|
||||
"label": "Check Sonarr Queue",
|
||||
"spokenPhraseNl": "controleer sonarr wachtrij",
|
||||
"spokenPhraseEn": "check sonarr queue",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/sonarr/queue",
|
||||
"method": "POST", "payload": {}, "action": "sonarr.queue_status", "target": "sonarr",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_sonarr_queue", "homeAssistantScriptName": "appops_check_sonarr_queue",
|
||||
"description": "Returns the current Sonarr queue summary.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "apps.radarr.health",
|
||||
"label": "Check Radarr",
|
||||
"spokenPhraseNl": "controleer radarr",
|
||||
"spokenPhraseEn": "check radarr",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/radarr/health",
|
||||
"method": "POST", "payload": {}, "action": "radarr.health", "target": "radarr",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_radarr", "homeAssistantScriptName": "appops_check_radarr",
|
||||
"description": "Returns Radarr health without modifying movies.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "apps.radarr.queue",
|
||||
"label": "Check Radarr Queue",
|
||||
"spokenPhraseNl": "controleer radarr wachtrij",
|
||||
"spokenPhraseEn": "check radarr queue",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/radarr/queue",
|
||||
"method": "POST", "payload": {}, "action": "radarr.queue_status", "target": "radarr",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_radarr_queue", "homeAssistantScriptName": "appops_check_radarr_queue",
|
||||
"description": "Returns the current Radarr queue summary.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "apps.prowlarr.indexers",
|
||||
"label": "Check Prowlarr Indexers",
|
||||
"spokenPhraseNl": "controleer prowlarr indexers",
|
||||
"spokenPhraseEn": "check prowlarr indexers",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/prowlarr/indexers",
|
||||
"method": "POST", "payload": {}, "action": "prowlarr.indexer_health", "target": "prowlarr",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_prowlarr", "homeAssistantScriptName": "appops_check_prowlarr",
|
||||
"description": "Returns Prowlarr indexer health and failures.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "apps.deluge.stalled",
|
||||
"label": "Check Stalled Downloads",
|
||||
"spokenPhraseNl": "controleer vastgelopen downloads",
|
||||
"spokenPhraseEn": "check stalled downloads",
|
||||
"category": "Downloads",
|
||||
"endpoint": "/api/voice/actions/apps/deluge/stalled",
|
||||
"method": "POST", "payload": {}, "action": "deluge.stalled_downloads", "target": "deluge",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_stalled_downloads", "homeAssistantScriptName": "appops_check_stalled_downloads",
|
||||
"description": "Returns stalled Deluge downloads without changing them.", "safetyNotes": ["Read-only", "No delete-with-data"],
|
||||
},
|
||||
{
|
||||
"id": "apps.plex.recent",
|
||||
"label": "Check Recent Plex Media",
|
||||
"spokenPhraseNl": "wat is recent toegevoegd aan plex",
|
||||
"spokenPhraseEn": "check recent plex media",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/plex/recent",
|
||||
"method": "POST", "payload": {}, "action": "plex.recent_media", "target": "plex",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_plex_recent", "homeAssistantScriptName": "appops_check_plex_recent",
|
||||
"description": "Returns recently added Plex media.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "apps.tautulli.activity",
|
||||
"label": "Check Tautulli Activity",
|
||||
"spokenPhraseNl": "controleer tautulli activiteit",
|
||||
"spokenPhraseEn": "check tautulli activity",
|
||||
"category": "Media",
|
||||
"endpoint": "/api/voice/actions/apps/tautulli/activity",
|
||||
"method": "POST", "payload": {}, "action": "tautulli.activity", "target": "tautulli",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_tautulli", "homeAssistantScriptName": "appops_check_tautulli",
|
||||
"description": "Returns current Tautulli stream and user activity.", "safetyNotes": ["Read-only"],
|
||||
},
|
||||
{
|
||||
"id": "security.exposure",
|
||||
"label": "Check Public Exposure",
|
||||
"spokenPhraseNl": "controleer publieke blootstelling",
|
||||
"spokenPhraseEn": "check public exposure",
|
||||
"category": "Safety",
|
||||
"endpoint": "/api/voice/actions/security/exposure",
|
||||
"method": "POST", "payload": {}, "action": "npm.exposure_summary", "target": "nginx_proxy_manager",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_exposure", "homeAssistantScriptName": "appops_check_exposure",
|
||||
"description": "Returns Nginx Proxy Manager exposure and certificate posture.", "safetyNotes": ["Read-only", "No proxy mutation"],
|
||||
},
|
||||
{
|
||||
"id": "security.authentik",
|
||||
"label": "Check Authentik",
|
||||
"spokenPhraseNl": "controleer authentik",
|
||||
"spokenPhraseEn": "check authentik",
|
||||
"category": "Safety",
|
||||
"endpoint": "/api/voice/actions/security/authentik",
|
||||
"method": "POST", "payload": {}, "action": "authentik.health", "target": "authentik",
|
||||
"risk": "low", "enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_check_authentik", "homeAssistantScriptName": "appops_check_authentik",
|
||||
"description": "Returns Authentik health without changing identity configuration.", "safetyNotes": ["Read-only", "No identity mutation"],
|
||||
},
|
||||
{
|
||||
"id": "container.widefrog.restart",
|
||||
"label": "Restart Widefrog",
|
||||
"spokenPhraseNl": "herstart widefrog",
|
||||
"spokenPhraseEn": "restart widefrog",
|
||||
"category": "Apps",
|
||||
"endpoint": "/api/voice/actions/container/widefrog/restart",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "container.restart",
|
||||
"target": "widefrog",
|
||||
"risk": "medium",
|
||||
"enabledByDefault": True,
|
||||
"homeAssistantRestCommandName": "appops_restart_widefrog",
|
||||
"homeAssistantScriptName": "appops_restart_widefrog",
|
||||
"description": "Policy-aware Widefrog container restart.",
|
||||
"safetyNotes": ["Only exported while policy allows voice restart", "Audited", "No stack redeploy"],
|
||||
},
|
||||
{
|
||||
"id": "playbooks.fix-widefrog.run",
|
||||
"label": "Run Fix Widefrog Playbook",
|
||||
"spokenPhraseNl": "voer widefrog herstel uit",
|
||||
"spokenPhraseEn": "run fix widefrog playbook",
|
||||
"category": "Playbooks",
|
||||
"endpoint": "/api/voice/actions/playbooks/fix-widefrog/run",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "playbook.run",
|
||||
"target": "fix-widefrog",
|
||||
"risk": "medium",
|
||||
"enabledByDefault": False,
|
||||
"homeAssistantRestCommandName": "appops_run_fix_widefrog_playbook",
|
||||
"homeAssistantScriptName": "appops_run_fix_widefrog_playbook",
|
||||
"description": "Requests policy evaluation for the Widefrog recovery playbook.",
|
||||
"safetyNotes": ["Approval may be required", "No direct connector call before policy"],
|
||||
},
|
||||
{
|
||||
"id": "notifications.evaluate.safety",
|
||||
"label": "Evaluate Safety Notifications",
|
||||
"spokenPhraseNl": "evalueer veiligheidsnotificaties",
|
||||
"spokenPhraseEn": "evaluate safety notifications",
|
||||
"category": "Notifications",
|
||||
"endpoint": "/api/voice/actions/notifications/evaluate/safety",
|
||||
"method": "POST",
|
||||
"payload": {},
|
||||
"action": "notifications.evaluate_safety",
|
||||
"target": "notifications",
|
||||
"risk": "medium",
|
||||
"enabledByDefault": False,
|
||||
"homeAssistantRestCommandName": "appops_evaluate_safety_notifications",
|
||||
"homeAssistantScriptName": "appops_evaluate_safety_notifications",
|
||||
"description": "Policy-aware safety notification evaluation.",
|
||||
"safetyNotes": ["May create notification events", "Does not expose HOME_ASSISTANT_TOKEN"],
|
||||
},
|
||||
]
|
||||
|
||||
BLOCKED_VOICE_EXAMPLES = [
|
||||
{
|
||||
"id": "container.authentik.restart",
|
||||
"label": "Restart Authentik direct command",
|
||||
"action": "container.restart",
|
||||
"target": "authentik",
|
||||
"reason": "Critical identity service restart is not exported as direct voice command.",
|
||||
},
|
||||
{
|
||||
"id": "bulk-download-pause",
|
||||
"label": "Pause all downloads",
|
||||
"target": "deluge",
|
||||
"reason": "Bulk download actions require explicit operator control.",
|
||||
},
|
||||
{
|
||||
"id": "stack-redeploy",
|
||||
"label": "Stack redeploy",
|
||||
"target": "stack",
|
||||
"reason": "Redeploy changes runtime topology and is not exported to voice.",
|
||||
},
|
||||
{
|
||||
"id": "unraid-mover-start",
|
||||
"label": "Unraid mover",
|
||||
"target": "unraid",
|
||||
"reason": "High-impact storage maintenance is blocked for voice.",
|
||||
},
|
||||
{
|
||||
"id": "volume-delete",
|
||||
"label": "Delete/prune/volume",
|
||||
"target": "volume",
|
||||
"reason": "Data destructive operations are blocked.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def list_commands(session: Session) -> list[dict[str, Any]]:
|
||||
exported_at = datetime.now(UTC).isoformat()
|
||||
commands = []
|
||||
for command in VOICE_COMMANDS:
|
||||
policy = evaluate_policy(session, action=command["action"], target=command["target"], source="voice")
|
||||
decision = str(policy["decision"])
|
||||
if decision == "blocked":
|
||||
continue
|
||||
commands.append(
|
||||
{
|
||||
**_public_command(command),
|
||||
"policyDecision": decision,
|
||||
"requiresApproval": decision == "approval_required",
|
||||
"lastExportedAt": exported_at,
|
||||
}
|
||||
)
|
||||
return commands
|
||||
|
||||
|
||||
def command_by_id(session: Session, command_id: str) -> dict[str, Any] | None:
|
||||
return next((command for command in list_commands(session) if command["id"] == command_id), None)
|
||||
|
||||
|
||||
def test_intent(session: Session, text: str) -> dict[str, Any]:
|
||||
normalized = " ".join(text.lower().strip().split())
|
||||
if not normalized:
|
||||
return {"status": "unknown", "executed": False, "message": "No spoken phrase supplied."}
|
||||
|
||||
for command in list_commands(session):
|
||||
phrases = [
|
||||
str(command.get("spokenPhraseNl", "")).lower(),
|
||||
str(command.get("spokenPhraseEn", "")).lower(),
|
||||
str(command.get("label", "")).lower(),
|
||||
]
|
||||
if normalized in phrases or any(phrase and phrase in normalized for phrase in phrases):
|
||||
return dry_run(session, command["id"])
|
||||
|
||||
return {"status": "unknown", "executed": False, "message": "No allowlisted voice command matched."}
|
||||
|
||||
|
||||
def dry_run(session: Session, command_id: str) -> dict[str, Any]:
|
||||
command = command_by_id(session, command_id)
|
||||
if not command:
|
||||
return {"status": "unknown", "executed": False, "message": "Voice command is not allowlisted or is blocked."}
|
||||
|
||||
policy = evaluate_policy(session, action=command["action"], target=command["target"], source="voice")
|
||||
return {
|
||||
"status": "matched",
|
||||
"executed": False,
|
||||
"command": command,
|
||||
"policyPreview": {
|
||||
"decision": policy["decision"],
|
||||
"reason": policy["reason"],
|
||||
"risk": policy.get("risk", command["risk"]),
|
||||
},
|
||||
"request": {
|
||||
"method": command["method"],
|
||||
"endpoint": command["endpoint"],
|
||||
"payload": command["payload"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def request_approval(session: Session, command_id: str) -> dict[str, Any]:
|
||||
command = command_by_id(session, command_id)
|
||||
if not command:
|
||||
return {"success": False, "status": "unknown", "message": "Voice command is not allowlisted or is blocked."}
|
||||
policy = evaluate_policy(session, action=command["action"], target=command["target"], source="voice")
|
||||
if policy["decision"] == "allowed":
|
||||
return {"success": False, "status": "not_required", "message": "Command is allowed and does not need approval.", "command": command}
|
||||
if policy["decision"] == "blocked":
|
||||
return {"success": False, "status": "blocked", "message": policy["reason"], "command": command}
|
||||
|
||||
from .approval_service import create_approval
|
||||
|
||||
approval = create_approval(
|
||||
session,
|
||||
action=command["action"],
|
||||
target=command["target"],
|
||||
source="voice",
|
||||
risk=str(policy.get("risk", command["risk"])),
|
||||
reason=policy["reason"],
|
||||
requester="voice",
|
||||
)
|
||||
return {"success": True, "status": "approval_requested", "approvalId": approval.id, "command": command}
|
||||
|
||||
|
||||
def home_assistant_export(session: Session, base_url: str = 'http://appops:1216') -> dict[str, Any]:
|
||||
commands = list_commands(session)
|
||||
normalized_base_url = base_url.rstrip('/')
|
||||
return {
|
||||
"yaml": "\n\n".join([rest_commands_yaml(commands, normalized_base_url), scripts_yaml(commands), dashboard_yaml(commands), secrets_yaml()]),
|
||||
"restCommandsYaml": rest_commands_yaml(commands, normalized_base_url),
|
||||
"scriptsYaml": scripts_yaml(commands),
|
||||
"dashboardYaml": dashboard_yaml(commands),
|
||||
"secretsYaml": secrets_yaml(),
|
||||
"commands": commands,
|
||||
"blocked": BLOCKED_VOICE_EXAMPLES,
|
||||
"endpoints": [
|
||||
{
|
||||
"name": command["label"],
|
||||
"path": command["endpoint"],
|
||||
"method": command["method"],
|
||||
"policyDecision": command["policyDecision"],
|
||||
"requiresApproval": command["requiresApproval"],
|
||||
}
|
||||
for command in commands
|
||||
],
|
||||
"token": {
|
||||
"name": "APP_OPS_VOICE_TOKEN",
|
||||
"status": "required",
|
||||
"homeAssistantSecret": "appops_voice_authorization",
|
||||
"note": "HOME_ASSISTANT_TOKEN is only for AppOps-to-Home-Assistant notification delivery.",
|
||||
},
|
||||
"publicBaseUrl": normalized_base_url,
|
||||
"homeAssistantUrl": str(settings.apis.get('HOME_ASSISTANT_URL') or '').rstrip('/'),
|
||||
"setupGuide": [
|
||||
{"id": "token", "title": "Bewaar de AppOps bearer header", "detail": "Plaats 'Bearer ' plus APP_OPS_VOICE_TOKEN als appops_voice_authorization in Home Assistant secrets.yaml."},
|
||||
{"id": "rest", "title": "Installeer rest_command", "detail": "Voeg het rest_command-blok toe aan configuration.yaml of een Home Assistant package."},
|
||||
{"id": "scripts", "title": "Installeer de scripts", "detail": "Voeg het script-blok toe en herlaad scripts of herstart Home Assistant."},
|
||||
{"id": "test", "title": "Test eerst read-only", "detail": "Voer Check AppOps setup handmatig uit en controleer AppOps Audit."},
|
||||
{"id": "gemini", "title": "Koppel Gemini aan Assist", "detail": "Configureer Google Gemini Conversation met Control Home Assistant ingeschakeld en exposeer alleen de gewenste AppOps-scripts aan Assist."},
|
||||
{"id": "phone", "title": "Gebruik de Home Assistant Assist-pipeline", "detail": "Selecteer de Gemini conversation agent in je Assist-pipeline en start die pipeline vanuit de Home Assistant Companion-app op je telefoon."},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def rest_commands_yaml(commands: list[dict[str, Any]], base_url: str = 'http://appops:1216') -> str:
|
||||
lines = ["rest_command:"]
|
||||
for command in commands:
|
||||
lines.extend(
|
||||
[
|
||||
f" {command['homeAssistantRestCommandName']}:",
|
||||
f" url: \"{base_url.rstrip('/')}{command['endpoint']}\"",
|
||||
f" method: {command['method'].lower()}",
|
||||
" headers:",
|
||||
" Authorization: !secret appops_voice_authorization",
|
||||
" Content-Type: \"application/json\"",
|
||||
" payload: '{}'",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def scripts_yaml(commands: list[dict[str, Any]]) -> str:
|
||||
lines = ["script:"]
|
||||
for command in commands:
|
||||
lines.extend(
|
||||
[
|
||||
f" {command['homeAssistantScriptName']}:",
|
||||
f" alias: \"{command['label']}\"",
|
||||
f" description: \"{command['description']} Tell the user the returned AppOps status and never claim execution beyond the response.\"",
|
||||
" mode: single",
|
||||
" sequence:",
|
||||
" - action: rest_command." + command["homeAssistantRestCommandName"],
|
||||
" response_variable: appops_result",
|
||||
" - variables:",
|
||||
" appops_response:",
|
||||
" status: \"{{ appops_result.status | default(0) }}\"",
|
||||
" content: \"{{ appops_result.content | default('{}') }}\"",
|
||||
" - stop: \"AppOps request completed\"",
|
||||
" response_variable: appops_response",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def dashboard_yaml(commands: list[dict[str, Any]]) -> str:
|
||||
lines = [
|
||||
"# Lovelace dashboard snippet",
|
||||
"type: entities",
|
||||
"title: AppOps Voice Commands",
|
||||
"entities:",
|
||||
]
|
||||
for command in commands:
|
||||
lines.append(f" - entity: script.{command['homeAssistantScriptName']}")
|
||||
lines.append(f" name: \"{command['label']}\"")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def secrets_yaml() -> str:
|
||||
return 'secrets.yaml:\n appops_voice_authorization: "Bearer PASTE_APP_OPS_VOICE_TOKEN_HERE"'
|
||||
|
||||
|
||||
def _public_command(command: dict[str, Any]) -> dict[str, Any]:
|
||||
public = {key: value for key, value in command.items() if key not in {"action", "target"}}
|
||||
public["action"] = command["action"]
|
||||
public["target"] = command["target"]
|
||||
return public
|
||||
@@ -0,0 +1,588 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import AppSetting, JobRun
|
||||
from .job_runner import set_job_progress
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/workmesh"
|
||||
_STATE_KEY = "deployment:workmesh:first-deploy:v1"
|
||||
_RELEASE_REVISION = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05"
|
||||
_CONTAINERS = (
|
||||
"workmesh-postgres-1",
|
||||
"workmesh-mock-ats-1",
|
||||
"workmesh-api-1",
|
||||
"workmesh-n8n-1",
|
||||
"workmesh-workmesh-1",
|
||||
)
|
||||
_PORTS = (1255, 1256, 1257, 1258)
|
||||
|
||||
|
||||
# Fixed first-deploy bootstrap. Callers provide no repository, path, command,
|
||||
# port, environment value, or secret. Secret material is generated inside the
|
||||
# root-owned deployment controller and is never attached to exec output.
|
||||
_FIRST_DEPLOY_SCRIPT = r"""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
repository = "jens/workmesh"
|
||||
expected_release_revision = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05"
|
||||
slug = "workmesh"
|
||||
compose_project = "workmesh"
|
||||
containers = [
|
||||
"workmesh-postgres-1",
|
||||
"workmesh-mock-ats-1",
|
||||
"workmesh-api-1",
|
||||
"workmesh-n8n-1",
|
||||
"workmesh-workmesh-1",
|
||||
]
|
||||
ports = {1255, 1256, 1257, 1258}
|
||||
controller_root = Path("/opt/gitea-deploy")
|
||||
registry_path = controller_root / "projects.json"
|
||||
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
env_dir = Path("/srv/example/appdata/workmesh")
|
||||
env_path = env_dir / ".env"
|
||||
bare_repo = Path("/gitea-repositories/jens/workmesh.git")
|
||||
source_dir = Path("/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh")
|
||||
deploy_py = controller_root / "deploy.py"
|
||||
|
||||
base_project = {
|
||||
"branch": "main",
|
||||
"slug": slug,
|
||||
"source_dir": str(source_dir),
|
||||
"exact_source": True,
|
||||
"compose_files": ["compose.yaml"],
|
||||
"compose_project": compose_project,
|
||||
"containers": containers,
|
||||
"verify_oci_revision": False,
|
||||
"health_timeout": 300,
|
||||
}
|
||||
project = {**base_project, "env_source": str(env_path)}
|
||||
|
||||
for path in (registry_path, host_registry_path, lock_path, bare_repo, deploy_py):
|
||||
if path.is_symlink():
|
||||
raise SystemExit(20)
|
||||
if not registry_path.is_file() or not host_registry_path.is_file() or not deploy_py.is_file():
|
||||
raise SystemExit(20)
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(21)
|
||||
try:
|
||||
if not os.path.samefile(registry_path, host_registry_path):
|
||||
raise SystemExit(22)
|
||||
except OSError:
|
||||
raise SystemExit(22)
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
main = git("rev-parse", "refs/heads/main")
|
||||
revision = main.stdout.strip()
|
||||
if main.returncode != 0 or len(revision) != 40 or revision != expected_release_revision:
|
||||
raise SystemExit(23)
|
||||
for required in ("compose.yaml", "VERSION", ".itworx/deployment-target.json"):
|
||||
if git("cat-file", "-e", f"{revision}:{required}").returncode != 0:
|
||||
raise SystemExit(24)
|
||||
version = git("show", f"{revision}:VERSION")
|
||||
if version.returncode != 0 or version.stdout.strip() != "0.1.0":
|
||||
raise SystemExit(25)
|
||||
descriptor = git("show", f"{revision}:.itworx/deployment-target.json")
|
||||
try:
|
||||
descriptor_payload = json.loads(descriptor.stdout)
|
||||
except Exception:
|
||||
raise SystemExit(26)
|
||||
if descriptor_payload != {
|
||||
"apiVersion": "itworx.deployment-target/v1",
|
||||
"applicationId": "workmesh",
|
||||
"composeFile": "compose.yaml",
|
||||
"composeProject": "workmesh",
|
||||
"healthTimeout": 300,
|
||||
"verifyOciRevision": False,
|
||||
}:
|
||||
raise SystemExit(26)
|
||||
|
||||
# Refuse to take host ports from any non-WorkMesh container, including stopped
|
||||
# containers that would become conflicting if restarted later.
|
||||
listed = subprocess.run(
|
||||
["docker", "ps", "-aq"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
if listed.returncode != 0:
|
||||
raise SystemExit(27)
|
||||
for container_id in [line.strip() for line in listed.stdout.splitlines() if line.strip()]:
|
||||
inspected = subprocess.run(
|
||||
["docker", "inspect", container_id],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(27)
|
||||
try:
|
||||
item = json.loads(inspected.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(27)
|
||||
labels = dict(dict(item.get("Config") or {}).get("Labels") or {})
|
||||
if labels.get("com.docker.compose.project") == compose_project:
|
||||
continue
|
||||
bindings = dict(dict(item.get("HostConfig") or {}).get("PortBindings") or {})
|
||||
for values in bindings.values():
|
||||
for binding in values or []:
|
||||
raw = str(dict(binding).get("HostPort") or "")
|
||||
if raw.isdigit() and int(raw) in ports:
|
||||
raise SystemExit(28)
|
||||
|
||||
required_fixed = {
|
||||
"WORKMESH_ENV": "production",
|
||||
"WORKMESH_API_PORT": "1256",
|
||||
"WORKMESH_WEB_PORT": "1255",
|
||||
"WORKMESH_BIND_HOST": "127.0.0.1",
|
||||
"MOCK_ATS_PORT": "1258",
|
||||
"WORKMESH_POSTGRES_DB": "workmesh",
|
||||
"WORKMESH_POSTGRES_USER": "workmesh",
|
||||
"N8N_BASE_URL": "http://n8n:5678",
|
||||
"WORKMESH_API_BASE_URL": "http://api:8000",
|
||||
"N8N_PORT": "1257",
|
||||
"MOCK_ATS_BASE_URL": "http://mock-ats:8010",
|
||||
"DEMO_CONTROLS_ENABLED": "true",
|
||||
"AI_PROVIDER": "disabled",
|
||||
"AI_MODEL": "qwen3.5:9b",
|
||||
"AI_TIMEOUT_SECONDS": "30",
|
||||
"OLLAMA_BASE_URL": "http://host.docker.internal:11434",
|
||||
"OPENAI_COMPATIBLE_BASE_URL": "",
|
||||
"OPENAI_COMPATIBLE_API_KEY": "",
|
||||
"KNOWLEDGE_PROVIDER": "fixture",
|
||||
"RAGCORE_BASE_URL": "",
|
||||
"RAGCORE_TOKEN": "",
|
||||
"RAGCORE_SPACE_IDS": "",
|
||||
"RAGCORE_PROFILE": "hybrid-v1",
|
||||
"RAGCORE_TIMEOUT_SECONDS": "10",
|
||||
"NEXT_PUBLIC_WORKMESH_API_URL": "http://127.0.0.1:1256",
|
||||
"CORS_ORIGINS": "http://localhost:1255,http://127.0.0.1:1255",
|
||||
}
|
||||
secret_keys = (
|
||||
"WORKMESH_POSTGRES_PASSWORD",
|
||||
"WORKMESH_N8N_CALLBACK_SECRET",
|
||||
"N8N_ENCRYPTION_KEY",
|
||||
)
|
||||
|
||||
|
||||
def parse_env(path):
|
||||
values = {}
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
raise SystemExit(31)
|
||||
key, value = line.split("=", 1)
|
||||
if key in values or not key:
|
||||
raise SystemExit(31)
|
||||
values[key] = value
|
||||
return values
|
||||
|
||||
|
||||
if env_path.exists():
|
||||
if env_path.is_symlink() or not env_path.is_file():
|
||||
raise SystemExit(29)
|
||||
info = env_path.stat()
|
||||
if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077:
|
||||
raise SystemExit(30)
|
||||
env = parse_env(env_path)
|
||||
if set(env) != set(required_fixed) | set(secret_keys):
|
||||
raise SystemExit(31)
|
||||
if any(env.get(key) != value for key, value in required_fixed.items()):
|
||||
raise SystemExit(32)
|
||||
if any(len(env.get(key, "")) < 32 or "change-me" in env.get(key, "").lower() for key in secret_keys):
|
||||
raise SystemExit(33)
|
||||
else:
|
||||
env_dir.mkdir(parents=True, exist_ok=True)
|
||||
if env_dir.is_symlink() or not env_dir.is_dir():
|
||||
raise SystemExit(29)
|
||||
generated = {key: secrets.token_urlsafe(48) for key in secret_keys}
|
||||
env = {**required_fixed, **generated}
|
||||
temporary_env = env_dir / f".env.{os.getpid()}.tmp"
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
||||
descriptor_fd = os.open(temporary_env, flags, 0o600)
|
||||
try:
|
||||
with os.fdopen(descriptor_fd, "w", encoding="utf-8") as handle:
|
||||
for key in sorted(env):
|
||||
handle.write(f"{key}={env[key]}\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_env, env_path)
|
||||
os.chmod(env_path, 0o600)
|
||||
directory_fd = os.open(env_dir, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
finally:
|
||||
try:
|
||||
temporary_env.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
lock_handle = lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(34)
|
||||
with lock_handle as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(34)
|
||||
try:
|
||||
projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(35)
|
||||
if not isinstance(projects, dict):
|
||||
raise SystemExit(35)
|
||||
current = projects.get(repository)
|
||||
if current is not None:
|
||||
if not isinstance(current, dict):
|
||||
raise SystemExit(36)
|
||||
allowed_metadata = {"gitea_repository_id", "gitea_webhook_secret_file", "validation"}
|
||||
if set(current) - set(project) - allowed_metadata:
|
||||
raise SystemExit(36)
|
||||
for key, value in base_project.items():
|
||||
if current.get(key) != value:
|
||||
raise SystemExit(36)
|
||||
current_env = current.get("env_source")
|
||||
if current_env not in {None, str(env_path)}:
|
||||
raise SystemExit(36)
|
||||
updated_project = {**current, **project}
|
||||
else:
|
||||
updated_project = project
|
||||
|
||||
if current != updated_project:
|
||||
backup = host_registry_path.with_name(
|
||||
f"projects.json.workmesh-backup-{int(time.time())}-{os.getpid()}"
|
||||
)
|
||||
shutil.copy2(host_registry_path, backup)
|
||||
updated = dict(projects)
|
||||
updated[repository] = updated_project
|
||||
temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
|
||||
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.S_IMODE(host_registry_path.stat().st_mode))
|
||||
os.replace(temporary, host_registry_path)
|
||||
directory_fd = os.open(host_registry_path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
|
||||
# The official controller is the sole deployment executor. It revalidates the
|
||||
# exact revision, source tree, Compose topology, health, receipt and rollback.
|
||||
preflight = subprocess.run(
|
||||
["python3", str(deploy_py), "preflight", repository, revision],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=300,
|
||||
check=False,
|
||||
)
|
||||
if preflight.returncode != 0:
|
||||
raise SystemExit(40)
|
||||
# Refuse branch drift between preflight and the mutating deployment.
|
||||
main_again = git("rev-parse", "refs/heads/main")
|
||||
if main_again.returncode != 0 or main_again.stdout.strip() != revision:
|
||||
raise SystemExit(41)
|
||||
deployed = subprocess.run(
|
||||
["python3", str(deploy_py), "deploy", repository, revision],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=1800,
|
||||
check=False,
|
||||
)
|
||||
if deployed.returncode != 0:
|
||||
raise SystemExit(42)
|
||||
raise SystemExit(0)
|
||||
"""
|
||||
|
||||
|
||||
class WorkMeshFirstDeployment:
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
@staticmethod
|
||||
def _state(session: Session) -> AppSetting:
|
||||
row = session.get(AppSetting, _STATE_KEY)
|
||||
if row:
|
||||
return row
|
||||
row = AppSetting(
|
||||
key=_STATE_KEY,
|
||||
value_json={"state": "idle", "version": 1},
|
||||
is_secret=False,
|
||||
)
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def tracked_job_id(session: Session) -> str | None:
|
||||
row = session.get(AppSetting, _STATE_KEY)
|
||||
state = dict(row.value_json or {}) if row else {}
|
||||
job_id = str(state.get("jobId") or "")
|
||||
return job_id or None
|
||||
|
||||
@staticmethod
|
||||
def _result_for_exit(exit_code: int) -> dict[str, Any]:
|
||||
if exit_code == 0:
|
||||
return {
|
||||
"status": "deployed",
|
||||
"repository": _REPOSITORY,
|
||||
"revision": _RELEASE_REVISION,
|
||||
"compose_project": "workmesh",
|
||||
"containers": list(_CONTAINERS),
|
||||
"ports": list(_PORTS),
|
||||
}
|
||||
reasons = {
|
||||
20: "deployment controller registry or executable is unsafe or unavailable",
|
||||
21: "mounted WorkMesh bare repository is unavailable",
|
||||
22: "deployment registry host alias does not match the controller registry",
|
||||
23: "WorkMesh protected main is not the fixed v0.1.0 release revision",
|
||||
24: "required WorkMesh deployment artifacts are absent from protected main",
|
||||
25: "WorkMesh VERSION is not 0.1.0",
|
||||
26: "WorkMesh deployment descriptor differs from the fixed first-deploy contract",
|
||||
27: "Docker inventory could not be inspected safely",
|
||||
28: "one of the reserved WorkMesh loopback ports is already owned by another container",
|
||||
29: "WorkMesh environment directory or file is unsafe",
|
||||
30: "WorkMesh environment file permissions are too broad",
|
||||
31: "WorkMesh environment file keys are outside the fixed contract",
|
||||
32: "WorkMesh environment fixed values differ from the reviewed contract",
|
||||
33: "WorkMesh environment secrets are missing, weak, or development placeholders",
|
||||
34: "deployment registry lock could not be acquired",
|
||||
35: "deployment project registry is invalid",
|
||||
36: "an existing WorkMesh deployment target conflicts with the fixed contract",
|
||||
40: "official WorkMesh deployment preflight failed",
|
||||
41: "WorkMesh protected main changed between preflight and deployment",
|
||||
42: "official WorkMesh first deployment failed or rolled back",
|
||||
}
|
||||
return {
|
||||
"status": "failed",
|
||||
"reason": reasons.get(exit_code, "WorkMesh first deployment failed closed"),
|
||||
}
|
||||
|
||||
def run(self, session: Session, *, job: JobRun) -> dict[str, Any]:
|
||||
row = self._state(session)
|
||||
state = dict(row.value_json or {})
|
||||
if state.get("state") == "completed" and isinstance(state.get("result"), dict):
|
||||
return dict(state["result"])
|
||||
|
||||
execution_id = str(state.get("executionId") or "")
|
||||
if state.get("state") == "creating":
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "WorkMesh deployment exec creation has an unknown outcome; do not retry",
|
||||
}
|
||||
if state.get("state") == "created" and execution_id:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": (
|
||||
"WorkMesh deployment execution was created but its start boundary "
|
||||
"was not durably confirmed; do not retry"
|
||||
),
|
||||
"execution_id": execution_id,
|
||||
}
|
||||
|
||||
controller_status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if controller_status.get("status") != "ok":
|
||||
return {
|
||||
"status": "outcome_unknown" if state.get("state") == "started" else "error",
|
||||
"reason": "deployment controller is unavailable",
|
||||
**({"execution_id": execution_id} if execution_id else {}),
|
||||
}
|
||||
controller = dict(controller_status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {
|
||||
"status": "outcome_unknown" if state.get("state") == "started" else "error",
|
||||
"reason": "deployment controller is not running",
|
||||
**({"execution_id": execution_id} if execution_id else {}),
|
||||
}
|
||||
controller_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not controller_id:
|
||||
return {
|
||||
"status": "outcome_unknown" if state.get("state") == "started" else "error",
|
||||
"reason": "Portainer deployment control is unavailable",
|
||||
**({"execution_id": execution_id} if execution_id else {}),
|
||||
}
|
||||
|
||||
original_controller_id = str(state.get("controllerId") or "")
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
|
||||
if state.get("state") == "started" and execution_id:
|
||||
if original_controller_id != controller_id:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "deployment controller identity changed while first-deploy outcome was unknown",
|
||||
"execution_id": execution_id,
|
||||
}
|
||||
try:
|
||||
inspected = self.portainer._request(
|
||||
"GET",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/json",
|
||||
).json()
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": f"existing WorkMesh deployment outcome remains unknown: {redact_message(str(exc))}",
|
||||
"execution_id": execution_id,
|
||||
}
|
||||
if (
|
||||
str(inspected.get("ID") or "") != execution_id
|
||||
or str(inspected.get("ContainerID") or "") != controller_id
|
||||
):
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "WorkMesh deployment execution identity could not be verified",
|
||||
"execution_id": execution_id,
|
||||
}
|
||||
if bool(inspected.get("Running")):
|
||||
if job.status not in {"succeeded", "failed", "cancelled"}:
|
||||
set_job_progress(session, job, max(job.progress, 50), "Official WorkMesh first deployment is still running.")
|
||||
return {"status": "in_progress", "execution_id": execution_id}
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
result = self._result_for_exit(exit_code)
|
||||
row.value_json = {
|
||||
**state,
|
||||
"state": "completed",
|
||||
"result": result,
|
||||
"completedAt": utc_now().isoformat(),
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
if state.get("state") not in {"idle", None}:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "WorkMesh first-deploy state is not safely restartable",
|
||||
**({"execution_id": execution_id} if execution_id else {}),
|
||||
}
|
||||
|
||||
safe_controller = quote(controller_id, safe="")
|
||||
row.value_json = {
|
||||
"state": "creating",
|
||||
"version": 1,
|
||||
"jobId": job.id,
|
||||
"controllerId": controller_id,
|
||||
"createdAt": utc_now().isoformat(),
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_controller}/exec",
|
||||
json={
|
||||
"AttachStdout": False,
|
||||
"AttachStderr": False,
|
||||
"Tty": False,
|
||||
"Cmd": ["python3", "-c", _FIRST_DEPLOY_SCRIPT],
|
||||
},
|
||||
).json()
|
||||
execution_id = str(created.get("Id") or "")
|
||||
if not execution_id:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "WorkMesh deployment exec creation identity is unknown; do not retry",
|
||||
}
|
||||
row.value_json = {
|
||||
"state": "created",
|
||||
"version": 1,
|
||||
"jobId": job.id,
|
||||
"executionId": execution_id,
|
||||
"controllerId": controller_id,
|
||||
"createdAt": utc_now().isoformat(),
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
row.value_json = {
|
||||
**dict(row.value_json or {}),
|
||||
"state": "started",
|
||||
"startedAt": utc_now().isoformat(),
|
||||
}
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": f"WorkMesh deployment exec outcome is unknown: {redact_message(str(exc))}",
|
||||
**({"execution_id": execution_id} if execution_id else {}),
|
||||
}
|
||||
set_job_progress(session, job, 25, "Official WorkMesh first deployment started.")
|
||||
return {"status": "in_progress", "execution_id": execution_id}
|
||||
|
||||
|
||||
|
||||
def workmesh_first_deploy_job_cancellation_reason(
|
||||
session: Session,
|
||||
job: JobRun,
|
||||
) -> str | None:
|
||||
# A running fixed-target deployment may have reached the controller even
|
||||
# when its state row was lost or does not yet name this job. Keep both
|
||||
# cancellation entry points fail-closed until execution is reconciled.
|
||||
if (
|
||||
job.action_id == "deployment.target.deploy_workmesh"
|
||||
and job.target == _REPOSITORY
|
||||
and job.status == "running"
|
||||
):
|
||||
return (
|
||||
"WorkMesh first deployment may own an unresolved controller execution and "
|
||||
"cannot be cancelled until that exact execution is reconciled."
|
||||
)
|
||||
row = session.get(AppSetting, _STATE_KEY)
|
||||
state = dict(row.value_json or {}) if row else {}
|
||||
if str(state.get("jobId") or "") != job.id:
|
||||
return None
|
||||
if state.get("state") in {"created", "started"}:
|
||||
return (
|
||||
"WorkMesh first deployment owns an active controller execution and "
|
||||
"cannot be cancelled until that exact execution is reconciled."
|
||||
)
|
||||
return None
|
||||
@@ -0,0 +1,722 @@
|
||||
"""Approval-bound promotion for the single registered WorkMesh product target."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import update
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import Approval, WorkMeshPromotion
|
||||
from .approval_service import approval_fingerprint
|
||||
from .audit_logger import log_event
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
from .secret_store import get_secret
|
||||
|
||||
_ACTION = "deployment.target.promote_workmesh"
|
||||
_TARGET = "jens/workmesh"
|
||||
_SECRET_REF = "WORKMESH_DEPLOYMENT_BROKER"
|
||||
_API_VERSION = "itworx.workmesh-promotion/v2"
|
||||
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
_CHECKS = {"exactMain", "pushValidation", "controllerPreflight", "dataBackup"}
|
||||
_SAFE_PREFLIGHT_FIELDS = {
|
||||
"apiVersion",
|
||||
"status",
|
||||
"repository",
|
||||
"revision",
|
||||
"preflightSha256",
|
||||
"backupInputsSha256",
|
||||
"checks",
|
||||
}
|
||||
_TERMINAL = {"succeeded", "failed", "preflight_changed"}
|
||||
|
||||
|
||||
class WorkMeshPromotionError(ValueError):
|
||||
"""A fail-closed error in the fixed-target promotion contract."""
|
||||
|
||||
|
||||
def _idempotency_digest(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not _IDEMPOTENCY.fullmatch(normalized):
|
||||
raise WorkMeshPromotionError("Idempotency-Key must contain 8 to 128 safe characters.")
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _request_id(revision: str, idempotency_key: str) -> str:
|
||||
value = json.dumps(
|
||||
{
|
||||
"repository": _TARGET,
|
||||
"revision": revision,
|
||||
"operation": "workmesh_promote",
|
||||
"idempotency": idempotency_key,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _safe_preflight(result: dict[str, Any], revision: str) -> dict[str, Any] | None:
|
||||
checks = result.get("checks")
|
||||
digest = str(result.get("preflightSha256") or "")
|
||||
if (
|
||||
result.get("status") != "preflight_ok"
|
||||
or result.get("apiVersion") != _API_VERSION
|
||||
or result.get("repository") != _TARGET
|
||||
or result.get("revision") != revision
|
||||
or not _SHA256.fullmatch(digest)
|
||||
or not _SHA256.fullmatch(str(result.get("backupInputsSha256") or ""))
|
||||
or not isinstance(checks, dict)
|
||||
or set(checks) != _CHECKS
|
||||
or any(checks.get(key) != "passed" for key in _CHECKS)
|
||||
):
|
||||
return None
|
||||
return {
|
||||
key: result[key]
|
||||
for key in _SAFE_PREFLIGHT_FIELDS
|
||||
if key in result
|
||||
}
|
||||
|
||||
|
||||
def _serialize(
|
||||
row: WorkMeshPromotion,
|
||||
*,
|
||||
idempotent_replay: bool = False,
|
||||
approval_status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"promotionId": row.id,
|
||||
"jobId": row.id,
|
||||
"revision": row.revision,
|
||||
"status": row.status,
|
||||
"stage": row.stage,
|
||||
"preflight": dict(row.preflight_json or {}),
|
||||
"approvalId": row.approval_id,
|
||||
"approvalStatus": approval_status,
|
||||
"request": row.broker_request_id,
|
||||
"evidence": dict(row.evidence_json or {}),
|
||||
"result": row.result_summary,
|
||||
"auditId": row.audit_event_id,
|
||||
"createdAt": row.created_at.isoformat(),
|
||||
"updatedAt": row.updated_at.isoformat(),
|
||||
"completedAt": row.completed_at.isoformat() if row.completed_at else None,
|
||||
"idempotentReplay": idempotent_replay,
|
||||
}
|
||||
|
||||
|
||||
def _record_preflight_error(result: dict[str, Any]) -> str:
|
||||
# Never surface arbitrary controller response text or its transport details.
|
||||
status = str(result.get("status") or "unavailable")
|
||||
if status in {"unreachable", "unavailable", "unknown", "missing_config", "missing_secret"}:
|
||||
return "The fixed WorkMesh controller preflight is unavailable."
|
||||
if status in {"preflight_failed", "preflight_changed", "deployment_policy_rejected"}:
|
||||
return "The fixed WorkMesh controller rejected the requested revision preflight."
|
||||
return "The controller response did not satisfy the WorkMesh promotion contract."
|
||||
|
||||
|
||||
class WorkMeshPromotionService:
|
||||
def __init__(self, *, broker: DeploymentBrokerClient | None = None):
|
||||
self.broker = broker or DeploymentBrokerClient()
|
||||
|
||||
def preflight(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
revision: str,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
if not _REVISION.fullmatch(revision):
|
||||
raise WorkMeshPromotionError("revision must be a full lowercase 40-character Git SHA.")
|
||||
key_digest = _idempotency_digest(idempotency_key)
|
||||
existing = session.exec(
|
||||
select(WorkMeshPromotion).where(
|
||||
WorkMeshPromotion.idempotency_sha256 == key_digest
|
||||
)
|
||||
).first()
|
||||
if existing:
|
||||
if existing.revision != revision:
|
||||
raise WorkMeshPromotionError(
|
||||
"Idempotency-Key was already used for a different WorkMesh revision."
|
||||
)
|
||||
if existing.approval_id:
|
||||
approval = session.get(Approval, existing.approval_id)
|
||||
return _serialize(
|
||||
existing,
|
||||
idempotent_replay=True,
|
||||
approval_status=approval.status if approval else None,
|
||||
)
|
||||
raise WorkMeshPromotionError(
|
||||
"The original preflight request is incomplete and requires operator review."
|
||||
)
|
||||
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception:
|
||||
secret = None
|
||||
if not secret:
|
||||
raise WorkMeshPromotionError(
|
||||
"The fixed WorkMesh deployment-controller secret is not configured."
|
||||
)
|
||||
try:
|
||||
result = self.broker.workmesh_promotion_preflight(revision, secret=secret)
|
||||
except Exception as exc:
|
||||
raise WorkMeshPromotionError(
|
||||
"The fixed WorkMesh controller preflight is unavailable."
|
||||
) from exc
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "invalid_response"}
|
||||
safe = _safe_preflight(result, revision)
|
||||
if safe is None:
|
||||
raise WorkMeshPromotionError(_record_preflight_error(result))
|
||||
|
||||
promotion_id = f"workmesh-prom-{uuid.uuid4().hex}"
|
||||
context = {
|
||||
"apiVersion": _API_VERSION,
|
||||
"promotionId": promotion_id,
|
||||
"repository": _TARGET,
|
||||
"revision": revision,
|
||||
"preflightSha256": safe["preflightSha256"],
|
||||
"backupInputsSha256": safe["backupInputsSha256"],
|
||||
"preflight": safe,
|
||||
}
|
||||
approval = Approval(
|
||||
id=f"appr-{uuid.uuid4().hex[:12]}",
|
||||
action=_ACTION,
|
||||
target=_TARGET,
|
||||
source="manual",
|
||||
risk="high",
|
||||
status="pending",
|
||||
requester="operator",
|
||||
reason=(
|
||||
f"Promote WorkMesh revision {revision} after preflight digest "
|
||||
f"{safe['preflightSha256']}."
|
||||
),
|
||||
expires_at=utc_now() + timedelta(minutes=30),
|
||||
request_context_json=context,
|
||||
action_fingerprint=approval_fingerprint(
|
||||
_ACTION, _TARGET, "manual", "high", context
|
||||
),
|
||||
)
|
||||
row = WorkMeshPromotion(
|
||||
id=promotion_id,
|
||||
revision=revision,
|
||||
preflight_sha256=safe["preflightSha256"],
|
||||
idempotency_sha256=key_digest,
|
||||
status="approval_required",
|
||||
stage="preflight_accepted",
|
||||
preflight_json=safe,
|
||||
approval_id=approval.id,
|
||||
)
|
||||
# The approval and its immutable preflight binding become durable together.
|
||||
session.add(row)
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="deployment",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="WorkMesh promotion preflight passed for the exact revision.",
|
||||
result="approval_requested",
|
||||
execution_plan={
|
||||
"steps": [
|
||||
"Verify exact protected-main revision and push validation",
|
||||
"Run fixed WorkMesh controller preflight",
|
||||
"Bind operator approval to the immutable preflight digest",
|
||||
"Verify a PostgreSQL and n8n backup with isolated restore evidence before apply",
|
||||
],
|
||||
"promotionId": row.id,
|
||||
"revision": revision,
|
||||
"preflightSha256": row.preflight_sha256,
|
||||
"backupInputsSha256": safe["backupInputsSha256"],
|
||||
"approvalId": approval.id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": approval.id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return _serialize(row, approval_status=approval.status)
|
||||
|
||||
def execute_approved(self, session: Session, approval: Approval) -> dict[str, Any]:
|
||||
context = dict(approval.request_context_json or {})
|
||||
promotion_id = str(context.get("promotionId") or "")
|
||||
row = session.get(WorkMeshPromotion, promotion_id)
|
||||
if (
|
||||
approval.action != _ACTION
|
||||
or approval.target != _TARGET
|
||||
or approval.source != "manual"
|
||||
or approval.risk != "high"
|
||||
or approval.status != "approved"
|
||||
or (approval.expires_at is not None and approval.expires_at <= utc_now())
|
||||
or context.get("apiVersion") != _API_VERSION
|
||||
or not row
|
||||
or row.approval_id != approval.id
|
||||
or row.revision != context.get("revision")
|
||||
or row.preflight_sha256 != context.get("preflightSha256")
|
||||
or dict(row.preflight_json or {}).get("backupInputsSha256")
|
||||
!= context.get("backupInputsSha256")
|
||||
):
|
||||
return {
|
||||
"success": False,
|
||||
"status": "failed",
|
||||
"result": "blocked",
|
||||
"message": "Approval is not bound to a valid fixed WorkMesh promotion preflight.",
|
||||
}
|
||||
|
||||
if row.status != "approval_required":
|
||||
# The apply boundary is one-shot. This also closes the crash window
|
||||
# after persisting `submitting` but before learning the HTTP outcome.
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception:
|
||||
secret = None
|
||||
if not secret:
|
||||
return {
|
||||
"success": False,
|
||||
"status": "failed",
|
||||
"result": "blocked",
|
||||
"message": "The fixed WorkMesh deployment-controller secret is unavailable.",
|
||||
}
|
||||
try:
|
||||
current = self.broker.workmesh_promotion_preflight(row.revision, secret=secret)
|
||||
except Exception:
|
||||
current = {"status": "unavailable"}
|
||||
if not isinstance(current, dict):
|
||||
current = {"status": "invalid_response"}
|
||||
safe = _safe_preflight(current, row.revision)
|
||||
if (
|
||||
safe is None
|
||||
or safe["preflightSha256"] != row.preflight_sha256
|
||||
or safe["backupInputsSha256"]
|
||||
!= dict(row.preflight_json or {}).get("backupInputsSha256")
|
||||
):
|
||||
now = utc_now()
|
||||
claim = session.exec(
|
||||
update(WorkMeshPromotion)
|
||||
.where(
|
||||
(WorkMeshPromotion.id == row.id)
|
||||
& (WorkMeshPromotion.status == "approval_required")
|
||||
)
|
||||
.values(
|
||||
status="preflight_changed",
|
||||
stage="approval_invalidated",
|
||||
evidence_json={"preflightChanged": True},
|
||||
result_summary="The controller preflight changed after approval; no promotion was queued.",
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if claim.rowcount != 1:
|
||||
session.rollback()
|
||||
session.refresh(row)
|
||||
return self._approval_execution_result(row)
|
||||
approval.status = "expired"
|
||||
approval.decided_at = now
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="WorkMesh preflight changed after operator approval; promotion was not queued.",
|
||||
result="blocked",
|
||||
redacted_metadata={"promotionId": row.id, "revision": row.revision},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return {
|
||||
"success": False,
|
||||
"status": row.status,
|
||||
"result": "preflight_changed",
|
||||
"message": row.result_summary,
|
||||
"jobId": row.id,
|
||||
"auditId": audit.id,
|
||||
}
|
||||
|
||||
idempotency_key = f"workmesh-promotion:{row.id}"
|
||||
request_id = _request_id(row.revision, idempotency_key)
|
||||
# This durable marker is committed before the only mutating controller
|
||||
# request. Re-entry reconciles status and never repeats apply.
|
||||
now = utc_now()
|
||||
claim = session.exec(
|
||||
update(WorkMeshPromotion)
|
||||
.where(
|
||||
(WorkMeshPromotion.id == row.id)
|
||||
& (WorkMeshPromotion.status == "approval_required")
|
||||
)
|
||||
.values(
|
||||
status="submitting",
|
||||
stage="controller_request_submitted",
|
||||
broker_request_id=request_id,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
if claim.rowcount != 1:
|
||||
session.refresh(row)
|
||||
return self._approval_execution_result(row)
|
||||
session.refresh(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="Submitting the exact approved WorkMesh revision to the fixed controller target.",
|
||||
result="pending",
|
||||
execution_plan={
|
||||
"promotionId": row.id,
|
||||
"revision": row.revision,
|
||||
"preflightSha256": row.preflight_sha256,
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
"controllerRequest": request_id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": approval.id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
try:
|
||||
result = self.broker.workmesh_promotion_apply(
|
||||
row.revision,
|
||||
secret=secret,
|
||||
preflight_sha256=row.preflight_sha256,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
except Exception:
|
||||
result = {"status": "unavailable"}
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "invalid_response"}
|
||||
return self._accept_apply_result(session, row, result, request_id)
|
||||
|
||||
def _accept_apply_result(
|
||||
self,
|
||||
session: Session,
|
||||
row: WorkMeshPromotion,
|
||||
result: dict[str, Any],
|
||||
request_id: str,
|
||||
) -> dict[str, Any]:
|
||||
identity_matches = (
|
||||
result.get("request") == request_id
|
||||
and result.get("repository") == _TARGET
|
||||
and result.get("revision") == row.revision
|
||||
and result.get("operation") == "workmesh_promote"
|
||||
)
|
||||
status = str(result.get("status") or "unknown")
|
||||
if not identity_matches:
|
||||
self._mark_outcome_unknown(session, row)
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
if (
|
||||
status == "preflight_changed"
|
||||
and result.get("reason") == "approved_preflight_no_longer_matches"
|
||||
):
|
||||
now = utc_now()
|
||||
current_digest = str(result.get("preflightSha256") or "")
|
||||
row.status = "preflight_changed"
|
||||
row.stage = "approval_invalidated"
|
||||
row.evidence_json = {
|
||||
"preflightChanged": True,
|
||||
**(
|
||||
{"currentPreflightSha256": current_digest}
|
||||
if _SHA256.fullmatch(current_digest)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
row.result_summary = (
|
||||
"Controller preflight changed before queueing; no deployment was started and approval expired."
|
||||
)
|
||||
row.completed_at = now
|
||||
row.updated_at = now
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
if approval and approval.status == "approved":
|
||||
approval.status = "expired"
|
||||
approval.decided_at = now
|
||||
session.add(approval)
|
||||
session.add(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason=(
|
||||
"Controller preflight changed before queueing; no deployment was started "
|
||||
"and the approval was expired."
|
||||
),
|
||||
result="blocked",
|
||||
execution_plan={
|
||||
"promotionId": row.id,
|
||||
"revision": row.revision,
|
||||
"approvedPreflightSha256": row.preflight_sha256,
|
||||
"controllerRequest": request_id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": row.approval_id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
if (
|
||||
status == "deployed"
|
||||
and _SHA256.fullmatch(str(result.get("receiptSha256") or ""))
|
||||
and _SHA256.fullmatch(str(result.get("backupReceiptSha256") or ""))
|
||||
):
|
||||
row.status = "succeeded"
|
||||
row.stage = "receipt_verified"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"receiptSha256": result["receiptSha256"],
|
||||
"backupReceiptSha256": result["backupReceiptSha256"],
|
||||
}
|
||||
row.result_summary = "The exact WorkMesh revision has a verified controller receipt."
|
||||
row.completed_at = utc_now()
|
||||
elif status in {"accepted", "already_queued", "pending", "running"}:
|
||||
row.status = "applying"
|
||||
row.stage = "controller_accepted"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
}
|
||||
row.result_summary = "The fixed controller accepted the WorkMesh promotion; reconcile its receipt."
|
||||
elif status == "failed" and result.get("reason") in {
|
||||
"controller_rejected_before_mutation",
|
||||
"workmesh_preflight_changed",
|
||||
}:
|
||||
row.status = "preflight_changed"
|
||||
row.stage = "approval_invalidated"
|
||||
row.evidence_json = {"preflightChanged": True, "noMutation": True}
|
||||
row.result_summary = (
|
||||
"The controller rejected the approved backup preflight before deployment; approval expired."
|
||||
)
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
if approval and approval.status == "approved":
|
||||
approval.status = "expired"
|
||||
approval.decided_at = utc_now()
|
||||
session.add(approval)
|
||||
row.completed_at = utc_now()
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="Controller rejected the approved WorkMesh backup preflight before mutation.",
|
||||
result="blocked",
|
||||
execution_plan={
|
||||
"promotionId": row.id,
|
||||
"revision": row.revision,
|
||||
"preflightSha256": row.preflight_sha256,
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
"controllerRequest": request_id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": row.approval_id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
elif status == "failed":
|
||||
row.status = "failed"
|
||||
row.stage = "controller_failed"
|
||||
row.evidence_json = {"repository": _TARGET, "revision": row.revision}
|
||||
row.result_summary = "The fixed controller recorded a failed WorkMesh promotion."
|
||||
row.completed_at = utc_now()
|
||||
else:
|
||||
self._mark_outcome_unknown(session, row)
|
||||
return self._approval_execution_result(row)
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
@staticmethod
|
||||
def _mark_outcome_unknown(session: Session, row: WorkMeshPromotion) -> None:
|
||||
row.status = "outcome_unknown"
|
||||
row.stage = "reconciliation_required"
|
||||
row.evidence_json = {"reconciliationRequired": True}
|
||||
row.result_summary = (
|
||||
"Controller acceptance is ambiguous. Reconcile the fixed request status; do not resubmit."
|
||||
)
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
def _approval_execution_result(row: WorkMeshPromotion) -> dict[str, Any]:
|
||||
# Once apply might have crossed the network boundary, success here means
|
||||
# the approval was consumed and cannot be executed a second time.
|
||||
return {
|
||||
"success": row.status != "preflight_changed",
|
||||
"status": row.status,
|
||||
"result": row.status,
|
||||
"message": row.result_summary or "WorkMesh promotion request was recorded.",
|
||||
"jobId": row.id,
|
||||
"auditId": row.audit_event_id or "",
|
||||
}
|
||||
|
||||
def refresh(self, session: Session, promotion_id: str) -> dict[str, Any] | None:
|
||||
row = session.get(WorkMeshPromotion, promotion_id)
|
||||
if not row:
|
||||
return None
|
||||
if row.status in _TERMINAL or not row.broker_request_id:
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
return _serialize(row, approval_status=approval.status if approval else None)
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception:
|
||||
secret = None
|
||||
if not secret:
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
return _serialize(row, approval_status=approval.status if approval else None)
|
||||
try:
|
||||
result = self.broker.workmesh_promotion_status(
|
||||
row.broker_request_id, secret=secret
|
||||
)
|
||||
except Exception:
|
||||
result = {"status": "unavailable"}
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "unavailable"}
|
||||
|
||||
matches = (
|
||||
result.get("request") == row.broker_request_id
|
||||
and result.get("repository") == _TARGET
|
||||
and result.get("revision") == row.revision
|
||||
and result.get("operation") == "workmesh_promote"
|
||||
)
|
||||
observed_status = str(result.get("status") or "unknown")
|
||||
previous = row.status
|
||||
if matches and observed_status == "deployed":
|
||||
receipt_sha256 = str(result.get("receiptSha256") or "")
|
||||
backup_receipt_sha256 = str(result.get("backupReceiptSha256") or "")
|
||||
if _SHA256.fullmatch(receipt_sha256) and _SHA256.fullmatch(backup_receipt_sha256):
|
||||
row.status = "succeeded"
|
||||
row.stage = "receipt_verified"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"receiptSha256": receipt_sha256,
|
||||
"backupReceiptSha256": backup_receipt_sha256,
|
||||
}
|
||||
row.result_summary = "The exact WorkMesh revision has a verified controller receipt."
|
||||
row.completed_at = utc_now()
|
||||
elif matches and observed_status in {"pending", "running", "accepted", "already_queued"}:
|
||||
row.status = "applying"
|
||||
row.stage = "controller_running"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
}
|
||||
elif matches and observed_status == "failed" and result.get("reason") in {
|
||||
"controller_rejected_before_mutation",
|
||||
"workmesh_preflight_changed",
|
||||
}:
|
||||
row.status = "preflight_changed"
|
||||
row.stage = "approval_invalidated"
|
||||
row.evidence_json = {"preflightChanged": True, "noMutation": True}
|
||||
row.result_summary = (
|
||||
"The controller rejected the approved backup preflight before deployment; approval expired."
|
||||
)
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
if approval and approval.status == "approved":
|
||||
approval.status = "expired"
|
||||
approval.decided_at = utc_now()
|
||||
session.add(approval)
|
||||
row.completed_at = utc_now()
|
||||
elif matches and observed_status == "failed":
|
||||
row.status = "failed"
|
||||
row.stage = "controller_failed"
|
||||
row.evidence_json = {"repository": _TARGET, "revision": row.revision}
|
||||
row.result_summary = "The fixed controller recorded a failed WorkMesh promotion."
|
||||
row.completed_at = utc_now()
|
||||
else:
|
||||
# Not found, transport failure, missing receipt, or identity drift
|
||||
# cannot prove that it is safe to submit again.
|
||||
row.status = "outcome_unknown"
|
||||
row.stage = "reconciliation_required"
|
||||
row.evidence_json = {"reconciliationRequired": True}
|
||||
row.result_summary = (
|
||||
"Controller acceptance is not verified. Keep reconciling this request; do not resubmit."
|
||||
)
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
if row.status != previous:
|
||||
audit = log_event(
|
||||
session,
|
||||
source="deployment",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="deployment",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="Reconciled the fixed WorkMesh controller request without resubmission.",
|
||||
result="success" if row.status == "succeeded" else row.status,
|
||||
execution_plan={"promotionId": row.id, "revision": row.revision},
|
||||
redacted_metadata={"promotionId": row.id, "controllerStatus": observed_status},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
return _serialize(row, approval_status=approval.status if approval else None)
|
||||
|
||||
@staticmethod
|
||||
def list_promotions(session: Session, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
rows = session.exec(
|
||||
select(WorkMeshPromotion)
|
||||
.order_by(WorkMeshPromotion.created_at.desc())
|
||||
.limit(max(1, min(limit, 50)))
|
||||
).all()
|
||||
# Listing is a durable local read. Reconciliation is an explicit GET by
|
||||
# promotion id so page loads never create an unbounded controller poll.
|
||||
result = []
|
||||
for row in rows:
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
result.append(_serialize(row, approval_status=approval.status if approval else None))
|
||||
return result
|
||||
Reference in new issue
Block a user