From 767e66921e33d9868604b0d484374cac50db00eb Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Sat, 3 Oct 2026 13:35:10 +0000 Subject: [PATCH] Publish curated Unraid-AppOps-Gateway source from a506550ef83ef2d81651e37c98a89f7ee330c730 --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 289 +- backend/app/connectors/gitea_actions.py | 18 +- backend/app/connectors/portainer.py | 20 +- backend/app/core/config.py | 46 + backend/app/core/security.py | 9 + backend/app/main.py | 16 +- backend/app/mcp_server.py | 8 + backend/app/runner_statistics_mcp.py | 164 + backend/app/services/action_execution.py | 365 ++- backend/app/services/action_registry.py | 12 + .../controller_artifact_helper_spec.py | 128 + .../services/controller_artifact_promoter.py | 87 +- backend/app/services/deployment_broker.py | 34 +- .../services/deployment_target_registry.py | 4 +- ...dec_dashboard_deployment_reconciliation.py | 4 +- .../portfolio_deployment_enrollment.py | 6 +- .../services/portfolio_deployment_recovery.py | 2 +- .../privatemuse_deployment_enrollment.py | 4 +- .../privatemuse_deployment_recovery.py | 2 +- .../services/privatemuse_webhook_repair.py | 4 +- .../services/repopulse_adoption_preflight.py | 4 +- backend/app/services/runner_statistics.py | 181 ++ .../runnerless_target_provisioning.py | 4 +- .../app/services/workmesh_first_deployment.py | 4 +- .../tests/test_appops_live_state_recovery.py | 5 + backend/tests/test_appops_migration_bridge.py | 198 ++ .../test_appops_self_upgrade_controller.py | 103 +- backend/tests/test_ci_job_docker.py | 550 ++++ backend/tests/test_container_restart.py | 6 +- .../test_controller_artifact_bootstrap.py | 1106 +++++++ .../test_controller_artifact_promoter.py | 99 +- .../test_controller_bootstrap_import_host.py | 41 + backend/tests/test_controller_config_paths.py | 69 + .../test_controller_hub_release_volume.py | 179 +- .../test_controller_state_reconciliation.py | 14 +- backend/tests/test_deploy_broker.py | 1442 ++++++++- .../tests/test_deployment_target_registry.py | 2 +- backend/tests/test_gitea_actions.py | 57 + ...t_host_guard_memory_threshold_installer.py | 228 ++ backend/tests/test_hub_candidate_delivery.py | 194 ++ backend/tests/test_hub_candidate_import.py | 837 +++++ backend/tests/test_hub_candidate_transport.py | 236 ++ .../tests/test_hub_permission_preparation.py | 775 +++++ .../test_mcp_hub_candidate_preparation.py | 147 + .../tests/test_mcp_hub_governed_topology.py | 162 +- ...t_mcp_hub_legacy_registry_compatibility.py | 104 + backend/tests/test_mcp_hub_release_store.py | 86 + ...dec_dashboard_deployment_reconciliation.py | 5 +- .../tests/test_portainer_restart_timeout.py | 106 +- .../test_portfolio_deployment_enrollment.py | 4 +- .../test_privatemuse_deployment_enrollment.py | 2 +- .../tests/test_privatemuse_webhook_repair.py | 2 +- .../test_repopulse_target_attestation.py | 50 + .../tests/test_runner_admission_controller.py | 1640 ++++++++++ backend/tests/test_runner_admission_policy.py | 179 ++ .../tests/test_runner_capacity_controller.py | 275 ++ .../tests/test_runner_controller_artifact.py | 69 + backend/tests/test_runner_fleet.py | 82 +- .../test_runner_fleet_scheduler_recovery.py | 537 +++- backend/tests/test_runner_fleet_specialize.py | 2719 +++++++++++++++++ backend/tests/test_runner_profiles.py | 90 + .../tests/test_runner_resource_envelope.py | 1106 +++++++ ...unner_specialization_controller_surface.py | 158 + .../tests/test_runner_statistics_binding.py | 391 +++ backend/tests/test_runner_statistics_mcp.py | 301 ++ .../tests/test_runner_statistics_surface.py | 234 ++ .../tests/test_runner_task_image_producer.py | 230 ++ 68 files changed, 15924 insertions(+), 313 deletions(-) create mode 100644 backend/app/runner_statistics_mcp.py create mode 100644 backend/app/services/controller_artifact_helper_spec.py create mode 100644 backend/app/services/runner_statistics.py create mode 100644 backend/tests/test_appops_migration_bridge.py create mode 100644 backend/tests/test_ci_job_docker.py create mode 100644 backend/tests/test_controller_artifact_bootstrap.py create mode 100644 backend/tests/test_host_guard_memory_threshold_installer.py create mode 100644 backend/tests/test_hub_candidate_delivery.py create mode 100644 backend/tests/test_hub_candidate_import.py create mode 100644 backend/tests/test_hub_candidate_transport.py create mode 100644 backend/tests/test_hub_permission_preparation.py create mode 100644 backend/tests/test_mcp_hub_legacy_registry_compatibility.py create mode 100644 backend/tests/test_runner_admission_controller.py create mode 100644 backend/tests/test_runner_admission_policy.py create mode 100644 backend/tests/test_runner_controller_artifact.py create mode 100644 backend/tests/test_runner_fleet_specialize.py create mode 100644 backend/tests/test_runner_profiles.py create mode 100644 backend/tests/test_runner_resource_envelope.py create mode 100644 backend/tests/test_runner_specialization_controller_surface.py create mode 100644 backend/tests/test_runner_statistics_binding.py create mode 100644 backend/tests/test_runner_statistics_mcp.py create mode 100644 backend/tests/test_runner_statistics_surface.py create mode 100644 backend/tests/test_runner_task_image_producer.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 991c00f..859e8eb 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `2e53f628d284213b90ca2f9c5e121671f5933574`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `a506550ef83ef2d81651e37c98a89f7ee330c730`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index ad032f1..1720422 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "2e53f628d284213b90ca2f9c5e121671f5933574", + "sourceRevision": "a506550ef83ef2d81651e37c98a89f7ee330c730", "files": [ { "path": ".dockerignore", @@ -294,8 +294,8 @@ }, { "path": "backend/app/connectors/gitea_actions.py", - "sha256": "711a92fcfdb0a27bd192d587d9b2597d2c64852f4343929a0e93f95042467323", - "bytes": 18661 + "sha256": "b367bebc60e2542c7d7763c78e4eabfa98bca5d2be753e387c8faa269fdbe330", + "bytes": 18440 }, { "path": "backend/app/connectors/gpu_metrics.py", @@ -314,8 +314,8 @@ }, { "path": "backend/app/connectors/portainer.py", - "sha256": "b22b3d36294d2aaa1dcbcc0e140d515893ec0ba58f095c5f6ccb6d82f161a517", - "bytes": 19001 + "sha256": "dcbfb3926929b10e08190944acf0217a71d9696f0c8e91b24fe832bc71034f8f", + "bytes": 19655 }, { "path": "backend/app/connectors/prowlarr.py", @@ -354,8 +354,8 @@ }, { "path": "backend/app/core/config.py", - "sha256": "c95e402679a238042c13d816d00a81ba4374d146eb13aab0b24ddb2f215fe8ea", - "bytes": 23746 + "sha256": "83441a676b4e6f415f80b283897bb613805ecbf82a6d405d767532e086dbc5fc", + "bytes": 25697 }, { "path": "backend/app/core/network_security.py", @@ -369,8 +369,8 @@ }, { "path": "backend/app/core/security.py", - "sha256": "2ddc677fe34e4818ba33cb287ad57df2caeb43849940f4d1ec760dab83be7bc3", - "bytes": 3257 + "sha256": "abc3de16a802253ea3fa7e0bdaed51efb4d99a82e353ec062fdadb6c20e5e313", + "bytes": 3837 }, { "path": "backend/app/core/time.py", @@ -399,13 +399,18 @@ }, { "path": "backend/app/main.py", - "sha256": "07ff0720575a518131ea6a80027cb515210bdf433f721c79050bbad0bddbf137", - "bytes": 17347 + "sha256": "17aa07c08bbe9eecbc0c100a6b573ef7db4b682e9bd7c6e1cb5982f72966d853", + "bytes": 18104 }, { "path": "backend/app/mcp_server.py", - "sha256": "75779ae0a366cca415c04fc6fe24f38d99c338939874acf17be6616cc12b4f25", - "bytes": 35237 + "sha256": "7cb748e5350231bfe938c91867b7eef408dd3589ba4e7ec078e07589338554c6", + "bytes": 35626 + }, + { + "path": "backend/app/runner_statistics_mcp.py", + "sha256": "72b72bd512c8ace97ce3eea54ee4babaa0156044535f793411cd0228eb294ec8", + "bytes": 5860 }, { "path": "backend/app/schemas/ai_remediation.py", @@ -444,13 +449,13 @@ }, { "path": "backend/app/services/action_execution.py", - "sha256": "fb32be9b89ac8235f826aabd8cb6ca0068fc9f5f2e5fc95a9963dac8c1df2cb3", - "bytes": 83876 + "sha256": "3ba0a33cd6947dee1d7961e63e6e9a4b247eda3ea8253537b9f879a57cf063ac", + "bytes": 99269 }, { "path": "backend/app/services/action_registry.py", - "sha256": "b34a8217b0bbdaf1a9cda8d7bc310f5577a3dea0abf4bcc95d106a8227dd8ea9", - "bytes": 20499 + "sha256": "11add43608abc05d51d89d29d772fe17f6d429a032c471376ad8f6d8400976b2", + "bytes": 21035 }, { "path": "backend/app/services/ai_remediation_planner.py", @@ -492,10 +497,15 @@ "sha256": "320b7edc1cc47701216897929bb4197fe7cf496a61ef4efae477658482aaf185", "bytes": 9150 }, + { + "path": "backend/app/services/controller_artifact_helper_spec.py", + "sha256": "1da47a3f061ef0b89158c9f646bb7018ce88b81c0687d8b27abe65b74df67fea", + "bytes": 4948 + }, { "path": "backend/app/services/controller_artifact_promoter.py", - "sha256": "6e740205449af2adc1fbbf9adf15989532fc6a92e252c8dcc81e7b4d644e58cd", - "bytes": 16389 + "sha256": "bc06a28941c111b5a0f6eed96e407259626c0b9fc1ba5ecb8953befc83b66304", + "bytes": 13602 }, { "path": "backend/app/services/controller_bootstrap_import.py", @@ -504,8 +514,8 @@ }, { "path": "backend/app/services/deployment_broker.py", - "sha256": "2feb43b65684f3f6a2dba688a47c18117d907ca34426919270f67e4de42db453", - "bytes": 23266 + "sha256": "829fe26916967dad3c6907989a5a8825a0aabbc294aa2cf600e701bc76c5e1e3", + "bytes": 24904 }, { "path": "backend/app/services/deployment_control_plane.py", @@ -514,8 +524,8 @@ }, { "path": "backend/app/services/deployment_target_registry.py", - "sha256": "c3a454fee8613439d28728d97f7db2d80e0854bd8b48e9d4f72378d1b6f6068b", - "bytes": 20175 + "sha256": "322208e3793306d72d47685ee1567a568a64078abe7547de3f2d0721ec888851", + "bytes": 20168 }, { "path": "backend/app/services/discovery_scheduler.py", @@ -599,8 +609,8 @@ }, { "path": "backend/app/services/media_codec_dashboard_deployment_reconciliation.py", - "sha256": "78648b078b7525e15a8ac136e13914db1fae6835325901a3ba5b4965e7f1adbe", - "bytes": 43243 + "sha256": "1f61df61d796e9d8aafa2f6dab137ccb8c5422e6e69a681d33362fe584c566e2", + "bytes": 43236 }, { "path": "backend/app/services/media_pipeline_service.py", @@ -644,13 +654,13 @@ }, { "path": "backend/app/services/portfolio_deployment_enrollment.py", - "sha256": "04e7a3b9a603f9cf50476d9e3f2e67e46d8e169b0a6952eb3753dc6d059d6418", - "bytes": 33458 + "sha256": "55acd53bf2a7b412935f61a60e5b21ad27eb591e6bfe0741fe85d4105e8a4d1e", + "bytes": 33452 }, { "path": "backend/app/services/portfolio_deployment_recovery.py", - "sha256": "c1adb3734ca68eb961b61def1925cdb08a82a562767813064c170f16b2754865", - "bytes": 17107 + "sha256": "f7bf495631ec35359f7adc47642b2ae2089ba2c9c8e73447664b6edcb9a7a8fb", + "bytes": 17108 }, { "path": "backend/app/services/preflight_service.py", @@ -664,18 +674,18 @@ }, { "path": "backend/app/services/privatemuse_deployment_enrollment.py", - "sha256": "523834b875641603fb3f3aedf29bcd72ec3bcfa2a9c5f2c587961adddae96f19", - "bytes": 21876 + "sha256": "fa12901f31b5ce13b6ad32669e7012906d1d713d6881e7d410b0ab10645e7116", + "bytes": 21869 }, { "path": "backend/app/services/privatemuse_deployment_recovery.py", - "sha256": "36b5a74971ef746712b3d557d3a35001334838e6b892ffbce9da4ac713b38b04", - "bytes": 47799 + "sha256": "df9cde501fa48bcad15387596c4c870cb8651ee8def93b92b65d274f1d7d4c9c", + "bytes": 47800 }, { "path": "backend/app/services/privatemuse_webhook_repair.py", - "sha256": "114cf8f6ee8cc2e93cdadaacc1034209dd25c11407cac6740a61d14db7c25d8c", - "bytes": 11884 + "sha256": "79a7ae42f57bb3e4b556230b9c7624fff8a072e80565c938992e242b6bf14839", + "bytes": 11886 }, { "path": "backend/app/services/repopulse_adoption_journal.py", @@ -689,8 +699,8 @@ }, { "path": "backend/app/services/repopulse_adoption_preflight.py", - "sha256": "b92cdccfdd0458bd5a9debc49b12887105fc93d19b169027fa40a2b661d104e0", - "bytes": 37153 + "sha256": "b993d8c12a16ffc720b207d9d5191ea41f603786e6c1f085895d431c799a02c8", + "bytes": 37146 }, { "path": "backend/app/services/repopulse_adoption_runner.py", @@ -732,10 +742,15 @@ "sha256": "2a10a4996456f5dec30b466fcb5c3fc375a31e6dd512df9ddf7c7ed055019ff3", "bytes": 3212 }, + { + "path": "backend/app/services/runner_statistics.py", + "sha256": "1f132221db4bcd2a7e122f0c0838ce2c8597e969dc912853444eb9a7fe52cbbf", + "bytes": 7735 + }, { "path": "backend/app/services/runnerless_target_provisioning.py", - "sha256": "99b503697365a750379bd6d15da3ee89d9fcc97c77c8aac2be6a98730cd2c3a9", - "bytes": 17552 + "sha256": "ddadcbf82d6b37df18f415f969891ecf31726aef7a9d880c7becf9cc96eb6ffc", + "bytes": 17602 }, { "path": "backend/app/services/runtime_service.py", @@ -774,8 +789,8 @@ }, { "path": "backend/app/services/workmesh_first_deployment.py", - "sha256": "4af175ab68a572779ada153783f417e1045321418ed41b87b4c226b2ed2a7a27", - "bytes": 22488 + "sha256": "1ce6581b378bb9ee341c93047e8e5430f80fb053462bf83ad50d766b98abe06f", + "bytes": 22487 }, { "path": "backend/app/services/workmesh_promotion.py", @@ -809,38 +824,53 @@ }, { "path": "backend/tests/test_appops_live_state_recovery.py", - "sha256": "33b5fa25e441128f84c4e518cef803fa9229ffe04a6ec4e980a3bb2707175959", - "bytes": 12368 + "sha256": "e65bf9968fba65318295ea3d2395da8f7a567657d8050029bd8db323cad5a0dc", + "bytes": 12658 + }, + { + "path": "backend/tests/test_appops_migration_bridge.py", + "sha256": "11af3dfaf72d35affa1b26a52d7850c8e4d44d36183728f326e5e615bcbde102", + "bytes": 11481 }, { "path": "backend/tests/test_appops_self_upgrade_controller.py", - "sha256": "ad6d8fba2213c1c8018bc9ff1c2ebd61684792c27fa00a07195c5d6aea99d69f", - "bytes": 20392 + "sha256": "b41b936e6351a9e2a5a947b7d69e13eb05d2eb6b02bd5bee1b4d3fdfa2c63eea", + "bytes": 23121 }, { "path": "backend/tests/test_authentik_connector.py", "sha256": "1ea6eaea54f0ce6f5a1e92a6c338edb880eff4c6bc43f09a32ea7a0e4a537870", "bytes": 28252 }, + { + "path": "backend/tests/test_ci_job_docker.py", + "sha256": "7292a3ad3e52e934e6a410fdea8adb6a00dc90fe8de28d8159205ec069951d73", + "bytes": 27439 + }, { "path": "backend/tests/test_container_restart.py", - "sha256": "2b84085e0e35d4405f39a6b5274b0ba123191c5855306bc0bd7bb67f6c3b7822", - "bytes": 108066 + "sha256": "6bbe8c44a07a0609873148c5b8a0775d375c92c2a5f44ffbfeafb1960d462cab", + "bytes": 108190 + }, + { + "path": "backend/tests/test_controller_artifact_bootstrap.py", + "sha256": "fcb29fae7778e00a3c84e6bcdb51a20ffaea73c24b5a8bd4304058825e30d359", + "bytes": 42896 }, { "path": "backend/tests/test_controller_artifact_promoter.py", - "sha256": "671f62d61ec1281644d0b49500141dd4a50ef75b7bf82accbe80dd0bf16d04ec", - "bytes": 3922 + "sha256": "8eee2bdaf60422824f605637b1820c431fdacea305e1b68b3618fffa0dc000ee", + "bytes": 7570 }, { "path": "backend/tests/test_controller_bootstrap_import_host.py", - "sha256": "541f786746b42447fb738f14343fb6d1cb0097d10a5e4949f0e940e111a6aec3", - "bytes": 1577 + "sha256": "58c153b8e3d1b016006b0bed5c98f9aa069d58e670d65edac368833ae38dd6f1", + "bytes": 3060 }, { "path": "backend/tests/test_controller_config_paths.py", - "sha256": "9cbe851651abec8c5a792d9843ceca308993e71f67bdcbc081ae7eb9a96a0888", - "bytes": 5207 + "sha256": "1081ecf7206e59be24aebf92284d3833743b1a84eeb1cefc3891a340dea883ae", + "bytes": 8254 }, { "path": "backend/tests/test_controller_host_dependencies.py", @@ -849,8 +879,8 @@ }, { "path": "backend/tests/test_controller_hub_release_volume.py", - "sha256": "8c7dd4f95407c9b274f8ee6ca9c98088ee0e842afa0c537cbd1d7311ec9bfa30", - "bytes": 32553 + "sha256": "2aedc6c8f97a9e6aaa58ab43c79c2189360e4f254519314558f77c2de72b25ee", + "bytes": 41287 }, { "path": "backend/tests/test_controller_root_procedure_contract.py", @@ -859,8 +889,8 @@ }, { "path": "backend/tests/test_controller_state_reconciliation.py", - "sha256": "d00b8bcf388e17f0c1b8999d4b511fede7212b8ef83282905328d1b0b4b70f76", - "bytes": 10523 + "sha256": "d9ddea9971414954a0b34738ef8de6bb330a867f7f41376063e473855ff5d17e", + "bytes": 10957 }, { "path": "backend/tests/test_deluge_connector.py", @@ -869,8 +899,8 @@ }, { "path": "backend/tests/test_deploy_broker.py", - "sha256": "6bc23cb5883961316acae07daf12ec261e1c21e229bf5ee8ebc56ff245f6a34d", - "bytes": 309532 + "sha256": "56dffec32ac7383b9d9cf0f82bfb7e586458cb2780bfefb0bfc5785b97bf4736", + "bytes": 381065 }, { "path": "backend/tests/test_deployment_auto_rollback.py", @@ -894,8 +924,8 @@ }, { "path": "backend/tests/test_deployment_target_registry.py", - "sha256": "32c90b7c065183bbc0595f568e1729fc98c787a9fa8a158b082d15f90700fde8", - "bytes": 9437 + "sha256": "2c28d05cbf6156d4ab53e2b52a770e2b0bda34138a3d715ba5d97bca0b9e8ce7", + "bytes": 9429 }, { "path": "backend/tests/test_dockdeck_classifier.py", @@ -929,8 +959,8 @@ }, { "path": "backend/tests/test_gitea_actions.py", - "sha256": "b2c99ac7e5548b9394cce6ce6f8a3fd0e4b7bdf6c72006c95edb6aac108c35fd", - "bytes": 10766 + "sha256": "7c4bebd43553b6de6738bdcb70153640a8ec6f2e855c09eac35a40345995b1d8", + "bytes": 12826 }, { "path": "backend/tests/test_gitea_project_auto_registration.py", @@ -947,6 +977,31 @@ "sha256": "718603f23478261e178bc4377fcea35322a59ff5cb2912e7d4a685fd9da7c2e4", "bytes": 8311 }, + { + "path": "backend/tests/test_host_guard_memory_threshold_installer.py", + "sha256": "c50a26646d73a5fe0279b5d9923f121c31fc2c59ee9b8f21cb2a34478729147c", + "bytes": 7846 + }, + { + "path": "backend/tests/test_hub_candidate_delivery.py", + "sha256": "582c987db50715930b79f0fe07d88e9acc79c41905c973e9935c865695a7e714", + "bytes": 9323 + }, + { + "path": "backend/tests/test_hub_candidate_import.py", + "sha256": "178a07b0dc9489545723c8ff5ecbfd7060aa0c1491ba6c1ced50edd7f3481e68", + "bytes": 30015 + }, + { + "path": "backend/tests/test_hub_candidate_transport.py", + "sha256": "da0b302e0bf45d87f0b3339b2c1392aec22e650f45468ab236fc37f448d9f9ea", + "bytes": 9353 + }, + { + "path": "backend/tests/test_hub_permission_preparation.py", + "sha256": "3ba8efe338a21b100e4bc20e4fbffe043eb57c411a4143c00332fdb368db5011", + "bytes": 28482 + }, { "path": "backend/tests/test_inventory_discovery.py", "sha256": "db7a7fbe23ef01f2a4e2e58ca72514889b9740a04374851b4a414118908bc594", @@ -984,8 +1039,8 @@ }, { "path": "backend/tests/test_mcp_hub_candidate_preparation.py", - "sha256": "aa5525214e034464ac3586252ba5bad09f59c96c861cb9449a3dc86ae7a91c74", - "bytes": 22525 + "sha256": "ed0d60e8f0e4371622405dc200e80ecfa3c4574ad2a9439373032f997a1af414", + "bytes": 27390 }, { "path": "backend/tests/test_mcp_hub_deployment_reconciliation.py", @@ -994,8 +1049,13 @@ }, { "path": "backend/tests/test_mcp_hub_governed_topology.py", - "sha256": "117c07dfb34e9ab0fac5675dd51585906b33d7bf5ea0a059d2d8f6eb4d2897a7", - "bytes": 118901 + "sha256": "9e306fbce479f15610a87ad666023de7deae588dc01f69be6024140d5df2158d", + "bytes": 125396 + }, + { + "path": "backend/tests/test_mcp_hub_legacy_registry_compatibility.py", + "sha256": "f382839f7ac5ec5a038f305bd9204e01514b73ad2b6116e6d388ec2d4f728b25", + "bytes": 4101 }, { "path": "backend/tests/test_mcp_hub_policy_reconciliation.py", @@ -1009,8 +1069,8 @@ }, { "path": "backend/tests/test_mcp_hub_release_store.py", - "sha256": "91dbd0da6a131f76a04c51f93e31a77f4acd057f788f4b1a6fce75d37c23525e", - "bytes": 18199 + "sha256": "4483020ccfe8cd31cc00ac4e9030a792af176e2d54929316a954bf738237f571", + "bytes": 21241 }, { "path": "backend/tests/test_mcp_rag_n8n_integrations.py", @@ -1019,8 +1079,8 @@ }, { "path": "backend/tests/test_media_codec_dashboard_deployment_reconciliation.py", - "sha256": "a7fd78d28e191f3370aec320c88dff150682d3ca8acfcb5766b59ed6978e6e9c", - "bytes": 9161 + "sha256": "7933221aedbaa0dd1b1c4e0fcde9987ec60d6b9f390ab5b3f59529e681a8d35a", + "bytes": 9088 }, { "path": "backend/tests/test_media_codec_dashboard_reconcile_diagnostics.py", @@ -1069,13 +1129,13 @@ }, { "path": "backend/tests/test_portainer_restart_timeout.py", - "sha256": "3502178913145313afe376ccde8edcfc452ca723fa161aa572debba0b2f57a96", - "bytes": 1339 + "sha256": "6ac7e5234de0d044b77c35390da14a26d26c99f21ecf35dd2e690ec8bcf28f3a", + "bytes": 4354 }, { "path": "backend/tests/test_portfolio_deployment_enrollment.py", - "sha256": "3d70cc965d56db374ab88d1df3a25ac497807d0b1f0c3d385789b20fd8fe57e4", - "bytes": 6734 + "sha256": "6ff8929444d3455db66955270dcd57750f9b3c4ab55c96daccc127386a68cb2e", + "bytes": 6727 }, { "path": "backend/tests/test_portfolio_deployment_recovery.py", @@ -1099,8 +1159,8 @@ }, { "path": "backend/tests/test_privatemuse_deployment_enrollment.py", - "sha256": "2739d9e0d31db4eaec6c18c2da07a0ba16200d379274a8d9a47516110518241e", - "bytes": 7490 + "sha256": "ec086f9dc6560066e7fe21a6052b0ec435ef9fcbd19f3a231a2d8cb07e1bf0e0", + "bytes": 7482 }, { "path": "backend/tests/test_privatemuse_deployment_recovery.py", @@ -1119,8 +1179,8 @@ }, { "path": "backend/tests/test_privatemuse_webhook_repair.py", - "sha256": "2c6ba79e8cd3d63faf6103a8578d9cb8e9ecb7278008e9ad36037cc08a02c2a7", - "bytes": 6391 + "sha256": "aa04a057b29302ae585751a5d4c5cdbb268e55350b6b6d20575c78808492a778", + "bytes": 6392 }, { "path": "backend/tests/test_production_runtime.py", @@ -1204,8 +1264,8 @@ }, { "path": "backend/tests/test_repopulse_target_attestation.py", - "sha256": "6c0c61262aceb58b924edab70bfa079e4b46b222897e00c72e39f7a73bd5c9e6", - "bytes": 11564 + "sha256": "1fccd197fbb8abe3c13187401b16ed21a44cc0f0137966f6427678a77a4d81f3", + "bytes": 14015 }, { "path": "backend/tests/test_repopulse_target_credential.py", @@ -1232,20 +1292,35 @@ "sha256": "7ea8409b3129938fd57390db26ba2e6068af90cf46db31204a0f0cd567c254ed", "bytes": 3242 }, + { + "path": "backend/tests/test_runner_admission_controller.py", + "sha256": "dcfb8e7a1d1ee86d6bfadef71bbd8be629a7f70a26baafb37bb134c61e78491d", + "bytes": 57459 + }, + { + "path": "backend/tests/test_runner_admission_policy.py", + "sha256": "e4bdf5b454b5e17d1785b2db82819c1e43150fe84de1bf0fd242288352f4ee26", + "bytes": 5951 + }, { "path": "backend/tests/test_runner_capacity_controller.py", - "sha256": "d757a661f06507ec7c685103de274c0dd26a987def676c70dd8b44be6e9c265d", - "bytes": 6909 + "sha256": "fbc1025db04ab98f5aaeee7c94bb6b5dfb1a5652bfe696d92f2ac5f631620448", + "bytes": 18051 }, { "path": "backend/tests/test_runner_capacity_policy.py", "sha256": "662bc7a4a9f0ec4cd721e032068e08f4530409aa2fc03636a04077d4d9b5e6f3", "bytes": 5292 }, + { + "path": "backend/tests/test_runner_controller_artifact.py", + "sha256": "ec3a53cebab1fae3a68817798d72b2831f35eda4c46fbb3d07ad84fc5eb55ab3", + "bytes": 2830 + }, { "path": "backend/tests/test_runner_fleet.py", - "sha256": "a442a54748caf77f912d05b7ed9d68645db502d1a2a96cd1838501343bccdf90", - "bytes": 5966 + "sha256": "88bfd354ceb8b7f7fbbd68142ed8b0dd14133de5844c89cedea72cb3909781db", + "bytes": 8826 }, { "path": "backend/tests/test_runner_fleet_cutover_safety.py", @@ -1254,19 +1329,59 @@ }, { "path": "backend/tests/test_runner_fleet_scheduler_recovery.py", - "sha256": "567c59526c430c61c7823a984fb8c2a7eb17c5614e3e0a28a2b25b9af85fc55f", - "bytes": 34952 + "sha256": "cc2e5c2b1076df701e4124e62fc9173b0ea3badb148e99ea5d54f84f40619568", + "bytes": 50645 + }, + { + "path": "backend/tests/test_runner_fleet_specialize.py", + "sha256": "e2b142ed5656e118156d44baf6f1f8568ba25297ea9af13b817861ce3c4fd4da", + "bytes": 107149 }, { "path": "backend/tests/test_runner_fleet_v2_execution_surface.py", "sha256": "62dcc692085ea9af004c4618e0c770701396be02254daf06bd5b55f56b5119fa", "bytes": 2337 }, + { + "path": "backend/tests/test_runner_profiles.py", + "sha256": "45a8dba20d1e8d778dc90c20a997630037e2eaa1d462fa62f48671285aa82e0e", + "bytes": 3111 + }, + { + "path": "backend/tests/test_runner_resource_envelope.py", + "sha256": "a0e469bc980dbec76cedda9a1ae5011fc1444d49ffed4c067268a10f1905a5a1", + "bytes": 40504 + }, { "path": "backend/tests/test_runner_scheduler_liveness_recovery.py", "sha256": "86e9a1e336206506f48e149fedf21e79f9c7ad54771dea4f53ec039ac81548af", "bytes": 3824 }, + { + "path": "backend/tests/test_runner_specialization_controller_surface.py", + "sha256": "dc26476686237318953bf335865ad0901635044629826690af0e1d98d786612b", + "bytes": 9174 + }, + { + "path": "backend/tests/test_runner_statistics_binding.py", + "sha256": "2a684cc43097a0ac86270140a2f47273be23d33964ed6c75e4bb9e063a79b228", + "bytes": 16674 + }, + { + "path": "backend/tests/test_runner_statistics_mcp.py", + "sha256": "330d1ba91fb47d932a03b3657782e92087e6f93bd7b9fa3120e2d2e4f502e9f6", + "bytes": 12567 + }, + { + "path": "backend/tests/test_runner_statistics_surface.py", + "sha256": "a5786c51bf0aff6982b8491d4f3c5c81e1e249ad5aeb3398d720402b712b15bd", + "bytes": 11212 + }, + { + "path": "backend/tests/test_runner_task_image_producer.py", + "sha256": "c55f54bf58fa81649ddb31a8953abadf299c2fe32d1b4411dd59889981cc7376", + "bytes": 12112 + }, { "path": "backend/tests/test_secret_storage.py", "sha256": "3e02994542763f4f7b2137356f883c48a17dc9ddb0af0efcd218e0d1c9ac6dce", diff --git a/backend/app/connectors/gitea_actions.py b/backend/app/connectors/gitea_actions.py index c731af2..f592da9 100644 --- a/backend/app/connectors/gitea_actions.py +++ b/backend/app/connectors/gitea_actions.py @@ -217,7 +217,9 @@ class GiteaActionsConnector(BaseConnector): "order": "asc", }, ) - pending_payload = self._request( + # Keep pending inventory in the all-or-error fetch, but it is not + # runnable queue pressure and must not drive runner diagnostics. + self._request( "/api/v1/admin/actions/jobs", { "status": "pending", @@ -267,9 +269,10 @@ class GiteaActionsConnector(BaseConnector): ] queued_jobs = [ item - for payload in (queued_payload, pending_payload) for item in ( - payload.get("jobs") if isinstance(payload.get("jobs"), list) else [] + queued_payload.get("jobs") + if isinstance(queued_payload.get("jobs"), list) + else [] ) if isinstance(item, dict) ] @@ -462,8 +465,7 @@ class GiteaActionsConnector(BaseConnector): "offline": sum(item["status"] != "online" for item in runners), }, "queue": { - "queued": int(queued_payload.get("total_count", 0) or 0) - + int(pending_payload.get("total_count", 0) or 0), + "queued": int(queued_payload.get("total_count", 0) or 0), "inProgress": int( active_payload.get("total_count", len(active_jobs)) or 0 ), @@ -476,12 +478,6 @@ class GiteaActionsConnector(BaseConnector): if isinstance(queued_payload.get("jobs"), list) else [] ) - or int(pending_payload.get("total_count", 0) or 0) - > len( - pending_payload.get("jobs") - if isinstance(pending_payload.get("jobs"), list) - else [] - ) ), "blockedByRunnerLabels": sorted(set(blocked_labels))[:20], "staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20], diff --git a/backend/app/connectors/portainer.py b/backend/app/connectors/portainer.py index e6ab66f..94abe39 100644 --- a/backend/app/connectors/portainer.py +++ b/backend/app/connectors/portainer.py @@ -11,8 +11,22 @@ from ..core.config import settings from ..core.redaction import redact_message from .base import BaseConnector - _PORTAINER_DEFAULT_TIMEOUT_SECONDS = 8.0 +_CONTROLLER_OWNED_CI_CONTAINER_NAMES = frozenset({ + 'itworx-ci-checks', + 'itworx-ci-backend', + 'itworx-ci-frontend', + 'itworx-ci-security', + 'itworx-ci-browser', + 'itworx-ci-build', + 'itworx-security-runner', + 'itworx-validation-runner-2', + 'itworx-validation-runner-3', + 'itworx-docker-runner', + 'itworx-browser-runner', + 'itworx-validation-runner-burst-1', + 'gitea-runner-dind', +}) @dataclass @@ -213,6 +227,10 @@ class PortainerConnector(BaseConnector): if not selected: return {'status': 'not_found', 'reason': f'Container {target} not found'} + selected_name = str(selected.get('name', '')).strip().strip('/').lower() + if selected_name in _CONTROLLER_OWNED_CI_CONTAINER_NAMES: + return {'status': 'blocked', 'reason': 'controller_owned_ci_lifecycle'} + container_name = selected.get('id') or selected.get('name') if not container_name: return {'status': 'error', 'reason': 'Unable to resolve container identifier'} diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8522dc9..2bb0a57 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -2,6 +2,8 @@ from __future__ import annotations import os from dataclasses import dataclass +from pathlib import Path +import stat from typing import Any @@ -34,6 +36,38 @@ def _secret_value(name: str, file_name: str) -> str | None: return None +def _private_secret_file(path: str | None) -> str | None: + """Read a small owner-private regular secret file without following links.""" + if not path: + return None + descriptor: int | None = None + try: + source = Path(path) + before = source.lstat() + if not source.is_absolute() or source.is_symlink() or not stat.S_ISREG(before.st_mode) or before.st_size > 512: + return None + descriptor = os.open( + source, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0), + ) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or opened.st_size > 512: + return None + if os.name != "nt" and ( + opened.st_uid != os.geteuid() or stat.S_IMODE(opened.st_mode) & 0o077 + ): + return None + value = os.read(descriptor, 513) + if len(value) > 512: + return None + return value.decode("utf-8").strip() or None + except (OSError, UnicodeDecodeError): + return None + finally: + if descriptor is not None: + os.close(descriptor) + + def _env_list(name: str, default: list[str] | None = None) -> list[str]: value = os.getenv(name) if value is None: @@ -86,6 +120,7 @@ class Settings: mcp_token: str | None = None mcp_token_file: str | None = None mcp_allowed_hosts: list[str] | None = None + runner_statistics_token_file: str | None = None n8n_enabled: bool = False n8n_token: str | None = None n8n_token_file: str | None = None @@ -120,6 +155,11 @@ class Settings: self.mcp_token = token_file.read().strip() or None except OSError: self.mcp_token = None + default_runner_statistics_token_file = str(Path(self.data_dir) / "runner-statistics-token") + self.runner_statistics_token_file = _env_value( + "APPOPS_RUNNER_STATISTICS_TOKEN_FILE", + self.runner_statistics_token_file or default_runner_statistics_token_file, + ) allowed_hosts = _env_value("APPOPS_MCP_ALLOWED_HOSTS", "localhost:*,127.0.0.1:*,198.18.10.150:*") or "" self.mcp_allowed_hosts = [item.strip() for item in allowed_hosts.split(",") if item.strip()] self.n8n_enabled = _env_bool("APPOPS_ENABLE_N8N", self.n8n_enabled) @@ -267,6 +307,10 @@ class Settings: "APPOPS_VERIFY_TLS": self.verify_tls, } + def read_runner_statistics_token(self) -> str | None: + """Read the fixed private credential at request time for no-restart provisioning.""" + return _private_secret_file(self.runner_statistics_token_file) + def redact_env_preview(self) -> dict[str, Any]: return { "APPOPS_BIND_HOST": self.bind_host, @@ -291,6 +335,8 @@ class Settings: "APPOPS_MCP_TOKEN_FILE": bool(self.mcp_token_file), "APPOPS_MCP_TOKEN_CONFIGURED": bool(self.mcp_token), "APPOPS_MCP_ALLOWED_HOSTS": list(self.mcp_allowed_hosts or []), + "APPOPS_RUNNER_STATISTICS_TOKEN_FILE": bool(self.runner_statistics_token_file), + "APPOPS_RUNNER_STATISTICS_TOKEN_CONFIGURED": bool(self.read_runner_statistics_token()), "APPOPS_ENABLE_N8N": self.n8n_enabled, "APPOPS_N8N_TOKEN_FILE": bool(self.n8n_token_file), "APPOPS_N8N_TOKEN_CONFIGURED": bool(self.n8n_token), diff --git a/backend/app/core/security.py b/backend/app/core/security.py index 2da7937..5415fb5 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -53,6 +53,15 @@ def require_service_request(request: Request, service: str) -> str: enabled, expected = settings.mcp_enabled, settings.mcp_token elif service == "n8n": enabled, expected = settings.n8n_enabled, settings.n8n_token + elif service == "mcp_runner_statistics": + expected = settings.read_runner_statistics_token() + if not expected: + raise HTTPException(404, "runner statistics MCP integration is disabled") + if not token_is_strong(expected): + raise HTTPException(503, "runner statistics MCP authentication is not safely configured") + if settings.mcp_token and hmac.compare_digest(expected, settings.mcp_token): + raise HTTPException(503, "runner statistics MCP token must be separate from the governed MCP token") + enabled = True elif service == "voice": enabled, expected = True, settings.voice_token or os.getenv("APP_OPS_VOICE_TOKEN") else: diff --git a/backend/app/main.py b/backend/app/main.py index 411d1a8..6a67478 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -73,6 +73,7 @@ from .mcp_server import ( restore_tool_metrics, set_session_running, ) +from .runner_statistics_mcp import runner_statistics_mcp_http_app, run_runner_statistics_mcp from .services import knowledge_service from .services.discovery_scheduler import run_periodic_discovery from .services.maintenance_scheduler import ( @@ -130,6 +131,7 @@ async def lifespan(_: FastAPI): if settings.mcp_enabled: await stack.enter_async_context(appops_mcp.session_manager.run()) set_session_running(True) + await stack.enter_async_context(run_runner_statistics_mcp()) try: yield finally: @@ -182,6 +184,7 @@ class MachineBodyLimitMiddleware: method = str(scope.get("method", "GET")).upper() is_machine_post = method not in _SAFE_METHODS and ( path == "/mcp" or path.startswith("/mcp/") + or path == "/mcp-runner-statistics" or path.startswith("/mcp-runner-statistics/") or path == "/api/integrations/n8n" or path.startswith("/api/integrations/n8n/") ) if scope["type"] != "http" or not is_machine_post: @@ -243,17 +246,23 @@ async def private_operator_boundary(request, call_next): is_api = path == "/api" or path.startswith("/api/") is_voice = path.startswith("/api/voice/") is_mcp = path == "/mcp" or path.startswith("/mcp/") + is_runner_statistics_mcp = path == "/mcp-runner-statistics" or path.startswith("/mcp-runner-statistics/") is_n8n = path == "/api/integrations/n8n" or path.startswith("/api/integrations/n8n/") is_public_auth = request.method in _SAFE_METHODS and ( path in {"/api/auth/session", "/api/auth/providers", "/api/health"} or path.startswith(("/api/auth/login/", "/api/auth/callback/")) ) - if is_mcp or is_n8n: + if is_runner_statistics_mcp: + try: + require_service_request(request, "mcp_runner_statistics") + except HTTPException as exc: + return finalize_response(JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})) + elif is_mcp or is_n8n: try: require_service_request(request, "mcp" if is_mcp else "n8n") except HTTPException as exc: return finalize_response(JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})) - if is_mcp or is_n8n: + if is_mcp or is_runner_statistics_mcp or is_n8n: content_length = request.headers.get("content-length") if content_length and content_length.isdigit() and int(content_length) > 1_048_576: return finalize_response(JSONResponse(status_code=413, content={"detail": "Request body exceeds 1 MiB limit"})) @@ -297,7 +306,7 @@ async def private_operator_boundary(request, call_next): response = await call_next(request) finally: actor_identity.reset(actor_token) - if is_api: + if is_api or is_runner_statistics_mcp: response.headers.setdefault("Cache-Control", "no-store") return finalize_response(response) @@ -356,6 +365,7 @@ app.include_router(integrations_router) app.include_router(n8n_router) app.include_router(knowledge_router) app.mount("/mcp", mcp_http_app) +app.mount("/mcp-runner-statistics", runner_statistics_mcp_http_app) @app.get('/api') diff --git a/backend/app/mcp_server.py b/backend/app/mcp_server.py index 06e22e1..0557d12 100644 --- a/backend/app/mcp_server.py +++ b/backend/app/mcp_server.py @@ -46,6 +46,7 @@ from .services.privatemuse_artifact_installer import ArtifactInstallError from .services.privatemuse_artifact_installer import install as install_artifact_file from .services.privatemuse_artifact_installer import preflight as artifact_preflight from .services.runtime_service import runtime_info +from .services.runner_statistics import RunnerStatisticsService _SESSION_RUNNING = False _SESSION_VERIFIED_AT: str | None = None @@ -233,6 +234,13 @@ def gitea_actions_runners_status() -> dict[str, Any]: return _json(GiteaActionsConnector().snapshot()) +@appops_mcp.tool(name="gitea.actions.runners.statistics", annotations=_READ_ONLY_TOOL) +@_tracked("gitea.actions.runners.statistics") +def gitea_actions_runner_statistics() -> dict[str, Any]: + """Read bounded CI profile budgets, measured usage and job statistics.""" + return _json(RunnerStatisticsService().snapshot()) + + def _json(value: Any) -> Any: if hasattr(value, "model_dump"): return _json(value.model_dump(mode="json")) diff --git a/backend/app/runner_statistics_mcp.py b/backend/app/runner_statistics_mcp.py new file mode 100644 index 0000000..5c06daf --- /dev/null +++ b/backend/app/runner_statistics_mcp.py @@ -0,0 +1,164 @@ +"""Dedicated one-tool MCP surface for bounded, read-only CI statistics.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from contextlib import asynccontextmanager +import ipaddress +import re +from typing import Any +from urllib.parse import urlsplit + +from mcp.server import MCPServer +from mcp.server.transport_security import TransportSecuritySettings +from mcp.types import ToolAnnotations +from starlette.applications import Starlette +from starlette.responses import Response +from starlette.types import ASGIApp, Receive, Scope, Send + +from .core.config import settings +from .core.version import APP_VERSION +from .services.runner_statistics import RunnerStatisticsService, sanitize_statistics, unavailable_statistics + +RUNNER_STATISTICS_TOOL = "gitea.actions.runners.statistics" +_READ_ONLY_TOOL = ToolAnnotations( + read_only_hint=True, + destructive_hint=False, + idempotent_hint=True, + open_world_hint=False, +) +_DNS_HOST_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)(?:\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?))*$") + + +def _validated_auth_public_host() -> tuple[str, int | None] | None: + try: + parsed = urlsplit(settings.auth_public_url) + hostname = parsed.hostname + port = parsed.port + except ValueError: + return None + if ( + parsed.scheme.lower() != "https" + or not hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or (port is not None and not 1 <= port <= 65535) + ): + return None + try: + normalized = hostname.rstrip(".").encode("idna").decode("ascii").lower() + except UnicodeError: + return None + if not normalized: + return None + if ":" in normalized: + try: + ipaddress.IPv6Address(normalized) + except ValueError: + return None + elif len(normalized) > 253 or not _DNS_HOST_PATTERN.fullmatch(normalized): + return None + return normalized, port + + +def _readonly_allowed_hosts() -> list[str]: + """Add only the validated public auth host to this read-only transport.""" + hosts = list(settings.mcp_allowed_hosts or []) + validated = _validated_auth_public_host() + if validated is None: + return hosts + normalized, port = validated + host_value = f"[{normalized}]" if ":" in normalized else normalized + candidates = [f"{host_value}:{port}" if port is not None else host_value] + for candidate in candidates: + if candidate not in hosts: + hosts.append(candidate) + return hosts + + +def _readonly_allowed_origins() -> list[str]: + """Keep existing frontend origin and admit only the validated auth origin.""" + origins = [settings.frontend_origin] if settings.frontend_origin else [] + validated = _validated_auth_public_host() + if validated is None: + return origins + normalized, port = validated + host_value = f"[{normalized}]" if ":" in normalized else normalized + netloc = f"{host_value}:{port}" if port is not None else host_value + origin = f"https://{netloc}" + if origin not in origins: + origins.append(origin) + return origins + + +def create_runner_statistics_mcp( + *, + snapshot_provider: Callable[[], object] | None = None, + allowed_hosts: Sequence[str] | None = None, + allowed_origins: Sequence[str] | None = None, +) -> tuple[MCPServer, Starlette]: + """Build an isolated MCP server exposing only the sanitized statistics tool.""" + provider = snapshot_provider or RunnerStatisticsService().snapshot + server = MCPServer( + "AppOps CI Runner Statistics", + version=APP_VERSION, + instructions=( + "Read-only CI runner profile statistics. This server exposes no operational, " + "administrative, deployment, filesystem, or mutation tools." + ), + ) + + @server.tool(name=RUNNER_STATISTICS_TOOL, annotations=_READ_ONLY_TOOL) + def gitea_actions_runner_statistics() -> dict[str, Any]: + """Read bounded runner budgets, measured usage, and job statistics.""" + try: + return sanitize_statistics(provider()) + except Exception: # noqa: BLE001 - keep internal broker details off the MCP surface. + return unavailable_statistics() + + transport = server.streamable_http_app( + streamable_http_path="/", + stateless_http=True, + json_response=True, + max_request_body_size=1_048_576, + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=_readonly_allowed_hosts() if allowed_hosts is None else list(allowed_hosts), + allowed_origins=_readonly_allowed_origins() if allowed_origins is None else list(allowed_origins), + ), + ) + return server, transport + + +class _ReplaceableASGIApp: + """Keep the mounted path stable while giving each app lifespan a fresh manager.""" + + def __init__(self) -> None: + self._app: ASGIApp | None = None + + def replace(self, app: ASGIApp | None) -> None: + self._app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if self._app is None: + response = Response("Runner statistics MCP is unavailable", status_code=503) + await response(scope, receive, send) + return + await self._app(scope, receive, send) + + +runner_statistics_mcp_http_app = _ReplaceableASGIApp() + + +@asynccontextmanager +async def run_runner_statistics_mcp(): + """Run a fresh stateless transport manager for this FastAPI lifespan.""" + server, transport = create_runner_statistics_mcp() + runner_statistics_mcp_http_app.replace(transport) + try: + async with server.session_manager.run(): + yield + finally: + runner_statistics_mcp_http_app.replace(None) diff --git a/backend/app/services/action_execution.py b/backend/app/services/action_execution.py index 52733a6..246568f 100644 --- a/backend/app/services/action_execution.py +++ b/backend/app/services/action_execution.py @@ -53,6 +53,9 @@ APPROVAL_EXECUTOR_ACTIONS = { "container.restart", "controller.artifact.promote", "runner.fleet.upgrade_v2", + "runner.fleet.specialize", + "runner.statistics.configure", + "hub.migration.prepare_permissions", "deployment.targets.sync", "deployment.runnerless.provision", "deployment.target.deploy_workmesh", @@ -1648,18 +1651,27 @@ def _execute_runner_fleet_upgrade_v2( risk: str, reason: str, original_policy_decision: str, + specialized: bool = False, + statistics_configuration: bool = False, ) -> dict[str, Any]: from .deployment_broker import DeploymentBrokerClient - action = "runner.fleet.upgrade_v2" - target = "tower/runner-fleet-v2" + action = ("runner.statistics.configure" if statistics_configuration else + "runner.fleet.specialize" if specialized else "runner.fleet.upgrade_v2") + target = ("tower/projectbrain-runner-statistics" if statistics_configuration else + "tower/runner-fleet-specialized" if specialized else "tower/runner-fleet-v2") steps = [ "Validate exact fixed runner-fleet target", "Call HMAC-signed root controller endpoint", "Prove protected main and successful push validation", - "Execute exact-revision fleet-v2 migration script", + "Execute exact-revision specialized migration script" if specialized + else "Execute exact-revision fleet-v2 migration script", "Verify specialized runner topology", ] + if statistics_configuration: + steps = ["Verify fixed read-only statistics target", "Prove protected controller/runtime source", + "Lock production configuration", "Provision private independent credential with durable CAS", + "Configure ProjectBrain for its governed delivery"] job = create_job( session, action_id=action, @@ -1670,24 +1682,31 @@ def _execute_runner_fleet_upgrade_v2( steps=steps, ) start_job(session, job) - set_job_progress(session, job, 25, "Requesting fixed Tower runner-fleet v2 migration.") + set_job_progress(session, job, 25, f"Requesting {action} through the fixed controller endpoint.") try: - result = DeploymentBrokerClient().runner_fleet_upgrade_v2(reason=reason) + broker = DeploymentBrokerClient() + result = (broker.runner_statistics_configure(reason=reason) if statistics_configuration else + broker.runner_fleet_specialize(reason=reason) if specialized else broker.runner_fleet_upgrade_v2(reason=reason)) status = str(result.get("status") or "error") - success = status in {"upgraded", "already_applied"} + success = status in ({"configured", "already_configured"} if statistics_configuration else + {"specialized", "already_applied"} if specialized else {"upgraded", "already_applied"}) message = str( result.get("reason") or ( - "Tower runner-fleet v2 migration verified." - if success - else "Tower runner-fleet v2 migration was rejected or failed." + ("ProjectBrain read-only statistics configuration verified." if statistics_configuration else + "Tower runner specialization verified." if specialized + else "Tower runner-fleet v2 migration verified.") + if success else + ("ProjectBrain statistics configuration was rejected or failed." if statistics_configuration else + "Tower runner specialization was rejected or failed." if specialized + else "Tower runner-fleet v2 migration was rejected or failed.") ) ) except Exception as exc: result = {"status": "error"} status = "error" success = False - message = f"{type(exc).__name__}: runner fleet v2 migration failed" + message = f"{type(exc).__name__}: fixed controller operation failed" completed = complete_job( session, @@ -1711,6 +1730,7 @@ def _execute_runner_fleet_upgrade_v2( "status": status, "revision": result.get("revision"), "verifiedRunners": result.get("verifiedRunners"), + "projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"), }, redacted_metadata={ "executor": action, @@ -1732,6 +1752,253 @@ def _execute_runner_fleet_upgrade_v2( "controllerStatus": status, "revision": result.get("revision"), "verifiedRunners": result.get("verifiedRunners"), + "projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"), + } + + +def _execute_hub_migration_prepare_permissions( + session: Session, + *, + source: str, + risk: str, + reason: str, + original_policy_decision: str, +) -> dict[str, Any]: + from .deployment_broker import DeploymentBrokerClient + + action = "hub.migration.prepare_permissions" + target = "tower/itworx-mcp-hub-permissions" + state_file = "/mnt/cache/appdata/gitea-deploy-runners/state/itworx-mcp-hub.json" + fixed_targets = {state_file} + steps = [ + "Validate the exact fixed MCP Hub permission target", + "Authenticate the fixed root-controller request with HMAC and replay protection", + "Verify the running controller is the protected current main release with full-push validation", + "Reinspect the fixed Hub state file and shared deployment lock", + "Apply only the fixed state-file mode correction and verify its final metadata", + ] + job = create_job( + session, + action_id=action, + target=target, + source=source, + risk=risk, + policy_decision="allowed", + steps=steps, + ) + start_job(session, job) + set_job_progress(session, job, 25, "Requesting fixed MCP Hub permission preparation through the controller.") + try: + result = DeploymentBrokerClient().hub_migration_prepare_permissions(reason=reason) + status = str(result.get("status") or "blocked") + revision_value = result.get("revision") + revision = ( + revision_value + if isinstance(revision_value, str) + and len(revision_value) == 40 + and all(character in "0123456789abcdef" for character in revision_value) + else None + ) + + changed_value = result.get("changedTargets", []) + changed_targets = ( + changed_value + if isinstance(changed_value, list) + and all(isinstance(path, str) and path in fixed_targets for path in changed_value) + and len(changed_value) == len(set(changed_value)) + else None + ) + changed_details_value = result.get("changedTargetDetails", []) + target_names = { + state_file: "state_file", + } + changed_details = ( + changed_details_value + if isinstance(changed_details_value, list) + and all( + isinstance(item, dict) + and set(item) == {"name", "path", "observedMode", "sameIdentityAtPath"} + and isinstance(item.get("path"), str) + and item.get("path") in fixed_targets + and item.get("name") == target_names.get(item.get("path")) + and isinstance(item.get("observedMode"), str) + and len(item["observedMode"]) == 4 + and item["observedMode"].startswith("0") + and all(character in "01234567" for character in item["observedMode"]) + and isinstance(item.get("sameIdentityAtPath"), bool) + and changed_targets is not None + and item["path"] in changed_targets + for item in changed_details_value + ) + and len({item["path"] for item in changed_details_value}) == len(changed_details_value) + else None + ) + uncertain_value = result.get("uncertainTargets", []) + uncertain_targets = ( + uncertain_value + if isinstance(uncertain_value, list) + and all(isinstance(path, str) and path in fixed_targets for path in uncertain_value) + and len(uncertain_value) == len(set(uncertain_value)) + else None + ) + expected_verified = { + "state_file": { + "path": state_file, + "uid": 0, + "gid": 0, + "mode": "0600", + "aclPresent": False, + }, + } + verified_value = result.get("verifiedTargets") + verified_targets = None + if isinstance(verified_value, list) and len(verified_value) == len(expected_verified): + by_name = { + item.get("name"): item + for item in verified_value + if isinstance(item, dict) and isinstance(item.get("name"), str) + } + if set(by_name) == set(expected_verified) and all( + all(by_name[name].get(key) == value for key, value in expected.items()) + for name, expected in expected_verified.items() + ): + verified_targets = [ + {"name": name, **expected_verified[name]} + for name in ("state_file",) + ] + + has_controller_projection = all( + key in result + for key in ( + "revision", + "changedTargets", + "changedTargetDetails", + "uncertainTargets", + ) + ) + success = ( + status in {"prepared", "already_compliant"} + and has_controller_projection + and revision is not None + and changed_targets is not None + and changed_details is not None + and uncertain_targets == [] + and verified_targets is not None + and (bool(changed_targets) if status == "prepared" else not changed_targets) + ) + if success: + message = ( + "The fixed MCP Hub state-file permission was prepared and verified." + if status == "prepared" + else "The fixed MCP Hub state-file permission was already compliant and verified." + ) + elif ( + status == "partial" + and has_controller_projection + and revision is not None + and changed_targets is not None + and changed_details is not None + and uncertain_targets is not None + and (changed_targets or uncertain_targets) + and len(changed_details) == len(changed_targets) + and {item["path"] for item in changed_details} == set(changed_targets) + ): + message = "Permission preparation stopped partway; inspect the exact changed and uncertain fixed targets before retrying." + elif ( + status in { + "blocked", + "unauthorized", + "missing_config", + "missing_secret", + "runtime_identity_unavailable", + "controller_source_mismatch", + "preflight_rejected", + "configuration_unavailable", + "exact_source_unavailable", + "invalid_request", + "invalid_body_size", + "authentication_rejected", + "forbidden", + } + and changed_targets == [] + and changed_details == [] + and uncertain_targets == [] + ): + status = "blocked" + message = "The fixed MCP Hub permission request was rejected before any target change." + else: + status = "outcome_unknown" + changed_targets = [] + changed_details = [] + uncertain_targets = sorted(fixed_targets) + verified_targets = None + message = ( + "The controller request outcome is unknown; the fixed Hub state file may have changed. " + "Inspect it before retrying." + ) + except Exception: + result = {"status": "outcome_unknown"} + status = "outcome_unknown" + revision = None + changed_targets = [] + changed_details = [] + uncertain_targets = sorted(fixed_targets) + verified_targets = None + success = False + message = ( + "The controller request outcome is unknown; the fixed Hub state file may have changed. " + "Inspect it before retrying." + ) + + completed = complete_job( + session, + job, + status="succeeded" if success else "failed", + result_summary=message, + ) + audit = log_event( + session, + source="action", + identity=source, + action_id=action, + target=target, + target_type="mcp-hub-permissions", + risk=risk, + policy_decision="allowed", + reason=reason, + result="success" if success else "outcome_unknown" if status == "outcome_unknown" else "failed", + execution_plan={"steps": steps, "jobId": completed.id}, + after_state={ + "status": status, + "revision": revision, + "changedTargets": changed_targets, + "changedTargetDetails": changed_details, + "uncertainTargets": uncertain_targets, + "verifiedTargets": verified_targets, + }, + redacted_metadata={ + "executor": action, + "integration": "mcp" if source == "chatgpt-mcp" else "operator", + "originalPolicyDecision": original_policy_decision, + "targetSelectionCallerSelectable": False, + "pathOrCommandCallerSelectable": False, + }, + ) + return { + "success": success, + "status": "success" if success else status, + "result": "success" if success else "outcome_unknown" if status == "outcome_unknown" else "failed", + "message": message, + "jobId": completed.id, + "auditId": audit.id, + "policyDecision": "allowed", + "originalPolicyDecision": original_policy_decision, + "controllerStatus": status, + "revision": revision, + "changedTargets": changed_targets, + "changedTargetDetails": changed_details, + "uncertainTargets": uncertain_targets, + "verifiedTargets": verified_targets, } @@ -1809,6 +2076,24 @@ def execute_governed_action( original_policy_decision=original_policy_decision, ) + if action == "hub.migration.prepare_permissions": + if target.strip().lower() != "tower/itworx-mcp-hub-permissions": + return { + "success": False, + "status": "failed", + "result": "blocked", + "message": "MCP Hub permission preparation accepts only tower/itworx-mcp-hub-permissions.", + "policyDecision": "blocked", + "originalPolicyDecision": original_policy_decision, + } + return _execute_hub_migration_prepare_permissions( + session, + source=source, + risk=risk, + reason=reason, + original_policy_decision=original_policy_decision, + ) + if action == "runner.fleet.upgrade_v2": if target.strip().lower() != "tower/runner-fleet-v2": return { @@ -1827,6 +2112,28 @@ def execute_governed_action( original_policy_decision=original_policy_decision, ) + if action == "runner.statistics.configure": + if target.strip().lower() != "tower/projectbrain-runner-statistics": + return {"success": False, "status": "failed", "result": "blocked", + "message": "Statistics configuration accepts only tower/projectbrain-runner-statistics.", + "policyDecision": "blocked", "originalPolicyDecision": original_policy_decision} + return _execute_runner_fleet_upgrade_v2( + session, source=source, risk=risk, reason=reason, + original_policy_decision=original_policy_decision, statistics_configuration=True, + ) + + if action == "runner.fleet.specialize": + if target.strip().lower() != "tower/runner-fleet-specialized": + return { + "success": False, "status": "failed", "result": "blocked", + "message": "Runner specialization accepts only tower/runner-fleet-specialized.", + "policyDecision": "blocked", "originalPolicyDecision": original_policy_decision, + } + return _execute_runner_fleet_upgrade_v2( + session, source=source, risk=risk, reason=reason, + original_policy_decision=original_policy_decision, specialized=True, + ) + if action == "deployment.runnerless.provision": return _execute_runnerless_target_provisioning( session, @@ -2287,6 +2594,44 @@ def execute_approved_action(session: Session, approval: Approval) -> dict[str, A original_policy_decision="approval_required", ) + if approval.action == "runner.statistics.configure": + if approval.target.strip().lower() != "tower/projectbrain-runner-statistics": + return {"success": False, "status": "failed", "result": "blocked", + "message": "Statistics configuration accepts only tower/projectbrain-runner-statistics."} + return _execute_runner_fleet_upgrade_v2( + session, source="approved_operator", risk=approval.risk, + reason=approval.reason or f"Approved through {approval.id}.", + original_policy_decision="approval_required", statistics_configuration=True, + ) + + if approval.action == "runner.fleet.specialize": + if approval.target.strip().lower() != "tower/runner-fleet-specialized": + return { + "success": False, "status": "failed", "result": "blocked", + "message": "Runner specialization accepts only tower/runner-fleet-specialized.", + } + return _execute_runner_fleet_upgrade_v2( + session, source="approved_operator", risk=approval.risk, + reason=approval.reason or f"Approved through {approval.id}.", + original_policy_decision="approval_required", specialized=True, + ) + + if approval.action == "hub.migration.prepare_permissions": + if approval.target.strip().lower() != "tower/itworx-mcp-hub-permissions": + return { + "success": False, + "status": "failed", + "result": "blocked", + "message": "MCP Hub permission preparation accepts only tower/itworx-mcp-hub-permissions.", + } + return _execute_hub_migration_prepare_permissions( + session, + source="approved_operator", + risk=approval.risk, + reason=approval.reason or f"Approved through {approval.id}.", + original_policy_decision="approval_required", + ) + if approval.action in {"unraid.vm.start", "unraid.vm.resume", "unraid.vm.pause"}: from .unraid_vm_service import execute_approved_vm_action diff --git a/backend/app/services/action_registry.py b/backend/app/services/action_registry.py index 6babb87..c4900dd 100644 --- a/backend/app/services/action_registry.py +++ b/backend/app/services/action_registry.py @@ -7,8 +7,11 @@ ACTIONS = [ 'container.restart', 'container.stop', 'runner.fleet.upgrade_v2', + 'runner.fleet.specialize', + 'runner.statistics.configure', 'controller.artifact.promote', 'controller.managed-storage.initialize', + 'hub.migration.prepare_permissions', 'gateway.clear_cache', 'stack.status', 'stack.redeploy', @@ -214,8 +217,11 @@ ACTION_RISK = { 'container.restart': 'medium', 'container.stop': 'high', 'runner.fleet.upgrade_v2': 'high', + 'runner.fleet.specialize': 'high', + 'runner.statistics.configure': 'high', 'controller.artifact.promote': 'high', 'controller.managed-storage.initialize': 'high', + 'hub.migration.prepare_permissions': 'high', 'gateway.clear_cache': 'high', 'stack.status': 'low', 'stack.redeploy': 'high', @@ -529,8 +535,14 @@ def action_default_target(action: str) -> str: return 'tower/deployment-controller' if action == 'controller.managed-storage.initialize': return 'tower/deployment-controller' + if action == 'hub.migration.prepare_permissions': + return 'tower/itworx-mcp-hub-permissions' if action == 'runner.fleet.upgrade_v2': return 'tower/runner-fleet-v2' + if action == 'runner.fleet.specialize': + return 'tower/runner-fleet-specialized' + if action == 'runner.statistics.configure': + return 'tower/projectbrain-runner-statistics' if action == 'deployment.targets.sync': return 'jens/*' if action == 'deployment.target.deploy_workmesh': diff --git a/backend/app/services/controller_artifact_helper_spec.py b/backend/app/services/controller_artifact_helper_spec.py new file mode 100644 index 0000000..ea491d1 --- /dev/null +++ b/backend/app/services/controller_artifact_helper_spec.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import re +from typing import Any + +SOURCE_REPOSITORY = "jens/unraid-appops-gateway" +HELPER_NAME = "itworx-controller-artifact-promoter-modern" +CONTROLLER_VERSION = "2.1.14" +HELPER_LABEL = "itworx.controller.promoter" +_REVISION = re.compile(r"^[0-9a-f]{40}$") +_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$") +_CONTAINER_ID = re.compile(r"^[0-9a-f]{64}$") +_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$") + + +def helper_container_spec( + *, + revision: str, + image_id: str, + current_controller_id: str | None = None, + current_controller_revision: str | None = None, + current_controller_tree: str | None = None, + current_controller_version: str | None = None, +) -> dict[str, Any]: + if not _REVISION.fullmatch(revision): + raise ValueError("controller_promoter_revision_invalid") + if not _IMAGE_ID.fullmatch(image_id): + raise ValueError("controller_promoter_image_invalid") + + bootstrap_values = ( + current_controller_id, + current_controller_revision, + current_controller_tree, + current_controller_version, + ) + if any(value is not None for value in bootstrap_values): + if ( + not _CONTAINER_ID.fullmatch(current_controller_id or "") + or not _REVISION.fullmatch(current_controller_revision or "") + or not _REVISION.fullmatch(current_controller_tree or "") + or not _VERSION.fullmatch(current_controller_version or "") + ): + raise ValueError("controller_promoter_bootstrap_binding_invalid") + + command = f"""set -Eeuo pipefail +umask 077 +revision='{revision}' +version='{CONTROLLER_VERSION}' +image_id='{image_id}' +repo='/gitea-repositories/jens/unraid-appops-gateway.git' +[[ \"$(stat -c '%a:%u:%g' /tmp)\" == '1777:0:0' ]] +work=\"$(mktemp -d -p /tmp itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX)\" +[[ -d \"$work\" && ! -L \"$work\" && \"$(stat -c '%u:%a' \"$work\")\" == '0:700' ]] +cleanup() {{ rm -rf -- \"$work\"; }} +trap cleanup EXIT +command -v git >/dev/null +command -v docker >/dev/null +docker compose version >/dev/null +command -v python3 >/dev/null +command -v tar >/dev/null +command -v curl >/dev/null +git --git-dir=\"$repo\" cat-file -e \"$revision^{{commit}}\" +tree=\"$(git --git-dir=\"$repo\" rev-parse \"$revision^{{tree}}\")\" +git --git-dir=\"$repo\" archive --format=tar \"$revision\" \\ + ops/gitea-deploy-broker \\ + scripts/prepare_controller_artifact.py \\ + scripts/reconcile_controller_state.py \\ + scripts/prepare_controller_hub_release_volume.py \\ + scripts/vendor \\ + scripts/run_repopulse_adoption.py \ + scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\" +chmod 0600 \"$work/controller.tar\" +archive_sha=\"$(sha256sum \"$work/controller.tar\" | cut -d ' ' -f 1)\" +git --git-dir=\"$repo\" show \"$revision:scripts/upgrade_deployment_controller.sh\" > \"$work/upgrade.sh\" +chmod 0700 \"$work/upgrade.sh\" +bash \"$work/upgrade.sh\" \\ + --archive \"$work/controller.tar\" \\ + --archive-sha256 \"$archive_sha\" \\ + --source-repository '{SOURCE_REPOSITORY}' \\ + --source-revision \"$revision\" \\ + --source-tree \"$tree\" \\ + --controller-version \"$version\" \\ + --image-id \"$image_id\" \\ + --confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2 +""" + + labels = { + HELPER_LABEL: "true", + "itworx.source.repository": SOURCE_REPOSITORY, + "itworx.source.revision": revision, + "itworx.controller.version": CONTROLLER_VERSION, + } + if current_controller_id is not None: + labels.update( + { + "itworx.controller.bootstrap.target.id": current_controller_id, + "itworx.controller.bootstrap.target.image": image_id, + "itworx.controller.bootstrap.target.revision": current_controller_revision or "", + "itworx.controller.bootstrap.target.tree": current_controller_tree or "", + "itworx.controller.bootstrap.target.version": current_controller_version or "", + } + ) + + return { + "Image": image_id, + "User": "0:0", + "Cmd": ["/bin/bash", "-lc", command], + "Labels": labels, + "HostConfig": { + "Binds": [ + "/mnt/cache:/mnt/cache", + "/mnt/user:/mnt/user", + "/var/lib/docker:/var/lib/docker:ro", + "/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro", + "/var/run/docker.sock:/var/run/docker.sock", + ], + "NetworkMode": "host", + "Privileged": False, + "ReadonlyRootfs": True, + "CpuPeriod": 100000, + "CpuQuota": 100000, + "Memory": 1024 * 1024 * 1024, + "PidsLimit": 128, + "Tmpfs": {"/tmp": "rw,nosuid,nodev,size=268435456,mode=1777"}, + "SecurityOpt": ["no-new-privileges:true"], + "RestartPolicy": {"Name": "no"}, + }, + } diff --git a/backend/app/services/controller_artifact_promoter.py b/backend/app/services/controller_artifact_promoter.py index c3b3f27..92d9eb7 100644 --- a/backend/app/services/controller_artifact_promoter.py +++ b/backend/app/services/controller_artifact_promoter.py @@ -11,19 +11,22 @@ import httpx from ..connectors.portainer import PortainerConnector from ..core.config import settings from ..core.redaction import redact_message +from .controller_artifact_helper_spec import ( + CONTROLLER_VERSION, + HELPER_LABEL, + HELPER_NAME, + SOURCE_REPOSITORY as SOURCE_REPOSITORY, + _IMAGE_ID, + _REVISION, + helper_container_spec, +) from .deployment_broker import DeploymentBrokerClient -SOURCE_REPOSITORY = "jens/unraid-appops-gateway" GITEA_OWNER = "Jens" GITEA_REPOSITORY = "Unraid-AppOps-Gateway" REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)" -HELPER_NAME = "itworx-controller-artifact-promoter" CONTROLLER_CONTAINER = "gitea-deploy-control" -CONTROLLER_VERSION = "2.1.13" -HELPER_LABEL = "itworx.controller.promoter" _HELPER_TIMEOUT_SECONDS = 240 -_REVISION = re.compile(r"^[0-9a-f]{40}$") -_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$") _TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$") @@ -31,78 +34,6 @@ class ControllerArtifactPromotionError(RuntimeError): pass -def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]: - if not _REVISION.fullmatch(revision): - raise ValueError("controller_promoter_revision_invalid") - if not _IMAGE_ID.fullmatch(image_id): - raise ValueError("controller_promoter_image_invalid") - - command = f"""set -Eeuo pipefail -umask 077 -revision='{revision}' -version='{CONTROLLER_VERSION}' -repo='/gitea-repositories/jens/unraid-appops-gateway.git' -work="$(mktemp -d -p /tmp 'itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX')" -[[ -d "$work" && ! -L "$work" && "$(stat -c '%u:%a' "$work")" == '0:700' ]] -cleanup() {{ rm -rf -- "$work"; }} -trap cleanup EXIT -command -v git >/dev/null -command -v docker >/dev/null -docker compose version >/dev/null -command -v python3 >/dev/null -command -v tar >/dev/null -command -v curl >/dev/null -git --git-dir="$repo" cat-file -e "$revision^{{commit}}" -tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")" -git --git-dir="$repo" archive --format=tar "$revision" \ - ops/gitea-deploy-broker \ - scripts/prepare_controller_artifact.py \ - scripts/reconcile_controller_state.py \ - scripts/run_repopulse_adoption.py \ - scripts/prepare_controller_hub_release_volume.py \ - scripts/vendor \ - scripts/upgrade_deployment_controller.sh > "$work/controller.tar" -chmod 0600 "$work/controller.tar" -archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)" -git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh" -chmod 0700 "$work/upgrade.sh" -bash "$work/upgrade.sh" \ - --archive "$work/controller.tar" \ - --archive-sha256 "$archive_sha" \ - --source-repository '{SOURCE_REPOSITORY}' \ - --source-revision "$revision" \ - --source-tree "$tree" \ - --controller-version "$version" \ - --image-id '{image_id}' \ - --confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2 -""" - - return { - "Image": image_id, - "User": "0:0", - "Cmd": ["/bin/bash", "-lc", command], - "Labels": { - HELPER_LABEL: "true", - "itworx.source.repository": SOURCE_REPOSITORY, - "itworx.source.revision": revision, - "itworx.controller.version": CONTROLLER_VERSION, - }, - "HostConfig": { - "Binds": [ - "/mnt/cache:/mnt/cache", - "/mnt/user:/mnt/user", - "/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro", - "/var/lib/docker:/var/lib/docker:ro", - "/var/run/docker.sock:/var/run/docker.sock", - ], - "NetworkMode": "host", - "Privileged": False, - "SecurityOpt": ["no-new-privileges:true"], - "RestartPolicy": {"Name": "no"}, - }, - } - - class ControllerArtifactPromoter: def __init__( self, diff --git a/backend/app/services/deployment_broker.py b/backend/app/services/deployment_broker.py index c90cff4..7bd4ada 100644 --- a/backend/app/services/deployment_broker.py +++ b/backend/app/services/deployment_broker.py @@ -551,6 +551,27 @@ class DeploymentBrokerClient: payload={"reason": reason[:1000]}, ) + def runner_fleet_specialize(self, *, reason: str) -> dict[str, Any]: + """Request the fixed protected-source CI specialization migration.""" + return self._request( + "POST", "/v1/appops/runner-fleet/specialize", + secret=settings.mcp_token or "", payload={"reason": reason[:1000]}, + ) + + def runner_statistics_configure(self, *, reason: str) -> dict[str, Any]: + """Provision the fixed independent read-only ProjectBrain capability.""" + return self._request( + "POST", "/v1/appops/runner-statistics/configure", + secret=settings.mcp_token or "", payload={"reason": reason[:1000]}, + ) + + def hub_migration_prepare_permissions(self, *, reason: str) -> dict[str, Any]: + """Prepare only the fixed Hub backup and controller-state permissions.""" + return self._request( + "POST", "/v1/appops/hub-migration/prepare-permissions", + secret=settings.mcp_token or "", payload={"reason": reason[:1000]}, + ) + def runner_fleet_status(self) -> dict[str, Any]: """Read the controller's private, bounded CI sandbox projection. @@ -558,16 +579,25 @@ class DeploymentBrokerClient: derives the payload from fixed runner identities and its private DinD daemon; AppOps never receives a Docker socket or a cleanup selector. """ + return self._runner_fleet_read("/v1/runner-fleet") + + def runner_fleet_statistics(self) -> dict[str, Any]: + """Read the persisted profile projection without scheduling work.""" + return self._runner_fleet_read("/v1/runner-fleet/statistics", timeout_seconds=min(self.timeout_seconds, 5)) + + def _runner_fleet_read(self, path: str, *, timeout_seconds: float | None = None) -> dict[str, Any]: + if path not in {"/v1/runner-fleet", "/v1/runner-fleet/statistics"}: + return {"status": "unavailable", "reason": "runner_fleet_path_invalid"} if not self.is_configured(): return {"status": "missing_config", "reason": self.configuration_error()} try: with httpx.Client( - timeout=self.timeout_seconds, + timeout=self.timeout_seconds if timeout_seconds is None else timeout_seconds, verify=self._verify_option(), follow_redirects=False, ) as client, client.stream( "GET", - f"{self.base_url}/v1/runner-fleet", + f"{self.base_url}{path}", headers={"Accept": "application/json"}, ) as response: if response.status_code != 200: diff --git a/backend/app/services/deployment_target_registry.py b/backend/app/services/deployment_target_registry.py index 26815ee..cc1ceb1 100644 --- a/backend/app/services/deployment_target_registry.py +++ b/backend/app/services/deployment_target_registry.py @@ -32,8 +32,8 @@ import subprocess import sys import time -path = Path("/opt/gitea-deploy/projects.json") -host_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") +path = Path("/controller-config/projects.json") +host_path = Path("/controller-hub-releases/controller-config/projects.json") lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") repository, revision, branch, raw_project = sys.argv[1:5] project = json.loads(raw_project) diff --git a/backend/app/services/media_codec_dashboard_deployment_reconciliation.py b/backend/app/services/media_codec_dashboard_deployment_reconciliation.py index fa8380d..907e5a4 100644 --- a/backend/app/services/media_codec_dashboard_deployment_reconciliation.py +++ b/backend/app/services/media_codec_dashboard_deployment_reconciliation.py @@ -29,8 +29,8 @@ import sys import time repository, expected_container = sys.argv[1:3] -registry_path = Path("/opt/gitea-deploy/projects.json") -host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") +registry_path = Path("/controller-config/projects.json") +host_registry_path = Path("/controller-hub-releases/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "media-codec-dashboard.json" project_lock_path = state_root / "locks" / "media-codec-dashboard.lock" diff --git a/backend/app/services/portfolio_deployment_enrollment.py b/backend/app/services/portfolio_deployment_enrollment.py index be69085..e308fd5 100644 --- a/backend/app/services/portfolio_deployment_enrollment.py +++ b/backend/app/services/portfolio_deployment_enrollment.py @@ -50,7 +50,7 @@ from pathlib import Path import sys repository = sys.argv[1] -path = Path("/opt/gitea-deploy/projects.json") +path = Path("/controller-config/projects.json") try: projects = json.loads(path.read_text(encoding="utf-8")) except Exception: @@ -145,8 +145,8 @@ import subprocess import sys import time -registry_path = Path("/opt/gitea-deploy/projects.json") -host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") +registry_path = Path("/controller-config/projects.json") +host_registry_path = Path("/controller-hub-releases/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "portfolio.json" project_lock_path = state_root / "locks" / "portfolio.lock" diff --git a/backend/app/services/portfolio_deployment_recovery.py b/backend/app/services/portfolio_deployment_recovery.py index a893e76..c5f0508 100644 --- a/backend/app/services/portfolio_deployment_recovery.py +++ b/backend/app/services/portfolio_deployment_recovery.py @@ -46,7 +46,7 @@ if ( raise SystemExit(40) controller_path = Path("/opt/gitea-deploy/deploy.py") -registry_path = Path("/opt/gitea-deploy/projects.json") +registry_path = Path("/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "portfolio.json" project_lock_path = state_root / "locks" / "portfolio.lock" diff --git a/backend/app/services/privatemuse_deployment_enrollment.py b/backend/app/services/privatemuse_deployment_enrollment.py index 1b4ea93..39d50e1 100644 --- a/backend/app/services/privatemuse_deployment_enrollment.py +++ b/backend/app/services/privatemuse_deployment_enrollment.py @@ -46,8 +46,8 @@ import subprocess import sys import time -registry_path = Path("/opt/gitea-deploy/projects.json") -host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") +registry_path = Path("/controller-config/projects.json") +host_registry_path = Path("/controller-hub-releases/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "privatemuse.json" project_lock_path = state_root / "locks" / "privatemuse.lock" diff --git a/backend/app/services/privatemuse_deployment_recovery.py b/backend/app/services/privatemuse_deployment_recovery.py index 9c6905d..392da5a 100644 --- a/backend/app/services/privatemuse_deployment_recovery.py +++ b/backend/app/services/privatemuse_deployment_recovery.py @@ -64,7 +64,7 @@ if ( raise SystemExit(40) controller_path = Path("/opt/gitea-deploy/deploy.py") -registry_path = Path("/opt/gitea-deploy/projects.json") +registry_path = Path("/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "privatemuse.json" project_lock_path = state_root / "locks" / "privatemuse.lock" diff --git a/backend/app/services/privatemuse_webhook_repair.py b/backend/app/services/privatemuse_webhook_repair.py index 6a30b45..e515588 100644 --- a/backend/app/services/privatemuse_webhook_repair.py +++ b/backend/app/services/privatemuse_webhook_repair.py @@ -26,8 +26,8 @@ repository, endpoint = sys.argv[1:3] if repository != "jens/privatemuse" or endpoint != "http://198.19.0.1:1299/v1/gitea/push/jens/privatemuse": raise SystemExit(40) -projects_path = Path("/opt/gitea-deploy/projects.json") -secret_root = Path("/opt/gitea-deploy/webhook-secrets") +projects_path = Path("/controller-config/projects.json") +secret_root = Path("/controller-config/webhook-secrets") secret_path = secret_root / "jens__privatemuse" admin_token_path = Path("/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token") expected_validation = { diff --git a/backend/app/services/repopulse_adoption_preflight.py b/backend/app/services/repopulse_adoption_preflight.py index 07528da..d6eb462 100644 --- a/backend/app/services/repopulse_adoption_preflight.py +++ b/backend/app/services/repopulse_adoption_preflight.py @@ -75,8 +75,8 @@ CONTAINER = "repopulse" VOLUME = "repopulse_data" APPDATA_ROOT = Path("/srv/example/appdata/repopulse") SCAN_ROOT = Path("/mnt/cache/appdata/gitea-deploy-runners/sources") -REGISTRY = Path("/opt/gitea-deploy/projects.json") -HOST_REGISTRY = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") +REGISTRY = Path("/controller-config/projects.json") +HOST_REGISTRY = Path("/controller-hub-releases/controller-config/projects.json") STATE = Path("/mnt/cache/appdata/gitea-deploy-runners/state/repopulse.json") BARE_REPO = Path("/gitea-repositories/jens/repopulse.git") SHA = re.compile(r"^[0-9a-f]{40}$") diff --git a/backend/app/services/runner_statistics.py b/backend/app/services/runner_statistics.py new file mode 100644 index 0000000..7b9c2ac --- /dev/null +++ b/backend/app/services/runner_statistics.py @@ -0,0 +1,181 @@ +"""Bounded read-only CI statistics; absent measurements remain unknown.""" + +from __future__ import annotations + +import copy +import re +import threading +import time +from datetime import UTC, datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from .deployment_broker import DeploymentBrokerClient + + +class _StatisticsModel(BaseModel): + model_config = ConfigDict(extra="ignore", allow_inf_nan=False, strict=True) + + +class _Budget(_StatisticsModel): + cpuLimit: float | int | None = Field(default=None, ge=0, le=20) + memoryLimitBytes: int | None = Field(default=None, ge=0) + activeJobLimit: int | None = Field(default=None, ge=0, le=6) + heavyJobLimit: int | None = Field(default=None, ge=0, le=6) + reservedCpu: float | int | None = Field(default=None, ge=0) + reservedMemoryBytes: int | None = Field(default=None, ge=0) + activeJobs: int | None = Field(default=None, ge=0) + heavyJobs: int | None = Field(default=None, ge=0) + + +class _Host(_StatisticsModel): + cpuBusyPercent: float | int | None = Field(default=None, ge=0, le=100) + loadRatio: float | int | None = Field(default=None, ge=0) + availableMemoryBytes: int | None = Field(default=None, ge=0) + cacheFreeBytes: int | None = Field(default=None, ge=0) + + +class _Profile(_StatisticsModel): + id: Literal["checks", "backend", "frontend", "security", "browser", "build"] + displayName: str = Field(min_length=1, max_length=80) + labels: list[str] = Field(max_length=8) + runnerName: str = Field(min_length=1, max_length=80) + status: str = Field(min_length=1, max_length=80) + queueDepth: int | None = Field(default=None, ge=0) + activeJobs: int | None = Field(default=None, ge=0) + oldestQueuedSeconds: float | int | None = Field(default=None, ge=0) + reason: str | None = Field(default=None, max_length=120) + cpuLimit: float | int | None = Field(default=None, ge=0, le=8) + memoryLimitBytes: int | None = Field(default=None, ge=0) + cpuUsed: float | int | None = Field(default=None, ge=0) + memoryUsedBytes: int | None = Field(default=None, ge=0) + completedJobs: int = Field(default=0, ge=0) + p50WaitSeconds: float | int | None = Field(default=None, ge=0) + p95WaitSeconds: float | int | None = Field(default=None, ge=0) + p50DurationSeconds: float | int | None = Field(default=None, ge=0) + p95DurationSeconds: float | int | None = Field(default=None, ge=0) + lastDurationSeconds: float | int | None = Field(default=None, ge=0) + + +class _Window(_StatisticsModel): + startAt: str | None = Field(default=None, max_length=40) + endAt: str | None = Field(default=None, max_length=40) + completedJobs: int = Field(default=0, ge=0) + samples: int = Field(default=0, ge=0) + coverage: Literal["bounded", "complete", "unavailable"] = "unavailable" + + +class _History(_StatisticsModel): + at: str = Field(min_length=1, max_length=40) + activeJobs: int | None = Field(default=None, ge=0) + queuedJobs: int | None = Field(default=None, ge=0) + cpuUsed: float | int | None = Field(default=None, ge=0) + memoryUsedBytes: int | None = Field(default=None, ge=0) + + +class RunnerStatistics(_StatisticsModel): + schemaVersion: Literal[1] + status: Literal["healthy", "degraded", "unavailable", "not_configured"] + scope: Literal["ci-fleet"] + generatedAt: str = Field(min_length=1, max_length=40) + budget: _Budget + host: _Host + profiles: list[_Profile] = Field(max_length=6) + window: _Window + history: list[_History] = Field(max_length=288) + + +def _valid_timestamp(value: str) -> bool: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + return parsed.tzinfo is not None and 0 < parsed.timestamp() <= time.time() + 15 + + +def unavailable_statistics(*, configured: bool = True) -> dict[str, object]: + return RunnerStatistics( + schemaVersion=1, + status="unavailable" if configured else "not_configured", + scope="ci-fleet", + generatedAt=datetime.now(UTC).isoformat(), + budget=_Budget(), + host=_Host(), + profiles=[], + window=_Window(), + history=[], + ).model_dump(mode="json") + + +def sanitize_statistics(payload: object) -> dict[str, object]: + """Accept only the public contract, never raw paths, credentials or jobs.""" + try: + snapshot = RunnerStatistics.model_validate(payload) + except (ValidationError, TypeError, ValueError): + return unavailable_statistics() + timestamps = [snapshot.generatedAt, *(sample.at for sample in snapshot.history)] + timestamps.extend(value for value in (snapshot.window.startAt, snapshot.window.endAt) if value) + if any(not _valid_timestamp(value) for value in timestamps): + return unavailable_statistics() + if len({profile.id for profile in snapshot.profiles}) != len(snapshot.profiles): + return unavailable_statistics() + for profile in snapshot.profiles: + allowed_labels = {f"itworx-ci-{profile.id}"} + if profile.id in {"backend", "frontend"}: + allowed_labels.update({"linux-validation", "ubuntu-latest"}) + elif profile.id == "build": + allowed_labels.add("docker-validation") + if ( + profile.displayName != f"ITWorx CI ยท {profile.id.title()}" + or profile.runnerName != f"itworx-ci-{profile.id}" + or not profile.labels + or len(set(profile.labels)) != len(profile.labels) + or not set(profile.labels).issubset(allowed_labels) + or re.fullmatch(r"[a-z][a-z0-9_-]{0,79}", profile.status) is None + or (profile.reason is not None and re.fullmatch(r"[a-z][a-z0-9_-]{0,119}", profile.reason) is None) + ): + return unavailable_statistics() + sample_time = datetime.fromisoformat(snapshot.generatedAt.replace("Z", "+00:00")).timestamp() + history_times = [datetime.fromisoformat(sample.at.replace("Z", "+00:00")).timestamp() for sample in snapshot.history] + if history_times != sorted(set(history_times)) or any(at > sample_time + 15 for at in history_times): + return unavailable_statistics() + window = snapshot.window + if (window.startAt is None) != (window.endAt is None): + return unavailable_statistics() + if window.startAt is not None and window.endAt is not None: + start = datetime.fromisoformat(window.startAt.replace("Z", "+00:00")).timestamp() + end = datetime.fromisoformat(window.endAt.replace("Z", "+00:00")).timestamp() + if not start <= end <= sample_time + 15: + return unavailable_statistics() + elif window.coverage != "unavailable": + return unavailable_statistics() + if time.time() - sample_time > 90 and snapshot.status == "healthy": + snapshot.status = "degraded" + return snapshot.model_dump(mode="json") + + +_lock = threading.Lock() +_cached: dict[str, object] | None = None +_cached_at = 0.0 +_CACHE_SECONDS = 30.0 + + +class RunnerStatisticsService: + def snapshot(self) -> dict[str, object]: + global _cached, _cached_at + with _lock: + now = time.monotonic() + if _cached is not None and now - _cached_at < _CACHE_SECONDS: + return copy.deepcopy(_cached) + broker = DeploymentBrokerClient() + if not broker.is_configured(): + snapshot = unavailable_statistics(configured=False) + else: + try: + snapshot = sanitize_statistics(broker.runner_fleet_statistics()) + except Exception: + snapshot = unavailable_statistics() + _cached = snapshot + _cached_at = time.monotonic() + return copy.deepcopy(snapshot) diff --git a/backend/app/services/runnerless_target_provisioning.py b/backend/app/services/runnerless_target_provisioning.py index 40f6c83..2f8db8e 100644 --- a/backend/app/services/runnerless_target_provisioning.py +++ b/backend/app/services/runnerless_target_provisioning.py @@ -36,8 +36,8 @@ if ( ): raise SystemExit(40) -projects_path = Path("/opt/gitea-deploy/projects.json") -secret_root = Path("/opt/gitea-deploy/webhook-secrets") +projects_path = Path("/controller-hub-releases/controller-config/projects.json") +secret_root = Path("/controller-hub-releases/controller-config/webhook-secrets") admin_token_path = Path( "/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token" ) diff --git a/backend/app/services/workmesh_first_deployment.py b/backend/app/services/workmesh_first_deployment.py index 0bd324f..757063d 100644 --- a/backend/app/services/workmesh_first_deployment.py +++ b/backend/app/services/workmesh_first_deployment.py @@ -53,8 +53,8 @@ containers = [ ] ports = {1255, 1256, 1257, 1258} controller_root = Path("/opt/gitea-deploy") -registry_path = controller_root / "projects.json" -host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") +registry_path = Path("/controller-config/projects.json") +host_registry_path = Path("/controller-hub-releases/controller-config/projects.json") lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") env_dir = Path("/srv/example/appdata/workmesh") env_path = env_dir / ".env" diff --git a/backend/tests/test_appops_live_state_recovery.py b/backend/tests/test_appops_live_state_recovery.py index c3f91c3..3cfa20c 100644 --- a/backend/tests/test_appops_live_state_recovery.py +++ b/backend/tests/test_appops_live_state_recovery.py @@ -123,9 +123,14 @@ def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery): before = r.path.read_bytes() compose = (r.repo / "docker-compose.yml").read_bytes() result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) + read_state, read_raw, read_sha256 = r.m.read_appops_state_preimage(r.project) state = json.loads(r.path.read_bytes()) backup = r.path.parent / result["backupFile"] assert backup.read_bytes() == before + assert read_state == state + assert read_raw == r.path.read_bytes() + assert read_sha256 == hashlib.sha256(read_raw).hexdigest() + assert os.name == "nt" or (backup.stat().st_mode & 0o777) == 0o600 assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest() assert state["last_attempt"] == r.prior["last_attempt"] assert state["live_revision"] == OBSERVED diff --git a/backend/tests/test_appops_migration_bridge.py b/backend/tests/test_appops_migration_bridge.py new file mode 100644 index 0000000..66439a4 --- /dev/null +++ b/backend/tests/test_appops_migration_bridge.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import copy +import json +import subprocess + +import pytest + +from test_deploy_broker import ( + _appops_known_legacy_state, + _appops_rollback_intent_fixture, + deploy_module as deploy_module, +) + + +@pytest.fixture +def bridge(deploy_module, monkeypatch, tmp_path): + m = deploy_module + revision = "c" * 40 + state_root = tmp_path / "state" + state_root.mkdir(mode=0o700) + monkeypatch.setattr(m, "STATE_ROOT", state_root) + monkeypatch.setattr(m, "APPOPS_LEGACY_INTENT_ROOT", state_root / "appops-legacy") + monkeypatch.setattr(m, "APPOPS_LEGACY_INTENT_PATH", state_root / "appops-legacy" / "unraid-appops-gateway.json") + prior = _appops_known_legacy_state() + monkeypatch.setattr(m, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", m._canonical_json_sha256(prior["last_attempt"]["metadata"])) + monkeypatch.setattr(m, "APPOPS_LEGACY_PROJECTION_SHA256", m._canonical_json_sha256(m._appops_legacy_state_projection(prior))) + repository, metadata, source, green, backup, sample = _appops_rollback_intent_fixture(m, revision, "e" * 64) + project = {"slug": "unraid-appops-gateway", "compose_project": "unraid-appops-gateway", + "_repository_name": repository, "branch": "main", "gitea_repository_id": 9, + "containers": ["Unraid-AppOps-Gateway"], "compose_files": ["docker-compose.yml"], + "verify_oci_revision": True, "environment": {"APPOPS_DATA_PATH": str(m.APPOPS_DATA_ROOT)}} + runtime = {"containerId": m.APPOPS_LEGACY_OLD_CONTAINER_ID, "imageId": m.APPOPS_LEGACY_OLD_IMAGE_ID, + "imageRef": m.APPOPS_LEGACY_OLD_IMAGE_REF, "sourceRepository": repository} + artifacts = m._appops_legacy_state_override(prior, runtime)["live_artifacts"] + artifacts["compose_sha256"] = "" + monkeypatch.setattr(m, "require_host_root", lambda: None) + monkeypatch.setattr(m, "_appops_recovery_config_unchanged", lambda _p: None) + monkeypatch.setattr(m, "validate_commit", lambda _r, _b, rev: rev) + monkeypatch.setattr(m, "validate_recorded_receipt", lambda *_a: None) + monkeypatch.setattr(m, "_appops_recovery_identity", lambda *_a: runtime.copy()) + monkeypatch.setattr(m, "deployment_container_artifacts", lambda _p: copy.deepcopy(artifacts["containers"])) + monkeypatch.setattr(m, "deployment_artifacts", lambda _p: copy.deepcopy(artifacts)) + monkeypatch.setattr(m, "verify_appops_green_delivery", lambda rev: green if rev == revision else pytest.fail("wrong candidate")) + monkeypatch.setattr(m, "verify_appops_legacy_source", lambda *_a: source) + monkeypatch.setattr(m, "revalidate_deployment_candidate", lambda *_a: None) + proof = sample["migrationProof"]["runtime"] + monkeypatch.setattr(m, "appops_legacy_compose_model", lambda *_a: ({"name": project["compose_project"]}, proof["resolvedComposeSha256"], proof["resolvedEnvironmentSha256"])) + monkeypatch.setattr(m, "_appops_legacy_inspect_runtime", lambda *_a, **_k: ({}, {}, runtime, None)) + monkeypatch.setattr(m, "appops_legacy_runtime_hashes", lambda *_a: proof) + events = [] + monkeypatch.setattr(m, "capture_appops_data_backup", lambda *_a: events.append("backup") or copy.deepcopy(backup)) + monkeypatch.setattr(m, "verify_appops_data_backup", lambda *_a: None) + monkeypatch.setattr(m, "capture_compose_rollback", lambda *_a, **_k: events.append("snapshot") or copy.deepcopy(sample["rollbackSnapshot"])) + monkeypatch.setattr(m, "_appops_legacy_tag_snapshot", lambda *_a: events.append("tag")) + path = state_root / "unraid-appops-gateway.json" + path.write_text(json.dumps(prior), encoding="utf-8") + path.chmod(0o644) + return m, project, tmp_path, revision, metadata, path, events + + +def prepare_bridge(bridge): + m, project, repo, revision, metadata, _path, _events = bridge + state, raw, digest = m.read_appops_state_preimage(project) + return m.prepare_appops_legacy_migration(project, project["_repository_name"], repo, revision, metadata, state, raw, digest) + + +def test_reattested_state_prepares_durable_migration_and_completed_rollback_retry(bridge): + m, project, repo, revision, metadata, path, events = bridge + m.reattest_appops_live_state(project, repo, m.APPOPS_LEGACY_OLD_REVISION) + state, raw, digest = m.read_appops_state_preimage(project) + migration = prepare_bridge(bridge) + assert events == ["backup", "snapshot", "tag"] + intent, intent_hash = m.read_appops_legacy_intent() + assert intent["stateSha256"] == digest + assert intent["migrationProof"]["recovery"]["auditSha256"] == m._canonical_json_sha256(state["recovery"]) + assert m._appops_legacy_pending_state_is_bound(state, raw, digest, intent, revision, project, repo) + wrong_intent = copy.deepcopy(intent) + wrong_intent["migrationProof"]["recovery"]["auditSha256"] = "f" * 64 + assert not m._appops_legacy_pending_state_is_bound(state, raw, digest, wrong_intent, revision, project, repo) + + project["_compose_rollback"] = migration["snapshot"] + project["_rollback_restored_revision"] = m.APPOPS_LEGACY_OLD_REVISION + m.write_state(project["_repository_name"], project, m.APPOPS_LEGACY_OLD_REVISION, "success", + m.APPOPS_LEGACY_ROLLBACK_STATE_MESSAGE, live_legacy_provenance=True, + live_receipt={}, rollback_snapshot={}, attempt_revision=revision, + attempt_metadata={"operation": "rollback", "from_revision": revision}, expected_state_sha256=digest) + restored, restored_raw, restored_hash = m.read_appops_state_preimage(project) + assert m._appops_legacy_rollback_state_matches(restored, project, revision, intent) + tampered = copy.deepcopy(restored) + tampered["recovery"]["at_epoch"] += 1 + assert not m._appops_legacy_rollback_state_matches(tampered, project, revision, intent) + completed = m._appops_legacy_rebind_observed_state(intent, restored_hash) + completed["status"] = "rolled_back" + m.write_appops_legacy_intent(completed, expected_sha256=intent_hash) + assert m.appops_legacy_rollback_state_anchor(restored, restored_raw, completed, revision) + events.clear() + retried = prepare_bridge(bridge) + assert events == ["tag"] + assert retried["intent"]["migrationProof"] == intent["migrationProof"] + assert retried["intent"]["status"] == "deploying" + + +def test_candidate_crash_reconciles_the_reattested_preimage_to_a_real_receipt(bridge, monkeypatch): + m, project, repo, revision, metadata, path, _events = bridge + m.reattest_appops_live_state(project, repo, m.APPOPS_LEGACY_OLD_REVISION) + original, raw, digest = m.read_appops_state_preimage(project) + prepare_bridge(bridge) + receipt = {"repository": project["_repository_name"], "revision": revision, + "tree": metadata["tree"], "build": metadata} + monkeypatch.setattr(m, "verify_deployment", lambda *_a: receipt) + result = m.reconcile_appops_legacy_candidate( + project, project["_repository_name"], repo, revision, metadata, original, raw, digest, + ) + assert result == receipt + state = json.loads(path.read_bytes()) + assert state["live_revision"] == revision + assert state["live_receipt"] == receipt + assert state["live_legacy_provenance"] is False + assert state["recovery"] == original["recovery"] + assert m.read_appops_legacy_intent()[0]["status"] == "deployed" + + +@pytest.mark.parametrize("damage", ["backup-bytes", "backup-mode", "backup-link", "audit", "state"]) +def test_reattest_bridge_rejects_tampering_before_backup_or_docker(bridge, damage): + m, project, repo, _revision, _metadata, path, events = bridge + result = m.reattest_appops_live_state(project, repo, m.APPOPS_LEGACY_OLD_REVISION) + backup = path.parent / result["backupFile"] + if damage == "backup-bytes": + backup.write_bytes(b"{}") + elif damage == "backup-mode": + backup.chmod(0o644) + elif damage == "backup-link": + target = backup.with_suffix(".saved") + backup.rename(target) + backup.symlink_to(target) + else: + state = json.loads(path.read_bytes()) + if damage == "audit": + state["recovery"]["observed_image_id"] = "sha256:" + "f" * 64 + else: + state["live_artifacts"]["containers"]["Unraid-AppOps-Gateway"]["image_id"] = "sha256:" + "f" * 64 + path.write_text(json.dumps(state)) + with pytest.raises(m.AppOpsLegacyNoMutation): + prepare_bridge(bridge) + assert events == [] + assert not m.APPOPS_LEGACY_INTENT_PATH.exists() + + +def prior_record(m, revision): + semantic = m._canonical_json_sha256({"gitea_repository_id": 9, "ref": "refs/heads/main", "revision": revision}) + record = {"schema": "gitea-webhook-inbox.v2", "semantic_sha256": semantic, + "body_sha256": "a" * 64, "delivery": "12345678-1234-4234-8234-123456789012", + "gitea_repository_id": 9, "ref": "refs/heads/main", "repository": "jens/unraid-appops-gateway", + "revision": revision, "source": "gitea_webhook", "status": "failed", "reason": "deployment_failed", + "validation": {"validation_gate_version": 1, "required_contexts": ["Managed validation / full (push)"], + "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, + "last_result": "pass", "observed": [{"context": "Managed validation / full (push)", "status": "success"}]}, + "audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"}, + {"stage": "deployment_failed", "reason": "deployment_failed"}]} + inbox = m.STATE_ROOT / "webhook-inbox" + inbox.mkdir(mode=0o700) + marker = inbox / f"{semantic}.json" + marker.write_text(json.dumps(record)) + marker.chmod(0o600) + return marker, record + + +@pytest.mark.parametrize("damage", [None, "deployed", "pending", "missing", "metadata", "ancestry"]) +def test_prior_main_attempt_requires_private_green_failure_and_git_proof(bridge, monkeypatch, damage): + m, _project, _repo, _revision, metadata, path, events = bridge + prior_revision = "b" * 40 + state = json.loads(path.read_bytes()) + prior_metadata = {**metadata, "revision": prior_revision, "id": prior_revision[:12]} + state["last_attempt"].update(revision=prior_revision, metadata=prior_metadata) + path.write_text(json.dumps(state)) + marker, record = prior_record(m, prior_revision) + monkeypatch.setattr(m, "deployment_metadata", lambda *_a, **_k: prior_metadata if damage != "metadata" else {**prior_metadata, "tree": "e" * 40}) + def ancestry(command, **_kwargs): + assert command[-4:] == ["merge-base", "--is-ancestor", prior_revision, metadata["revision"]] + if damage == "ancestry": + raise subprocess.CalledProcessError(1, command) + monkeypatch.setattr(m.subprocess, "run", ancestry) + if damage == "deployed": + record["status"] = "deployed" # Retain the old failure audit deliberately. + elif damage == "pending": + record["validation"]["observed"][0]["status"] = "pending" + marker.write_text(json.dumps(record)) + if damage == "missing": + marker.unlink() + if damage is not None: + with pytest.raises(m.AppOpsLegacyNoMutation): + prepare_bridge(bridge) + assert events == [] + else: + migration = prepare_bridge(bridge) + assert migration["intent"]["migrationProof"]["priorAttempt"]["metadataSha256"] == m._canonical_json_sha256(prior_metadata) + assert events == ["backup", "snapshot", "tag"] diff --git a/backend/tests/test_appops_self_upgrade_controller.py b/backend/tests/test_appops_self_upgrade_controller.py index 498299e..3df871b 100644 --- a/backend/tests/test_appops_self_upgrade_controller.py +++ b/backend/tests/test_appops_self_upgrade_controller.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib import importlib.util import json +import os import subprocess import sys import tarfile @@ -53,7 +54,10 @@ def _project(): "containers": ["Unraid-AppOps-Gateway"], "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], - "environment": {"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216"}, + "environment": { + "APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216", + "APPOPS_PUBLISH_HOST": "198.18.10.150", + }, "verify_oci_revision": True, "validation": { "validation_gate_version": 1, "required": True, @@ -71,6 +75,9 @@ def _ready(controller, monkeypatch): monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", [])) monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: { "revision": "a" * 40, "imageId": "sha256:" + "b" * 64, + "containerId": "c" * 64, + "imageRef": "unraid-appops-gateway:production", + "sourceTree": "d" * 40, "dataPath": "/srv/example/appdata/unraid-appops-gateway-1216", }) monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3)) @@ -150,24 +157,65 @@ def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controll def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch): + monkeypatch.setattr(controller, "_gitea_validation_settings", lambda: ("https://gitea.example.com/api/v1", b"t" * 32)) payload = [{ "State": {"Running": True, "Health": {"Status": "healthy"}}, - "Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}], - "Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40, - "org.opencontainers.image.source": "jens/unraid-appops-gateway"}}, - "NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}}, + "Id": "1" * 64, + "Mounts": [ + {"Type": "bind", "Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}, + {"Type": "bind", "Destination": "/mnt/privatemuse-comfyui-models", "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models", "RW": True}, + ], + "Config": { + "Image": "unraid-appops-gateway:platform-5553c04ce497", + "Hostname": "unraid-appops-gateway", + "Labels": { + "com.docker.compose.project": "unraid-appops-gateway", + "com.docker.compose.service": "appops-gateway", + "com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml", + "org.opencontainers.image.revision": "5553c04ce497c08a35ed301052dc5e0b02e5d9b4", + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.source-tree": "1624e1748053f998d8427c8c8a4203277d22f13c", + "org.opencontainers.image.version": "1.8.0-platform-5553c04ce497", + "org.opencontainers.image.created": "2026-10-01T16:32:01Z", + }, + }, + "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, + "NetworkSettings": { + "Ports": {"1216/tcp": [{"HostIp": "198.18.10.150", "HostPort": "1216"}]}, + "Networks": {"itworx-mcp-hub-npm": {}, "unraid-appops-gateway_default": {}}, + }, "Image": "sha256:" + "b" * 64, }] + image = [{"Id": payload[0]["Image"], "Config": {"Labels": { + key: value for key, value in payload[0]["Config"]["Labels"].items() + if key.startswith("org.opencontainers.image.") + }}}] names = ["Unraid-AppOps-Gateway"] def fake_run(args, **kwargs): - return subprocess.CompletedProcess(args, 0, json.dumps(payload) if args[1] == "inspect" else "\n".join(names), "") + if args[1] == "inspect": + result = payload + elif args[1] == "image": + result = image + else: + result = "\n".join(names) + return subprocess.CompletedProcess(args, 0, json.dumps(result) if isinstance(result, list) else result, "") monkeypatch.setattr(controller.subprocess, "run", fake_run) - assert controller._self_upgrade_runtime()["revision"] == "a" * 40 + runtime = controller._self_upgrade_runtime() + assert runtime["revision"] == "5553c04ce497c08a35ed301052dc5e0b02e5d9b4" + assert runtime["imageRef"] == "unraid-appops-gateway:platform-5553c04ce497" + assert runtime["sourceTree"] == "1624e1748053f998d8427c8c8a4203277d22f13c" payload[0]["Mounts"][0]["Source"] = "/wrong" assert controller._self_upgrade_runtime() is None payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216" + payload[0]["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "127.0.0.1" + assert controller._self_upgrade_runtime() is None + payload[0]["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "198.18.10.150" + image[0]["Id"] = "sha256:" + "0" * 64 + assert controller._self_upgrade_runtime() is None + image[0]["Id"] = payload[0]["Image"] names.append("Unraid-AppOps-Gateway-copy") assert controller._self_upgrade_runtime() is None @@ -184,30 +232,13 @@ def test_runtime_requires_exact_container_mount_port_and_health(controller, monk ("http://gitea.example.com/Jens/Unraid-AppOps-Gateway", False), ], ) -def test_runtime_accepts_only_fixed_appops_oci_sources(controller, monkeypatch, source, accepted): +def test_source_validation_accepts_only_fixed_appops_oci_sources(controller, monkeypatch, source, accepted): monkeypatch.setattr( controller, "_gitea_validation_settings", lambda: ("https://gitea.example.com/api/v1", b"t" * 32), ) - payload = [{ - "State": {"Running": True, "Health": {"Status": "healthy"}}, - "Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}], - "Config": {"Labels": { - "org.opencontainers.image.revision": "a" * 40, - "org.opencontainers.image.source": source, - }}, - "NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}}, - "Image": "sha256:" + "b" * 64, - }] - - def fake_run(args, **kwargs): - stdout = json.dumps(payload) if args[1] == "inspect" else "Unraid-AppOps-Gateway" - return subprocess.CompletedProcess(args, 0, stdout, "") - - monkeypatch.setattr(controller.subprocess, "run", fake_run) - - assert (controller._self_upgrade_runtime() is not None) is accepted + assert controller._self_upgrade_source_is_valid(source) is accepted def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch): @@ -219,12 +250,16 @@ def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch): mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345") +@pytest.mark.skipif(sys.platform == 'win32', reason='Linux controller pinned directory descriptors') def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path): secret = b"fixed-test-self-upgrade-capability-0123456789" token_file = tmp_path / "mcp-token" token_file.write_bytes(secret + b"\n") token_file.chmod(0o600) - monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file) + import runner_statistics_binding + monkeypatch.setattr(runner_statistics_binding, 'APPOPS_DATA', tmp_path) + monkeypatch.setattr(runner_statistics_binding, 'ADMIN_TOKEN_FILE', token_file) + monkeypatch.setattr(runner_statistics_binding, 'ROOT_UID', os.getuid()) monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces") stamp = str(int(time.time())) nonce = "a" * 32 @@ -245,6 +280,7 @@ def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, mo ) +@pytest.mark.skipif(sys.platform == 'win32', reason='Linux controller pinned directory descriptors') def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path): _ready(controller, monkeypatch) monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests") @@ -254,7 +290,10 @@ def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, token_file = tmp_path / "mcp-token" token_file.write_bytes(secret + b"\n") token_file.chmod(0o600) - monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file) + import runner_statistics_binding + monkeypatch.setattr(runner_statistics_binding, 'APPOPS_DATA', tmp_path) + monkeypatch.setattr(runner_statistics_binding, 'ADMIN_TOKEN_FILE', token_file) + monkeypatch.setattr(runner_statistics_binding, 'ROOT_UID', os.getuid()) monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None) server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -333,6 +372,9 @@ def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, ) monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: { "revision": "c" * 40, "imageId": "sha256:" + "b" * 64, + "containerId": "d" * 64, + "imageRef": "unraid-appops-gateway:production", + "sourceTree": "e" * 40, "dataPath": "/srv/example/appdata/unraid-appops-gateway-1216", }) monkeypatch.setattr(controller, "_deployment_state", lambda slug: {}) @@ -389,7 +431,10 @@ def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch, archive = Path(receipt["archive"]) assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"] assert receipt["previousRevision"] == "a" * 40 - assert json.loads((archive.parent / "receipt.json").read_text()) == receipt + stored_receipt = json.loads((archive.parent / "receipt.json").read_text()) + assert {**stored_receipt, "receiptSha256": hashlib.sha256( + json.dumps(stored_receipt, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest()} == receipt with tarfile.open(archive, "r:gz") as backup: assert backup.extractfile("./state.txt").read() == b"preserved" diff --git a/backend/tests/test_ci_job_docker.py b/backend/tests/test_ci_job_docker.py new file mode 100644 index 0000000..6add6eb --- /dev/null +++ b/backend/tests/test_ci_job_docker.py @@ -0,0 +1,550 @@ +"""Job-local Docker producer contract without starting a container daemon.""" +from __future__ import annotations + +import hashlib +import runpy +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +PROFILES = ("checks", "backend", "frontend", "security", "browser", "build") +BASE_LABELS = { + "io.buildah.version": "1.35.0", + "org.opencontainers.image.authors": "Gitea", + "org.opencontainers.image.created": "2026-10-01T00:01:57Z", + "org.opencontainers.image.description": "Official docker images used by Gitea runner to run workflows.", + "org.opencontainers.image.documentation": "https://gitea.com/gitea/runner-images/src/branch/main/README.md", + "org.opencontainers.image.licenses": "MIT", + "org.opencontainers.image.title": "Official Gitea Actions runner images", + "org.opencontainers.image.url": "https://gitea.com/gitea/runner-images", + "org.opencontainers.image.vendor": "Gitea", + "org.opencontainers.image.version": "v26.10.01", +} + + +def _job_local_producer( + tmp_path: Path, + *, + wrong_driver: bool = False, + wrong_security: bool = False, + job_local: bool = True, +): + repo = tmp_path / "repository" + repo.mkdir() + shutil.copytree(ROOT / "ops" / "runner-task-images", repo / "ops" / "runner-task-images") + scripts = repo / "scripts" + scripts.mkdir() + shutil.copyfile(ROOT / "scripts" / "normalize_ci_task_image_archive.py", scripts / "normalize_ci_task_image_archive.py") + code = repo / "ops" / "gitea-deploy-broker" + code.mkdir() + shutil.copyfile(ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py", code / "runner_profiles.py") + for args in ( + ["init", "-q"], + ["add", "ops", "scripts"], + ["-c", "user.name=Producer test", "-c", "user.email=producer-test@localhost", "commit", "-qm", "fixture"], + ): + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + docker = fake_bin / "docker" + docker.write_text( + f"#!{sys.executable}\n" + "import hashlib,io,json,os,sys,tarfile\n" + "from pathlib import Path\n" + "a=sys.argv[1:]; rows=Path(os.environ['FAKE_DOCKER_ROWS']); calls=Path(os.environ['FAKE_DOCKER_CALLS'])\n" + "history=json.loads(calls.read_text()) if calls.exists() else []\n" + "history.append({'argv':a,'host':os.environ.get('DOCKER_HOST'),'context':os.environ.get('DOCKER_CONTEXT'),'tls':os.environ.get('DOCKER_TLS_VERIFY'),'buildkit':os.environ.get('DOCKER_BUILDKIT')})\n" + "calls.write_text(json.dumps(history))\n" + "layerbuf=io.BytesIO()\n" + "with tarfile.open(fileobj=layerbuf,mode='w:') as t:\n" + " p=b'producer fixture layer'; x=tarfile.TarInfo('fixture.txt'); x.size=len(p); t.addfile(x,io.BytesIO(p))\n" + "layer=layerbuf.getvalue(); layer_sha=hashlib.sha256(layer).hexdigest()\n" + "if a[:1]==['info']:\n" + " fmt=a[a.index('--format')+1] if '--format' in a else ''\n" + f" driver={('vfs' if wrong_driver else 'overlay2')!r}\n" + " security='[\"name=seccomp,profile=builtin\",\"name=rootless\",\"name=cgroupns\"]'\n" + " if os.environ.get('FAKE_DOCKER_WRONG_SECURITY')=='1': security='[\"name=seccomp,profile=builtin\",\"name=rootless\",\"name=cgroupns\",\"name=unknown\"]'\n" + " if fmt=='{{.Driver}}|{{.CgroupDriver}}|{{.DockerRootDir}}|{{json .SecurityOptions}}': print(driver+'|none|/home/itworx-builder/.local/share/docker|'+security); sys.exit(0)\n" + " sys.exit(0)\n" + "if a[0]=='build':\n" + " builds=Path(os.environ['FAKE_DOCKER_BUILDS']); argv=json.loads(builds.read_text()) if builds.exists() else []\n" + " pairs=[a[i+1].split('=',1) for i,x in enumerate(a) if x=='--build-arg']\n" + " is_build=dict(pairs).get('PROFILE')=='build'\n" + " staged=Path(a[-1]).is_dir() and all((Path(a[-1])/n).is_file() for n in ['job_docker.py','itworx-ci-job-docker-probe','normalize_ci_task_image_archive.py']) if is_build else True\n" + " argv.append({'argv':a,'staged':staged}); builds.write_text(json.dumps(argv))\n" + " b=dict(pairs); profile=b['PROFILE']; ref=a[a.index('--tag')+1]\n" + f" labels={BASE_LABELS!r}.copy()\n" + " labels.update({'org.opencontainers.image.source':'https://gitea.example.com/Jens/Unraid-AppOps-Gateway','org.opencontainers.image.revision':b['SOURCE_REVISION'],'tech.itworx.source-tree':b['SOURCE_TREE'],'tech.itworx.ci.profile':profile})\n" + " config={'architecture':'amd64','os':'linux','config':{'Labels':labels},'rootfs':{'type':'layers','diff_ids':['sha256:'+layer_sha]}}\n" + " config_bytes=json.dumps(config,sort_keys=True,separators=(',',':')).encode(); image_id='sha256:'+hashlib.sha256(config_bytes).hexdigest()\n" + " current=json.loads(rows.read_text()) if rows.exists() else []\n" + " current.append({'Id':image_id,'Os':'linux','Architecture':'amd64','RepoTags':[ref],'Config':{'Labels':labels}})\n" + " Path(os.environ['FAKE_DOCKER_CONFIGS']+'/'+profile+'.json').write_bytes(config_bytes)\n" + " rows.write_text(json.dumps(current)); sys.exit(0)\n" + "if a[:2]==['image','inspect']: print(rows.read_text()); sys.exit(0)\n" + "if a[:2]==['image','save']:\n" + " selected=set(a[a.index('--output')+2:]); current=[r for r in json.loads(rows.read_text()) if r['RepoTags'][0] in selected]; items=[]; members={'blobs/sha256/'+layer_sha:layer}\n" + " for row in current:\n" + " profile=row['Config']['Labels']['tech.itworx.ci.profile']; config=Path(os.environ['FAKE_DOCKER_CONFIGS']+'/'+profile+'.json').read_bytes(); config_sha=hashlib.sha256(config).hexdigest(); config_path='blobs/sha256/'+config_sha; members[config_path]=config\n" + " items.append({'Config':config_path,'RepoTags':row['RepoTags'],'Layers':['blobs/sha256/'+layer_sha],'LayerSources':{'sha256:'+layer_sha:{'mediaType':'application/vnd.oci.image.layer.v1.tar','size':len(layer),'digest':'sha256:'+layer_sha}}})\n" + " members['manifest.json']=json.dumps(items,separators=(',',':')).encode(); members['oci-layout']=b'{\"imageLayoutVersion\":\"1.0.0\"}'; members['index.json']=b'{\"schemaVersion\":2,\"manifests\":[]}'\n" + " with tarfile.open(a[a.index('--output')+1],mode='w:') as t:\n" + " for name,payload in members.items():\n" + " x=tarfile.TarInfo(name); x.size=len(payload); t.addfile(x,io.BytesIO(payload))\n" + " sys.exit(0)\n" + "sys.exit('unexpected Docker invocation')\n", + encoding="utf-8", + ) + docker.chmod(0o755) + output = tmp_path / "output" + env = { + **os.environ, + "PATH": str(fake_bin) + os.pathsep + os.environ["PATH"], + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "Jens/Unraid-AppOps-Gateway", + "DOCKER_HOST": "tcp://shared-dind:2375", + "DOCKER_CONTEXT": "shared-context", + "DOCKER_TLS_VERIFY": "1", + "FAKE_DOCKER_ROWS": str(tmp_path / "docker-rows.json"), + "FAKE_DOCKER_CALLS": str(tmp_path / "docker-calls.json"), + "FAKE_DOCKER_BUILDS": str(tmp_path / "docker-builds.json"), + "FAKE_DOCKER_CONFIGS": str(tmp_path / "docker-configs"), + "FAKE_DOCKER_WRONG_SECURITY": "1" if wrong_security else "0", + } + Path(env["FAKE_DOCKER_CONFIGS"]).mkdir() + command = ["bash", str(ROOT / "scripts" / "build_ci_task_images.sh"), revision, "123", str(output)] + if job_local: + command.append("--job-local") + result = subprocess.run( + command, + cwd=repo, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], cwd=repo, text=True).strip() + return result, output, tmp_path, wrong_driver, revision, tree + + +def _oci_task_image_fixture( + tmp_path: Path, + *, + layer_media_type: str = "application/vnd.oci.image.layer.v1.tar", + profiles: tuple[str, ...] = PROFILES, +): + layer_buffer = io.BytesIO() + with tarfile.open(fileobj=layer_buffer, mode="w:") as layer_tar: + payload = b"task image rootfs fixture" + member = tarfile.TarInfo("fixture.txt") + member.size = len(payload) + layer_tar.addfile(member, io.BytesIO(payload)) + layer_bytes = layer_buffer.getvalue() + layer_sha = hashlib.sha256(layer_bytes).hexdigest() + labels_base = { + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.revision": "a" * 40, + "tech.itworx.source-tree": "b" * 40, + } + rows = [] + source_manifest = [] + members: dict[str, bytes] = {f"blobs/sha256/{layer_sha}": layer_bytes} + for profile in profiles: + labels = {**labels_base, "tech.itworx.ci.profile": profile} + config = { + "architecture": "amd64", + "os": "linux", + "config": {"Labels": labels}, + "rootfs": {"type": "layers", "diff_ids": [f"sha256:{layer_sha}"]}, + } + config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() + config_sha = hashlib.sha256(config_bytes).hexdigest() + config_path = f"blobs/sha256/{config_sha}" + members[config_path] = config_bytes + reference = f"localhost/itworx-ci-{profile}:sha-{'a' * 40}" + rows.append({ + "Id": f"sha256:{config_sha}", + "Os": "linux", + "Architecture": "amd64", + "RepoTags": [reference], + "Config": {"Labels": labels}, + }) + source_manifest.append({ + "Config": config_path, + "RepoTags": [reference], + "Layers": [f"blobs/sha256/{layer_sha}"], + "LayerSources": { + f"sha256:{layer_sha}": { + "mediaType": layer_media_type, + "size": len(layer_bytes), + "digest": f"sha256:{layer_sha}", + }, + }, + }) + members["manifest.json"] = json.dumps(source_manifest, separators=(",", ":")).encode() + members["oci-layout"] = b'{"imageLayoutVersion":"1.0.0"}' + members["index.json"] = b'{"schemaVersion":2,"manifests":[]}' + source = tmp_path / "source-oci.tar" + with tarfile.open(source, mode="w:") as archive: + for name, payload in members.items(): + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + inspection = tmp_path / "inspection.json" + inspection.write_text(json.dumps(rows), encoding="utf-8") + output = tmp_path / "normalized.tar" + return source, inspection, output, rows, layer_bytes + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_archive_normalizer_converts_uncompressed_oci_blobs_to_legacy_docker_save_layout(tmp_path): + source, inspection, output, rows, layer_bytes = _oci_task_image_fixture(tmp_path) + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "normalize_ci_task_image_archive.py"), str(source), str(output), str(inspection)], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + with tarfile.open(output, mode="r:") as archive: + members = {member.name: member for member in archive} + normalized = json.loads(archive.extractfile(members["manifest.json"]).read()) + assert len(normalized) == len(PROFILES) + assert all(set(item) == {"Config", "RepoTags", "Layers"} for item in normalized) + for item, expected in zip(normalized, rows, strict=True): + assert item["Config"] == expected["Id"].removeprefix("sha256:") + ".json" + assert item["RepoTags"] == expected["RepoTags"] + assert len(item["Layers"]) == 1 + assert item["Layers"][0].endswith("/layer.tar") + config_bytes = archive.extractfile(members[item["Config"]]).read() + assert "sha256:" + hashlib.sha256(config_bytes).hexdigest() == expected["Id"] + assert json.loads(config_bytes)["config"]["Labels"] == expected["Config"]["Labels"] + assert archive.extractfile(members[item["Layers"][0]]).read() == layer_bytes + assert not any(name.startswith("blobs/") for name in members) + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_archive_normalizer_rejects_compressed_oci_layers_before_emitting_output(tmp_path): + source, inspection, output, _rows, _layer_bytes = _oci_task_image_fixture( + tmp_path, + layer_media_type="application/vnd.oci.image.layer.v1.tar+gzip", + ) + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "normalize_ci_task_image_archive.py"), str(source), str(output), str(inspection)], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode != 0 + assert "compressed or unsupported layer media type" in result.stderr + assert not output.exists() + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_archive_normalizer_probe_mode_accepts_and_emits_one_build_image(tmp_path): + source, inspection, output, rows, _layer_bytes = _oci_task_image_fixture(tmp_path, profiles=("build",)) + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "normalize_ci_task_image_archive.py"), "--probe", str(source), str(output), str(inspection)], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + with tarfile.open(output, mode="r:") as archive: + manifest = json.loads(archive.extractfile("manifest.json").read()) + assert len(manifest) == 1 + assert set(manifest[0]) == {"Config", "RepoTags", "Layers"} + assert manifest[0]["RepoTags"] == rows[0]["RepoTags"] + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_job_local_mode_builds_all_profiles_through_private_rootless_engine_without_sibling_caps(tmp_path): + result, output, scratch, _wrong_driver, revision, tree = _job_local_producer(tmp_path) + assert result.returncode == 0, result.stderr + + calls = json.loads((scratch / "docker-calls.json").read_text()) + assert calls[0] == { + "argv": ["info", "--format", "{{.Driver}}|{{.CgroupDriver}}|{{.DockerRootDir}}|{{json .SecurityOptions}}"], + "host": "unix:///run/itworx-job-docker/docker.sock", + "context": None, + "tls": None, + "buildkit": "1", + } + builds = json.loads((scratch / "docker-builds.json").read_text()) + assert len(builds) == len(PROFILES) + assert all(row["staged"] for row in builds) + build_arguments = [row["argv"] for row in builds] + assert [next(args[i + 1].split("=", 1)[1] for i, arg in enumerate(args[:-1]) if arg == "--build-arg" and args[i + 1].startswith("PROFILE=")) for args in build_arguments] == list(PROFILES) + for args in build_arguments: + assert not any(arg.startswith(("--cpu-period", "--cpu-quota", "--memory", "--cgroup-parent", "--cpus")) for arg in args) + assert "--rm=true" in args and "--force-rm=true" in args + + manifest = json.loads((output / "task-image-manifest.json").read_text()) + assert manifest["apiVersion"] == "itworx.runner-task-images/v1" + assert manifest["sourceRevision"] == revision + assert manifest["sourceTree"] == tree + assert tuple(row["profile"] for row in manifest["images"]) == PROFILES + assert manifest["dockerTarSha256"] == hashlib.sha256((output / "task-images.tar").read_bytes()).hexdigest() + for row in manifest["images"]: + assert row["labels"] == { + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.revision": revision, + "tech.itworx.source-tree": tree, + "tech.itworx.ci.profile": row["profile"], + } + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_job_local_mode_rejects_non_overlay_or_non_rootless_server_before_build(tmp_path): + result, output, scratch, _wrong_driver, _revision, _tree = _job_local_producer(tmp_path, wrong_driver=True) + assert result.returncode != 0 + assert "job-local Docker daemon verification failed" in result.stderr + assert not (scratch / "docker-builds.json").exists() + assert not (output / "task-image-manifest.json").exists() + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_job_local_mode_rejects_unknown_daemon_security_options_before_build(tmp_path): + result, output, scratch, _wrong_driver, _revision, _tree = _job_local_producer(tmp_path, wrong_security=True) + assert result.returncode != 0 + assert "job-local Docker daemon verification failed" in result.stderr + assert not (scratch / "docker-builds.json").exists() + assert not (output / "task-image-manifest.json").exists() + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_bootstrap_mode_keeps_classic_shared_daemon_caps_and_archive_contract(tmp_path): + result, output, scratch, _wrong_driver, _revision, _tree = _job_local_producer(tmp_path, job_local=False) + assert result.returncode == 0, result.stderr + calls = json.loads((scratch / "docker-calls.json").read_text()) + assert calls[0]["argv"] == ["info"] + assert calls[0]["host"] == "tcp://shared-dind:2375" + assert calls[0]["buildkit"] == "0" + builds = json.loads((scratch / "docker-builds.json").read_text()) + assert len(builds) == len(PROFILES) + assert all(row["staged"] for row in builds) + build_arguments = [row["argv"] for row in builds] + assert all(any(arg.startswith("--cpu-quota=") for arg in args) for args in build_arguments) + assert all(any(arg.startswith("--memory=5120m") for arg in args) for args in build_arguments) + assert (output / "task-images.tar").is_file() + with tarfile.open(output / "task-images.tar", mode="r:") as archive: + manifest = json.loads(archive.extractfile("manifest.json").read()) + assert len(manifest) == len(PROFILES) + assert all(set(item) == {"Config", "RepoTags", "Layers"} for item in manifest) + + +def test_build_image_uses_only_minimal_file_capabilities_for_uidmap_helpers(): + dockerfile = (ROOT / "ops" / "runner-task-images" / "build" / "Dockerfile").read_text(encoding="utf-8") + pins = json.loads((ROOT / "ops" / "runner-task-images" / "pins.json").read_text(encoding="utf-8")) + assert pins["jobLocalDocker"]["uidmapFileCapabilities"] == { + "newuidmap": "cap_setuid=ep", + "newgidmap": "cap_setgid=ep", + "setuidBits": False, + } + assert '"$setcap_bin" cap_setuid=ep "$newuidmap_bin"' in dockerfile + assert '"$setcap_bin" cap_setgid=ep "$newgidmap_bin"' in dockerfile + assert 'test "$("$getcap_bin" "$newuidmap_bin")" = "$newuidmap_bin cap_setuid=ep"' in dockerfile + assert 'test "$("$getcap_bin" "$newgidmap_bin")" = "$newgidmap_bin cap_setgid=ep"' in dockerfile + assert '! test -u "$newuidmap_bin"' in dockerfile + assert '! test -u "$newgidmap_bin"' in dockerfile + + +def test_build_image_declares_exactly_one_private_daemon_data_volume(): + dockerfile = (ROOT / "ops" / "runner-task-images" / "build" / "Dockerfile").read_text(encoding="utf-8") + pins = json.loads((ROOT / "ops" / "runner-task-images" / "pins.json").read_text(encoding="utf-8")) + volumes = [line.strip() for line in dockerfile.splitlines() if line.strip().startswith("VOLUME")] + assert volumes == ['VOLUME ["/home/itworx-builder/.local/share/docker"]'] + assert pins["jobLocalDocker"]["dataRoot"] == "/home/itworx-builder/.local/share/docker" + assert pins["jobLocalDocker"]["anonymousDataVolume"] == volumes[0].split('"')[1] + + +@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only") +@pytest.mark.parametrize("mode", [0o700, 0o710]) +def test_job_docker_accepts_private_data_root_modes_used_before_and_after_daemon_start(tmp_path, monkeypatch, mode): + sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build")) + import job_docker + + runtime = tmp_path / "runtime" + data_root = tmp_path / "data" + runtime.mkdir(mode=0o700) + data_root.mkdir(mode=mode) + os.chmod(runtime, 0o700) + os.chmod(data_root, mode) + data_info = data_root.stat() + monkeypatch.setattr(job_docker, "BUILDER_UID", data_info.st_uid) + monkeypatch.setattr(job_docker, "BUILDER_GID", data_info.st_gid) + monkeypatch.setattr(job_docker, "RUNTIME", runtime) + monkeypatch.setattr(job_docker, "DATA_ROOT", data_root) + + job_docker._ensure_runtime() + + +@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only") +@pytest.mark.parametrize("mode", [0o711, 0o750, 0o777]) +def test_job_docker_rejects_unexpected_private_data_root_modes(tmp_path, monkeypatch, mode): + sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build")) + import job_docker + + runtime = tmp_path / "runtime" + data_root = tmp_path / "data" + runtime.mkdir(mode=0o700) + data_root.mkdir(mode=mode) + os.chmod(runtime, 0o700) + os.chmod(data_root, mode) + data_info = data_root.stat() + monkeypatch.setattr(job_docker, "BUILDER_UID", data_info.st_uid) + monkeypatch.setattr(job_docker, "BUILDER_GID", data_info.st_gid) + monkeypatch.setattr(job_docker, "RUNTIME", runtime) + monkeypatch.setattr(job_docker, "DATA_ROOT", data_root) + + with pytest.raises(job_docker.WrapperError, match="data directory ownership invalid"): + job_docker._ensure_runtime() + + +@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only") +def test_job_docker_rejects_symlink_private_data_root(tmp_path, monkeypatch): + sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build")) + import job_docker + + runtime = tmp_path / "runtime" + real_data_root = tmp_path / "real-data" + data_root_link = tmp_path / "data-link" + runtime.mkdir(mode=0o700) + real_data_root.mkdir(mode=0o700) + os.chmod(runtime, 0o700) + os.chmod(real_data_root, 0o700) + data_root_link.symlink_to(real_data_root, target_is_directory=True) + real_data_info = real_data_root.stat() + monkeypatch.setattr(job_docker, "BUILDER_UID", real_data_info.st_uid) + monkeypatch.setattr(job_docker, "BUILDER_GID", real_data_info.st_gid) + monkeypatch.setattr(job_docker, "RUNTIME", runtime) + monkeypatch.setattr(job_docker, "DATA_ROOT", data_root_link) + + with pytest.raises(job_docker.WrapperError, match="data directory ownership invalid"): + job_docker._ensure_runtime() + + +def test_private_socket_ping_reads_the_complete_bounded_docker_response(monkeypatch): + sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build")) + import job_docker + + class ChunkedSocket: + def __init__(self, response: bytes): + self.response = response + self.offset = 0 + self.recv_calls = 0 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def settimeout(self, _timeout): + pass + + def connect(self, _path): + pass + + def sendall(self, _request): + pass + + def recv(self, size): + self.recv_calls += 1 + end = min(self.offset + min(size, 256), len(self.response)) + chunk = self.response[self.offset:end] + self.offset = end + return chunk + + response = ( + b"HTTP/1.1 200 OK\r\nX-Long-Header: " + + b"x" * 300 + + b"\r\nContent-Length: 2\r\n\r\nOK" + ) + assert len(response) > 256 + + wrapper_socket = ChunkedSocket(response) + monkeypatch.setattr( + job_docker, + "socket", + type("SocketModule", (), { + "AF_UNIX": 1, + "SOCK_STREAM": 2, + "socket": lambda *_args: wrapper_socket, + }), + ) + assert job_docker._socket_ping(Path("/run/itworx-job-docker/docker.sock")) + assert wrapper_socket.recv_calls == 2 + + probe = runpy.run_path(str(ROOT / "ops" / "runner-task-images" / "build" / "itworx-ci-job-docker-probe")) + probe_socket = ChunkedSocket(response) + probe["_socket_ready"].__globals__["socket"] = type("SocketModule", (), { + "AF_UNIX": 1, + "SOCK_STREAM": 2, + "socket": lambda *_args: probe_socket, + }) + assert probe["_socket_ready"]() + assert probe_socket.recv_calls == 2 + + +@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only") +def test_job_docker_wrapper_sanitizes_remote_endpoint_environment_and_rejects_cli_overrides(monkeypatch): + sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build")) + import job_docker + + monkeypatch.setenv("DOCKER_HOST", "tcp://shared-dind:2375") + monkeypatch.setenv("DOCKER_CONTEXT", "shared-context") + monkeypatch.setenv("DOCKER_TLS_VERIFY", "1") + monkeypatch.setenv("DOCKER_CERT_PATH", "/shared/certs") + monkeypatch.setenv("DOCKER_CONFIG", "/job/config") + environment = job_docker._client_environment() + assert environment["DOCKER_HOST"] == "unix:///run/itworx-job-docker/docker.sock" + assert environment["DOCKER_CONFIG"] == "/job/config" + assert not any(name in environment for name in ("DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH")) + with pytest.raises(job_docker.WrapperError, match="endpoint override"): + job_docker._reject_endpoint_override(["--host=tcp://shared-dind:2375", "build"]) + with pytest.raises(job_docker.WrapperError, match="endpoint override"): + job_docker._reject_endpoint_override(["--context", "shared-context", "info"]) + job_docker._reject_endpoint_override(["run", "alpine", "sh", "-c", "echo --context"]) + assert job_docker._is_local_version_request(["compose", "version"]) + assert job_docker._is_local_version_request(["compose", "--version"]) + assert job_docker._is_local_version_request(["buildx", "version"]) + assert job_docker._is_local_version_request(["buildx", "--version"]) + assert not job_docker._is_local_version_request(["compose", "up"]) + + +@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only") +def test_job_docker_daemon_command_uses_fixed_rootlesskit_and_unix_only_engine(): + sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build")) + import job_docker + + command = job_docker._daemon_command() + assert command[:8] == [ + "/usr/local/bin/rootlesskit", + "--state-dir=/run/itworx-job-docker/rootlesskit", + "--net=slirp4netns", + "--mtu=1500", + "--disable-host-loopback", + "--port-driver=builtin", + "--copy-up=/etc", + "--copy-up=/run", + ] + assert command[8:11] == ["/usr/local/bin/docker-init", "--", "/usr/local/bin/dockerd"] + daemon_args = command[11:] + assert "--host=unix:///run/itworx-job-docker/docker.sock" in daemon_args + assert not any(argument.startswith("--host=tcp:") for argument in daemon_args) + assert "--storage-driver=overlay2" in daemon_args + assert "--feature=containerd-snapshotter=false" in daemon_args + assert "--data-root=/home/itworx-builder/.local/share/docker" in daemon_args diff --git a/backend/tests/test_container_restart.py b/backend/tests/test_container_restart.py index 94725c6..3a32b5d 100644 --- a/backend/tests/test_container_restart.py +++ b/backend/tests/test_container_restart.py @@ -344,7 +344,11 @@ def test_workmesh_first_deploy_script_accepts_the_pinned_release_descriptor(tmp_ for original, replacement in ( ('Path("/opt/gitea-deploy")', f'Path({json.dumps(str(controller_root))})'), ( - 'Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")', + 'Path("/controller-config/projects.json")', + f'Path({json.dumps(str(registry_path))})', + ), + ( + 'Path("/controller-hub-releases/controller-config/projects.json")', f'Path({json.dumps(str(registry_path))})', ), ( diff --git a/backend/tests/test_controller_artifact_bootstrap.py b/backend/tests/test_controller_artifact_bootstrap.py new file mode 100644 index 0000000..313ad57 --- /dev/null +++ b/backend/tests/test_controller_artifact_bootstrap.py @@ -0,0 +1,1106 @@ +from __future__ import annotations + +import ast +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from scripts import bootstrap_controller_artifact_promoter as bootstrap + +ROOT = Path(__file__).resolve().parents[2] +LEGACY_REVISION = "5553c04ce497c08a35ed301052dc5e0b02e5d9b4" +LEGACY_TREE = "1624e1748053f998d8427c8c8a4203277d22f13c" +APPOPS_CONTAINER_ID = "72daa9f638defc4f8897d8aad23c4181054b2a1bc04c993078a5ac9597eeb0ec" +APPOPS_IMAGE_ID = "sha256:9b5eddf32c8eb5361106fc35cc0e056c49d287a20d1fb26490662cc8447f09d4" +CONTROLLER_CONTAINER_ID = "e4e7550405aa2a9a91901faf1177d46a35b0fc43c304976ec8971865d43e30eb" +CONTROLLER_IMAGE_ID = "sha256:c474d435a4f55ac5205570da0caa47e103f89a6c2ef96ea45861a23eb7b005dc" +CONTROLLER_IMAGE_REF = "docker.gitea.com/runner-images:ubuntu-latest" +CONTROLLER_REVISION = "034c9f844d2455850ba37b31379b7edaa786583e" +CONTROLLER_TREE = "6b8e92942e6e5eca8a9aa9da0a1509b2019d7b7b" +REVISION = "a" * 40 +TREE = "b" * 40 +CALLER_ID = "c" * 64 +MODERN_HELPER_ID = "f" * 64 + + +LEGACY_HELPER_SOURCE_FIXTURE = ''' +def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]: + if not _REVISION.fullmatch(revision): + raise ValueError("controller_promoter_revision_invalid") + if not _IMAGE_ID.fullmatch(image_id): + raise ValueError("controller_promoter_image_invalid") + + command = f"""set -Eeuo pipefail +revision='{revision}' +version='{CONTROLLER_VERSION}' +repo='/gitea-repositories/jens/unraid-appops-gateway.git' +work='/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{revision[:12]}' +rm -rf -- "$work" +install -d -m 0700 "$work" +cleanup() {{ rm -rf -- "$work"; }} +trap cleanup EXIT +command -v git >/dev/null +command -v docker >/dev/null +docker compose version >/dev/null +command -v python3 >/dev/null +command -v tar >/dev/null +command -v curl >/dev/null +git --git-dir="$repo" cat-file -e "$revision^{{commit}}" +tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")" +git --git-dir="$repo" archive --format=tar "$revision" \\ + ops/gitea-deploy-broker \\ + scripts/prepare_controller_artifact.py \\ + scripts/reconcile_controller_state.py \\ + scripts/run_repopulse_adoption.py > "$work/controller.tar" +archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)" +git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh" +chmod 0700 "$work/upgrade.sh" +bash "$work/upgrade.sh" \\ + --archive "$work/controller.tar" \\ + --archive-sha256 "$archive_sha" \\ + --source-repository '{SOURCE_REPOSITORY}' \\ + --source-revision "$revision" \\ + --source-tree "$tree" \\ + --controller-version "$version" \\ + --confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2 +""" + + return { + "Image": image_id, + "User": "0:0", + "Cmd": ["/bin/bash", "-lc", command], + "Labels": { + HELPER_LABEL: "true", + "itworx.source.repository": SOURCE_REPOSITORY, + "itworx.source.revision": revision, + "itworx.controller.version": CONTROLLER_VERSION, + }, + "HostConfig": { + "Binds": [ + "/mnt/cache:/mnt/cache", + "/mnt/user:/mnt/user", + "/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro", + "/var/run/docker.sock:/var/run/docker.sock", + ], + "NetworkMode": "host", + "Privileged": False, + "SecurityOpt": ["no-new-privileges:true"], + "RestartPolicy": {"Name": "no"}, + }, + } +''' + + +def _legacy_helper_spec() -> dict[str, Any]: + tree = ast.parse(LEGACY_HELPER_SOURCE_FIXTURE) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name == "helper_container_spec" + ) + namespace: dict[str, Any] = { + "re": re, + "Any": Any, + "_REVISION": re.compile(r"^[0-9a-f]{40}$"), + "_IMAGE_ID": re.compile(r"^sha256:[0-9a-f]{64}$"), + "SOURCE_REPOSITORY": "jens/unraid-appops-gateway", + "HELPER_NAME": "itworx-controller-artifact-promoter", + "CONTROLLER_VERSION": "2.1.5", + "HELPER_LABEL": "itworx.controller.promoter", + } + exec( + compile(ast.Module(body=[function], type_ignores=[]), "legacy-promoter", "exec"), + namespace, + ) + return namespace["helper_container_spec"](revision=REVISION, image_id=CONTROLLER_IMAGE_ID) + + +def _image(labels: dict[str, str]) -> dict[str, Any]: + return {"Id": APPOPS_IMAGE_ID, "Config": {"Labels": dict(labels)}} + + +def _appops_labels() -> dict[str, str]: + return { + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.revision": LEGACY_REVISION, + "org.opencontainers.image.source-tree": LEGACY_TREE, + "org.opencontainers.image.version": "1.8.0-platform-5553c04ce497", + "org.opencontainers.image.created": "2026-10-01T16:32:01Z", + "com.docker.compose.project": "unraid-appops-gateway", + "com.docker.compose.service": "appops-gateway", + "com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml", + } + + +def _controller_labels() -> dict[str, str]: + return { + "io.buildah.version": "1.35.0", + "org.opencontainers.image.authors": "Gitea", + "org.opencontainers.image.created": "2026-10-01T00:01:57Z", + "org.opencontainers.image.description": "Official docker images used by Gitea runner to run workflows.", + "org.opencontainers.image.documentation": "https://gitea.com/gitea/runner-images/src/branch/main/README.md", + "org.opencontainers.image.licenses": "MIT", + "org.opencontainers.image.revision": "317f8e3c510e5ec773a103f5d9a9e5769105c59b", + "org.opencontainers.image.source": "https://gitea.com/gitea/runner-images", + "org.opencontainers.image.title": "Official Gitea Actions runner images", + "org.opencontainers.image.url": "https://gitea.com/gitea/runner-images", + "org.opencontainers.image.vendor": "Gitea", + "org.opencontainers.image.version": "v26.10.01", + } + + +def _controller_container_labels() -> dict[str, str]: + return { + **_controller_labels(), + "com.docker.compose.config-hash": "577d77247471213db8a5c465ee2450765ba52cb4048f983c42083ac45a16ddb7", + "com.docker.compose.container-number": "1", + "com.docker.compose.depends_on": "", + "com.docker.compose.image": "sha256:adbb51b2873bf05546019c341caf6fd4368159546c36a5aea81b4d8bf088804e", + "com.docker.compose.oneoff": "False", + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.project.config_files": "/mnt/cache/appdata/gitea-deploy-runners/docker-compose.yml", + "com.docker.compose.project.working_dir": "/mnt/cache/appdata/gitea-deploy-runners", + "com.docker.compose.replace": "gitea-deploy-control", + "com.docker.compose.service": "deploy-control", + "com.docker.compose.version": "2.40.3", + "tech.itworx.deployment.target": "unraid", + "tech.itworx.role": "gitea-deploy-controller", + "tech.itworx.gitea.scope": "Jens", + } + + +def _modern_controller_labels() -> dict[str, str]: + return { + **_controller_labels(), + "com.docker.compose.config-hash": "a" * 64, + "com.docker.compose.container-number": "1", + "com.docker.compose.depends_on": "", + "com.docker.compose.image": CONTROLLER_IMAGE_ID, + "com.docker.compose.oneoff": "False", + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.project.config_files": "-", + "com.docker.compose.project.environment_file": "/dev/null", + "com.docker.compose.project.working_dir": "/var/lib/docker/volumes/itworx-controller-hub-releases/_data", + "com.docker.compose.replace": "gitea-deploy-control", + "com.docker.compose.service": "deploy-control", + "com.docker.compose.version": "2.40.3", + "tech.itworx.deployment.target": "unraid", + "tech.itworx.gitea.scope": "Jens", + "tech.itworx.role": "gitea-deploy-controller", + } + + +def _modern_controller_mounts(source_revision: str) -> list[dict[str, Any]]: + artifact_source = ( + "/var/lib/docker/volumes/itworx-controller-hub-releases/_data/" + f"controller-artifacts/{source_revision}" + ) + return [ + {"Type": "bind", "Source": "/boot", "Destination": "/boot", "RW": True, "Propagation": "rprivate"}, + {"Type": "bind", "Source": "/var/lib/docker", "Destination": "/var/lib/docker", "RW": False, "Propagation": "rslave"}, + { + "Type": "bind", + "Source": "/srv/example/appdata/gitea/git/repositories", + "Destination": "/gitea-repositories", + "RW": False, + "Propagation": "rprivate", + }, + {"Type": "bind", "Source": artifact_source, "Destination": "/opt/gitea-deploy", "RW": False, "Propagation": "rprivate"}, + {"Type": "bind", "Source": "/var/run/docker.sock", "Destination": "/var/run/docker.sock", "RW": True, "Propagation": "rprivate"}, + {"Type": "bind", "Source": "/mnt/user", "Destination": "/mnt/user", "RW": True, "Propagation": "rprivate"}, + {"Type": "bind", "Source": "/mnt/cache", "Destination": "/mnt/cache", "RW": True, "Propagation": "rprivate"}, + {"Type": "bind", "Source": "/mnt/disks", "Destination": "/mnt/disks", "RW": False, "Propagation": "rprivate"}, + { + "Type": "volume", + "Name": "itworx-controller-hub-releases", + "Source": "/var/lib/docker/volumes/itworx-controller-hub-releases/_data", + "Destination": "/controller-hub-releases", + "RW": True, + "Propagation": "", + }, + { + "Type": "bind", + "Source": "/var/lib/docker/volumes/itworx-controller-hub-releases/_data/controller-config", + "Destination": "/controller-config", + "RW": False, + "Propagation": "rprivate", + }, + ] + + +def _modern_controller_inspect(source_revision: str) -> dict[str, Any]: + controller = _controller_inspect() + controller.update({"Id": "e" * 64, "Image": CONTROLLER_IMAGE_ID}) + controller["Config"]["Image"] = CONTROLLER_IMAGE_ID + controller["Config"]["Labels"] = _modern_controller_labels() + controller["Mounts"] = _modern_controller_mounts(source_revision) + return controller + + +def _appops_inspect() -> dict[str, Any]: + labels = _appops_labels() + return { + "Id": APPOPS_CONTAINER_ID, + "Image": APPOPS_IMAGE_ID, + "Name": "/Unraid-AppOps-Gateway", + "State": {"Running": True, "Health": {"Status": "healthy"}}, + "Config": { + "Hostname": "unraid-appops-gateway", + "Image": "unraid-appops-gateway:platform-5553c04ce497", + "Labels": labels, + }, + "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, + "Mounts": [ + { + "Type": "bind", + "Source": "/srv/example/appdata/unraid-appops-gateway-1216", + "Destination": "/data", + "RW": True, + }, + { + "Type": "bind", + "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models", + "Destination": "/mnt/privatemuse-comfyui-models", + "RW": True, + }, + ], + } + + +def _controller_inspect() -> dict[str, Any]: + return { + "Id": CONTROLLER_CONTAINER_ID, + "Image": CONTROLLER_IMAGE_ID, + "Name": "/gitea-deploy-control", + "State": {"Running": True}, + "Config": {"Image": CONTROLLER_IMAGE_REF, "Labels": _controller_container_labels()}, + "HostConfig": { + "Binds": [ + "/mnt/user:/mnt/user:rw", + "/mnt/cache:/mnt/cache:rw", + "/mnt/disks:/mnt/disks:ro", + "/boot:/boot:rw", + "/var/lib/docker:/var/lib/docker:ro", + "/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro", + "/mnt/cache/appdata/gitea-deploy-runners/controller:/opt/gitea-deploy:ro", + "/var/run/docker.sock:/var/run/docker.sock:rw", + ], + "Mounts": None, + }, + "Mounts": [ + { + "Type": "bind", + "Source": "/boot", + "Destination": "/boot", + "RW": True, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": "/var/lib/docker", + "Destination": "/var/lib/docker", + "RW": False, + "Propagation": "rslave", + }, + { + "Type": "bind", + "Source": "/srv/example/appdata/gitea/git/repositories", + "Destination": "/gitea-repositories", + "RW": False, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": "/mnt/cache/appdata/gitea-deploy-runners/controller", + "Destination": "/opt/gitea-deploy", + "RW": False, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": "/var/run/docker.sock", + "Destination": "/var/run/docker.sock", + "RW": True, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": "/mnt/user", + "Destination": "/mnt/user", + "RW": True, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": "/mnt/cache", + "Destination": "/mnt/cache", + "RW": True, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": "/mnt/disks", + "Destination": "/mnt/disks", + "RW": False, + "Propagation": "rprivate", + }, + ], + } + + +def _inspect_from_spec( + name: str, + identifier: str, + image_id: str, + spec: dict[str, Any], + *, + state: dict[str, Any], +) -> dict[str, Any]: + return { + "Id": identifier, + "Image": image_id, + "Name": f"/{name}", + "State": dict(state), + "Config": { + "Image": spec["Image"], + "User": spec["User"], + "Cmd": spec["Cmd"], + "Labels": {**_controller_labels(), **spec["Labels"]}, + "Hostname": identifier[:12], + }, + "HostConfig": {"ReadonlyRootfs": False, **spec["HostConfig"]}, + } + + +class FakeDocker: + def __init__(self, caller_spec: dict[str, Any], *, existing_child: dict[str, Any] | None = None): + self.containers = { + APPOPS_CONTAINER_ID: _appops_inspect(), + "gitea-deploy-control": _controller_inspect(), + CALLER_ID[:12]: _inspect_from_spec( + "itworx-controller-artifact-promoter", + CALLER_ID, + CONTROLLER_IMAGE_ID, + caller_spec, + state={"Running": True}, + ), + } + self.images = { + APPOPS_IMAGE_ID: _image(_appops_labels()), + CONTROLLER_IMAGE_ID: { + "Id": CONTROLLER_IMAGE_ID, + "Config": {"Labels": _controller_labels()}, + }, + } + self.existing_child = existing_child + self.created: list[dict[str, Any]] = [] + self.started: list[str] = [] + self.log_bytes = b"controller_v2_ready revision=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + + def inspect_container(self, reference: str) -> dict[str, Any] | None: + if reference == "itworx-controller-artifact-promoter-modern": + return self.existing_child + return self.containers.get(reference) + + def inspect_image(self, image_id: str) -> dict[str, Any] | None: + return self.images.get(image_id) + + def create_container(self, name: str, spec: dict[str, Any]) -> str: + assert name == "itworx-controller-artifact-promoter-modern" + self.created.append(spec) + container = _inspect_from_spec( + name, + MODERN_HELPER_ID, + spec["Image"], + spec, + state={"Running": False, "ExitCode": 0, "Status": "exited"}, + ) + self.containers[name] = container + self.containers[MODERN_HELPER_ID] = container + return MODERN_HELPER_ID + + def start_container(self, identifier: str) -> None: + self.started.append(identifier) + source_revision = self.created[-1]["Labels"]["itworx.source.revision"] + self.containers["gitea-deploy-control"] = _modern_controller_inspect(source_revision) + + def container_logs(self, identifier: str, *, tail: int, max_bytes: int) -> bytes: + assert identifier == MODERN_HELPER_ID + assert tail <= 40 + assert max_bytes <= 8192 + return self.log_bytes[:max_bytes] + + +class FakeProtectedGit: + def __init__(self, helper_source: str): + self.revision = REVISION + self.tree = TREE + self.helper_source = helper_source + self.blobs_read: list[tuple[str, str]] = [] + self.main_checks = 0 + + def verify_current_main(self, revision: str, tree: str) -> None: + self.main_checks += 1 + if (revision, tree) != (self.revision, self.tree): + raise bootstrap.BootstrapError("controller_promoter_source_not_current_main") + + def verify_legacy_revision(self, revision: str, tree: str) -> None: + if (revision, tree) != (LEGACY_REVISION, LEGACY_TREE): + raise bootstrap.BootstrapError("controller_promoter_legacy_provenance_invalid") + + def read_blob(self, revision: str, path: str) -> bytes: + self.blobs_read.append((revision, path)) + if revision != self.revision or path != "backend/app/services/controller_artifact_helper_spec.py": + raise bootstrap.BootstrapError("controller_promoter_source_blob_invalid") + return self.helper_source.encode("utf-8") + + def read_legacy_blob(self, revision: str, path: str) -> bytes: + self.blobs_read.append((revision, path)) + if (revision, path) != ( + LEGACY_REVISION, + "backend/app/services/controller_artifact_promoter.py", + ): + raise bootstrap.BootstrapError("controller_promoter_legacy_blob_invalid") + return LEGACY_HELPER_SOURCE_FIXTURE.encode("utf-8") + + +def _health( + *, + revision: str = REVISION, + tree: str = TREE, + version: str = "2.1.14", + artifact_sha256: str = "1" * 64, +) -> dict[str, Any]: + return { + "status": "ok", + "protocolVersion": "itworx-controller-request/v2", + "artifact": { + "apiVersion": "itworx.controller-artifact/v1", + "sourceRepository": "jens/unraid-appops-gateway", + "sourceRevision": revision, + "sourceTree": tree, + "controllerVersion": version, + "artifactSha256": artifact_sha256, + }, + "delivery": { + "health": { + "configuration": "healthy", + "inbox": "healthy", + "artifact": "healthy", + } + }, + } + + +def _run_bootstrap( + docker: FakeDocker, + git: FakeProtectedGit, + *, + health: dict[str, Any] | None = None, + initial_health: dict[str, Any] | None = None, + hostname: str = CALLER_ID[:12], +): + source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text( + encoding="utf-8" + ) + git.helper_source = source + initial = initial_health or _health( + revision=CONTROLLER_REVISION, + tree=CONTROLLER_TREE, + version="2.1.12", + artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256, + ) + health_reads = [initial, health or _health()] + + def read_health() -> dict[str, Any]: + return health_reads.pop(0) if health_reads else health or _health() + + return bootstrap.run_bootstrap( + "jens/unraid-appops-gateway", + REVISION, + TREE, + docker=docker, + protected_git=git, + hostname=hostname, + health_reader=read_health, + monotonic=lambda: 0.0, + sleeper=lambda _seconds: None, + child_timeout_seconds=0.0, + ) + + +def test_initial_controller_runtime_accepts_exact_current_compose_container(): + docker = FakeDocker(_legacy_helper_spec()) + base_image_labels = bootstrap._inspect_controller_base_image(docker) + health = _health( + revision=CONTROLLER_REVISION, + tree=CONTROLLER_TREE, + version="2.1.12", + artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256, + ) + + assert ( + bootstrap._verify_controller_runtime( + docker, + health_payload=health, + base_image_labels=base_image_labels, + ) + == base_image_labels + ) + + +def test_immutable_legacy_helper_source_fixture_passes_bootstrap_identity_guard(): + legacy_spec = _legacy_helper_spec() + docker = FakeDocker(legacy_spec) + git = FakeProtectedGit("") + + result = _run_bootstrap(docker, git) + + assert result["status"] == "promoted" + assert docker.created + assert git.blobs_read == [ + (REVISION, "backend/app/services/controller_artifact_helper_spec.py"), + (LEGACY_REVISION, "backend/app/services/controller_artifact_promoter.py"), + ] + + +@pytest.mark.parametrize( + "mutation", + [ + "caller-image", + "caller-label", + "caller-revision", + "caller-command", + "caller-bind", + "appops-image", + "appops-provenance", + "target-id", + "target-source", + "target-tree", + "target-version", + "old-target-id", + "old-target-image", + "old-target-version", + "target-config-image", + "target-compose-hash", + "target-compose-missing", + "target-compose-extra", + "target-compose-image", + "target-tech-label", + "target-tech-extra", + "target-path", + "target-extra-mount", + "target-bind-mode", + "base-image-installed-provenance", + ], +) +def test_bootstrap_rejects_unknown_runtime_identity_without_creating_child(mutation: str): + docker = FakeDocker(_legacy_helper_spec()) + if mutation == "caller-image": + docker.containers[CALLER_ID[:12]]["Image"] = "sha256:" + "9" * 64 + elif mutation == "caller-label": + docker.containers[CALLER_ID[:12]]["Config"]["Labels"]["itworx.controller.version"] = "2.1.4" + elif mutation == "caller-revision": + docker.containers[CALLER_ID[:12]]["Config"]["Labels"]["itworx.source.revision"] = "d" * 40 + elif mutation == "caller-command": + docker.containers[CALLER_ID[:12]]["Config"]["Cmd"].append("--image-id") + elif mutation == "caller-bind": + docker.containers[CALLER_ID[:12]]["HostConfig"]["Binds"].append("/tmp:/payload") + elif mutation == "appops-image": + docker.containers[APPOPS_CONTAINER_ID]["Image"] = "sha256:" + "8" * 64 + elif mutation == "appops-provenance": + docker.images[APPOPS_IMAGE_ID]["Config"]["Labels"]["org.opencontainers.image.revision"] = "f" * 40 + elif mutation == "target-id": + docker.containers["gitea-deploy-control"]["Id"] = "0" * 64 + elif mutation == "old-target-id": + docker.containers["gitea-deploy-control"]["Id"] = "5539b672e66890e657d0f3b0e6e94f9c9ca5b47c245e120d330f6f1d5be73fae" + elif mutation == "old-target-image": + docker.containers["gitea-deploy-control"]["Image"] = "sha256:adbb51b2873bf05546019c341caf6fd4368159546c36a5aea81b4d8bf088804e" + elif mutation == "target-source": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.revision"] = "f" * 40 + elif mutation == "target-tree": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.source-tree"] = "f" * 40 + elif mutation == "target-version": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.version"] = "2.1.11" + elif mutation == "old-target-version": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["org.opencontainers.image.version"] = "v26.09.29" + elif mutation == "target-config-image": + docker.containers["gitea-deploy-control"]["Config"]["Image"] = "docker.gitea.com/runner-images:ubuntu-22.04" + elif mutation == "target-compose-hash": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["com.docker.compose.config-hash"] = "0" * 64 + elif mutation == "target-compose-missing": + docker.containers["gitea-deploy-control"]["Config"]["Labels"].pop("com.docker.compose.service") + elif mutation == "target-compose-extra": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["com.docker.compose.container_name"] = "gitea-deploy-control" + elif mutation == "target-compose-image": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["com.docker.compose.image"] = CONTROLLER_IMAGE_ID + elif mutation == "target-tech-label": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["tech.itworx.role"] = "other" + elif mutation == "target-tech-extra": + docker.containers["gitea-deploy-control"]["Config"]["Labels"]["tech.itworx.extra"] = "unexpected" + elif mutation == "target-path": + docker.containers["gitea-deploy-control"]["Mounts"][3]["Destination"] = "/controller-config" + elif mutation == "target-extra-mount": + docker.containers["gitea-deploy-control"]["Mounts"].append( + { + "Type": "bind", + "Source": "/tmp", + "Destination": "/tmp", + "RW": True, + "Propagation": "rprivate", + } + ) + docker.containers["gitea-deploy-control"]["HostConfig"]["Binds"].append( + "/tmp:/tmp:rw" + ) + elif mutation == "target-bind-mode": + docker.containers["gitea-deploy-control"]["HostConfig"]["Binds"][6] = ( + "/mnt/cache/appdata/gitea-deploy-runners/controller:/opt/gitea-deploy:rw" + ) + docker.containers["gitea-deploy-control"]["Mounts"][3]["RW"] = True + elif mutation == "base-image-installed-provenance": + labels = docker.images[CONTROLLER_IMAGE_ID]["Config"]["Labels"] + labels["org.opencontainers.image.revision"] = CONTROLLER_REVISION + labels["org.opencontainers.image.source-tree"] = CONTROLLER_TREE + git = FakeProtectedGit("") + + with pytest.raises(bootstrap.BootstrapError): + _run_bootstrap(docker, git) + + assert docker.created == [] + + +@pytest.mark.parametrize( + "mutation", + ["revision", "tree", "version", "artifact-sha", "delivery-health"], +) +def test_bootstrap_requires_exact_mounted_controller_artifact_health(mutation: str): + initial = _health( + revision=CONTROLLER_REVISION, + tree=CONTROLLER_TREE, + version="2.1.12", + artifact_sha256=bootstrap.LEGACY_CONTROLLER_ARTIFACT_SHA256, + ) + artifact = initial["artifact"] + if mutation == "revision": + artifact["sourceRevision"] = "9" * 40 + elif mutation == "tree": + artifact["sourceTree"] = "9" * 40 + elif mutation == "version": + artifact["controllerVersion"] = "2.1.11" + elif mutation == "artifact-sha": + artifact["artifactSha256"] = "9" * 64 + elif mutation == "delivery-health": + initial["delivery"]["health"]["artifact"] = "degraded" + + docker = FakeDocker(_legacy_helper_spec()) + git = FakeProtectedGit("") + with pytest.raises(bootstrap.BootstrapError): + _run_bootstrap(docker, git, initial_health=initial) + assert docker.created == [] + + +def test_running_modern_child_returns_unknown_without_duplicate_or_stop(): + helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text( + encoding="utf-8" + ) + helper_spec = bootstrap.render_modern_helper_spec( + helper_source, + revision=REVISION, + image_id=CONTROLLER_IMAGE_ID, + current_controller_id=CONTROLLER_CONTAINER_ID, + current_controller_revision=CONTROLLER_REVISION, + current_controller_tree=CONTROLLER_TREE, + current_controller_version="2.1.12", + ) + existing = _inspect_from_spec( + "itworx-controller-artifact-promoter-modern", + MODERN_HELPER_ID, + CONTROLLER_IMAGE_ID, + helper_spec, + state={"Running": True, "Status": "running"}, + ) + docker = FakeDocker(_legacy_helper_spec(), existing_child=existing) + git = FakeProtectedGit(helper_source) + + result = _run_bootstrap(docker, git, health=_health()) + + assert result["status"] == "outcome_unknown" + assert result["reason"] == "controller_promoter_modern_helper_running" + assert docker.created == [] + assert docker.started == [] + + +def test_completed_modern_child_with_wrong_artifact_is_not_relaunched(): + helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text( + encoding="utf-8" + ) + helper_spec = bootstrap.render_modern_helper_spec( + helper_source, + revision=REVISION, + image_id=CONTROLLER_IMAGE_ID, + current_controller_id=CONTROLLER_CONTAINER_ID, + current_controller_revision=CONTROLLER_REVISION, + current_controller_tree=CONTROLLER_TREE, + current_controller_version="2.1.12", + ) + existing = _inspect_from_spec( + "itworx-controller-artifact-promoter-modern", + MODERN_HELPER_ID, + CONTROLLER_IMAGE_ID, + helper_spec, + state={"Running": False, "ExitCode": 0, "Status": "exited"}, + ) + docker = FakeDocker(_legacy_helper_spec(), existing_child=existing) + git = FakeProtectedGit(helper_source) + + result = _run_bootstrap( + docker, + git, + initial_health=_health(tree="9" * 40), + ) + + assert result["status"] == "outcome_unknown" + assert result["reason"] == "controller_promoter_postcheck_failed" + assert docker.created == [] + + +def test_completed_child_uses_new_health_after_controller_recreation(): + helper_source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_text( + encoding="utf-8" + ) + helper_spec = bootstrap.render_modern_helper_spec( + helper_source, + revision=REVISION, + image_id=CONTROLLER_IMAGE_ID, + current_controller_id=CONTROLLER_CONTAINER_ID, + current_controller_revision=CONTROLLER_REVISION, + current_controller_tree=CONTROLLER_TREE, + current_controller_version="2.1.12", + ) + existing = _inspect_from_spec( + "itworx-controller-artifact-promoter-modern", + MODERN_HELPER_ID, + CONTROLLER_IMAGE_ID, + helper_spec, + state={"Running": False, "ExitCode": 0, "Status": "exited"}, + ) + docker = FakeDocker(_legacy_helper_spec(), existing_child=existing) + docker.containers["gitea-deploy-control"] = _modern_controller_inspect(REVISION) + git = FakeProtectedGit(helper_source) + + result = _run_bootstrap( + docker, + git, + initial_health=_health(), + ) + + assert result["status"] == "already_applied" + assert result["revision"] == REVISION + assert docker.created == [] + assert docker.started == [] + + +@pytest.mark.parametrize( + "mutation", + [ + "post-image-config", + "post-compose-hash", + "post-compose-image", + "post-compose-extra", + "post-compose-environment-file", + "post-tech-label", + "post-tech-extra", + "post-artifact-revision", + "post-volume-name", + "post-volume-source", + "post-volume-mode", + "post-config-bind-source", + "post-config-bind-mode", + "post-extra-mount", + ], +) +def test_postactivation_controller_requires_exact_compose_runtime(mutation: str): + docker = FakeDocker(_legacy_helper_spec()) + controller = _modern_controller_inspect(REVISION) + docker.containers["gitea-deploy-control"] = controller + + if mutation == "post-image-config": + controller["Config"]["Image"] = CONTROLLER_IMAGE_REF + elif mutation == "post-compose-hash": + controller["Config"]["Labels"]["com.docker.compose.config-hash"] = "bad" + elif mutation == "post-compose-image": + controller["Config"]["Labels"]["com.docker.compose.image"] = "sha256:" + "d" * 64 + elif mutation == "post-compose-extra": + controller["Config"]["Labels"]["com.docker.compose.extra"] = "unexpected" + elif mutation == "post-compose-environment-file": + controller["Config"]["Labels"]["com.docker.compose.project.environment_file"] = "/tmp/.env" + elif mutation == "post-tech-label": + controller["Config"]["Labels"]["tech.itworx.gitea.scope"] = "Other" + elif mutation == "post-tech-extra": + controller["Config"]["Labels"]["tech.itworx.extra"] = "unexpected" + elif mutation == "post-artifact-revision": + controller["Mounts"][3]["Source"] = ( + "/var/lib/docker/volumes/itworx-controller-hub-releases/_data/" + f"controller-artifacts/{'d' * 40}" + ) + elif mutation == "post-volume-name": + controller["Mounts"][8]["Name"] = "foreign-volume" + elif mutation == "post-volume-source": + controller["Mounts"][8]["Source"] = "/tmp/foreign-volume" + elif mutation == "post-volume-mode": + controller["Mounts"][8]["RW"] = False + elif mutation == "post-config-bind-source": + controller["Mounts"][9]["Source"] = "/tmp/config" + elif mutation == "post-config-bind-mode": + controller["Mounts"][9]["RW"] = True + elif mutation == "post-extra-mount": + controller["Mounts"].append(dict(controller["Mounts"][0])) + + assert not bootstrap._verified_live_controller(docker, source_revision=REVISION) + + +def test_postactivation_accepts_well_formed_compose_hash(): + docker = FakeDocker(_legacy_helper_spec()) + controller = _modern_controller_inspect(REVISION) + controller["Config"]["Labels"]["com.docker.compose.config-hash"] = "b" * 64 + docker.containers["gitea-deploy-control"] = controller + + assert bootstrap._verified_live_controller(docker, source_revision=REVISION) + + +def test_successful_child_requires_exact_source_tree_version_and_health(): + docker = FakeDocker(_legacy_helper_spec()) + git = FakeProtectedGit("") + + result = _run_bootstrap(docker, git) + + assert result["status"] == "promoted" + assert result["revision"] == REVISION + assert result["controllerVersion"] == "2.1.14" + assert result["artifactSha256"] == "1" * 64 + assert docker.started == [MODERN_HELPER_ID] + assert docker.created[0]["Image"] == CONTROLLER_IMAGE_ID + assert docker.created[0]["HostConfig"]["ReadonlyRootfs"] is True + + +def test_child_logs_are_bounded_and_redacted_before_result_output(): + docker = FakeDocker(_legacy_helper_spec()) + docker.log_bytes = ( + b"controller_v2_ready revision=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" + b"GITEA_ADMIN_TOKEN=very-secret-value\n" + ) + git = FakeProtectedGit("") + + result = _run_bootstrap(docker, git) + + assert result["status"] == "promoted" + assert "very-secret-value" not in result["logTail"] + assert "[redacted]" in result["logTail"] + assert len(result["logTail"].encode("utf-8")) <= 4096 + + +def test_legacy_upgrade_signature_fetches_bootstrap_only_from_protected_git(tmp_path: Path): + bash = shutil.which("bash") + if bash is None: + pytest.skip("bash is required to exercise the upgrade script") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + git_trace = tmp_path / "git-args.jsonl" + python_trace = tmp_path / "python-args.json" + protected_source = "# trusted protected Git bootstrap source\n" + git_shim = bin_dir / "git" + git_shim.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + f"trace = {str(git_trace)!r}\n" + f"with open(trace, 'a', encoding='utf-8') as stream: stream.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "args = [arg for arg in sys.argv[1:] if not arg.startswith('--git-dir=')]\n" + f"revision = {REVISION!r}\n" + f"tree = {TREE!r}\n" + "if args[-3:] == ['rev-parse', 'refs/heads/main', '--'] or args[-2:] == ['rev-parse', 'refs/heads/main']: print(revision)\n" + "elif any(arg.endswith('^{tree}') for arg in args): print(tree)\n" + f"elif len(args) >= 3 and args[:2] == ['cat-file', '-s']: print({len(protected_source)})\n" + "elif len(args) >= 3 and args[0] == 'cat-file' and args[1] == 'blob' and args[-1].endswith(':scripts/bootstrap_controller_artifact_promoter.py'):\n" + f" sys.stdout.write({protected_source!r})\n" + "else: raise SystemExit(91)\n", + encoding="utf-8", + ) + stat_shim = bin_dir / "stat" + stat_shim.write_text( + f"#!{sys.executable}\n" + "import pathlib, sys\n" + "args = sys.argv[1:]\n" + "target = args[-1]\n" + "if target == '/tmp': print('1777:0:0')\n" + "elif args[1] == '%u:%a': print('0:700')\n" + "elif args[1] == '%u:%a:%s': print('0:600:' + str(pathlib.Path(target).stat().st_size))\n" + "else: raise SystemExit(1)\n", + encoding="utf-8", + ) + python_shim = bin_dir / "python3" + python_shim.write_text( + f"#!{sys.executable}\n" + "import json, pathlib, sys\n" + f"path = pathlib.Path({str(python_trace)!r})\n" + "path.write_text(json.dumps({'script': pathlib.Path(sys.argv[1]).read_text(), 'args': sys.argv[2:]}), encoding='utf-8')\n", + encoding="utf-8", + ) + git_shim.chmod(0o755) + stat_shim.chmod(0o755) + python_shim.chmod(0o755) + archive = f"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{REVISION[:12]}/controller.tar" + result = subprocess.run( + [ + bash, + str(ROOT / "scripts/upgrade_deployment_controller.sh"), + "--archive", + archive, + "--archive-sha256", + "a" * 64, + "--source-repository", + "jens/unraid-appops-gateway", + "--source-revision", + REVISION, + "--source-tree", + TREE, + "--controller-version", + "2.1.5", + "--confirm", + "UPGRADE_DEPLOYMENT_CONTROLLER_V2", + ], + cwd=ROOT, + env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"}, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + bootstrap_call = json.loads(python_trace.read_text(encoding="utf-8")) + assert bootstrap_call["script"] == protected_source + assert bootstrap_call["args"] == [ + "--source-repository", + "jens/unraid-appops-gateway", + "--source-revision", + REVISION, + "--source-tree", + TREE, + ] + calls = [json.loads(line) for line in git_trace.read_text(encoding="utf-8").splitlines()] + assert any(args[-1:] == [f"{REVISION}:scripts/bootstrap_controller_artifact_promoter.py"] for args in calls) + assert all(archive not in args for args in calls) + + +@pytest.mark.parametrize( + "mutation", + ["source", "revision", "tree", "version", "extra", "image-id"], +) +def test_upgrade_script_rejects_mutated_legacy_call_without_bootstrap( + tmp_path: Path, mutation: str +): + bash = shutil.which("bash") + if bash is None: + pytest.skip("bash is required to exercise the upgrade script") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + marker = tmp_path / "bootstrap-ran" + python_shim = bin_dir / "python3" + python_shim.write_text( + f"#!/usr/bin/env python3\nfrom pathlib import Path\nPath({str(marker)!r}).write_text('ran')\n", + encoding="utf-8", + ) + python_shim.chmod(0o755) + args = [ + "--archive", + f"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{REVISION[:12]}/controller.tar", + "--archive-sha256", + "a" * 64, + "--source-repository", + "jens/unraid-appops-gateway", + "--source-revision", + REVISION, + "--source-tree", + TREE, + "--controller-version", + "2.1.5", + "--confirm", + "UPGRADE_DEPLOYMENT_CONTROLLER_V2", + ] + if mutation == "source": + args[args.index("jens/unraid-appops-gateway")] = "attacker/repo" + elif mutation == "revision": + args[args.index(REVISION)] = "A" * 40 + elif mutation == "tree": + args[args.index(TREE)] = "f" * 40 + elif mutation == "version": + args[args.index("2.1.5")] = "2.1.4" + elif mutation == "extra": + args.extend(["--unexpected", "value"]) + elif mutation == "image-id": + args.extend(["--image-id", CONTROLLER_IMAGE_ID]) + result = subprocess.run( + [bash, str(ROOT / "scripts/upgrade_deployment_controller.sh"), *args], + cwd=ROOT, + env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"}, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert not marker.exists() + + +@pytest.mark.parametrize("modern_call", ["missing-image", "shared-archive"]) +def test_modern_upgrade_path_still_requires_private_archive_and_image_id( + tmp_path: Path, modern_call: str +): + bash = shutil.which("bash") + if bash is None: + pytest.skip("bash is required to exercise the upgrade script") + archive = ( + f"/tmp/itworx-controller-artifact-promoter-{REVISION[:12]}.AAAAAA/controller.tar" + if modern_call == "missing-image" + else f"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-{REVISION[:12]}/controller.tar" + ) + args = [ + "--archive", + archive, + "--archive-sha256", + "a" * 64, + "--source-repository", + "jens/unraid-appops-gateway", + "--source-revision", + REVISION, + "--source-tree", + TREE, + "--controller-version", + "2.1.14", + "--confirm", + "UPGRADE_DEPLOYMENT_CONTROLLER_V2", + ] + if modern_call == "shared-archive": + args.extend(["--image-id", CONTROLLER_IMAGE_ID]) + result = subprocess.run( + [bash, str(ROOT / "scripts/upgrade_deployment_controller.sh"), *args], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + if modern_call == "missing-image": + assert "image ID" in result.stderr + else: + assert "archive is outside" in result.stderr diff --git a/backend/tests/test_controller_artifact_promoter.py b/backend/tests/test_controller_artifact_promoter.py index baf2bce..b8cfd09 100644 --- a/backend/tests/test_controller_artifact_promoter.py +++ b/backend/tests/test_controller_artifact_promoter.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import shlex from pathlib import Path import pytest @@ -8,6 +9,22 @@ import pytest ROOT = Path(__file__).resolve().parents[2] +def test_promoter_archive_covers_every_required_installer_source(): + from backend.app.services.controller_artifact_promoter import helper_container_spec + + spec = helper_container_spec(revision="a" * 40, image_id="sha256:" + "b" * 64) + command = spec["Cmd"][2].replace("\\\n", " ") + archive = re.search(r'archive --format=tar "\$revision"(.*?)> "\$work/controller.tar"', command, re.S) + assert archive is not None + packaged = shlex.split(archive.group(1)) + installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8") + required_block = re.search(r"(?m)^source_paths=\(\n(.*?)\n\)", installer, re.S) + assert required_block is not None + required = shlex.split(required_block.group(1)) + assert required + assert all(any(path == prefix or path.startswith(prefix + "/") for prefix in packaged) for path in required) + + def test_controller_promoter_action_is_fixed_and_registered(): registry = (ROOT / "backend/app/services/action_registry.py").read_text(encoding="utf-8") execution = (ROOT / "backend/app/services/action_execution.py").read_text(encoding="utf-8") @@ -28,39 +45,103 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded(): image_id = "sha256:" + "b" * 64 spec = helper_container_spec(revision=revision, image_id=image_id) - assert HELPER_NAME == "itworx-controller-artifact-promoter" - assert CONTROLLER_VERSION == "2.1.13" + assert HELPER_NAME == "itworx-controller-artifact-promoter-modern" + assert CONTROLLER_VERSION == "2.1.14" assert re.fullmatch(r"2\.1\.\d+", CONTROLLER_VERSION) assert spec["Image"] == image_id assert spec["HostConfig"]["Privileged"] is False assert spec["HostConfig"]["NetworkMode"] == "host" assert spec["HostConfig"]["SecurityOpt"] == ["no-new-privileges:true"] + assert spec["HostConfig"]["ReadonlyRootfs"] is True + assert spec["HostConfig"]["PidsLimit"] <= 128 + assert spec["HostConfig"]["Memory"] <= 1024 * 1024 * 1024 + assert spec["HostConfig"]["CpuQuota"] <= spec["HostConfig"]["CpuPeriod"] + assert set(spec["HostConfig"]["Tmpfs"]) == {"/tmp"} assert set(spec["HostConfig"]["Binds"]) == { "/mnt/cache:/mnt/cache", "/mnt/user:/mnt/user", - "/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro", "/var/lib/docker:/var/lib/docker:ro", + "/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro", "/var/run/docker.sock:/var/run/docker.sock", } assert spec["Cmd"][:2] == ["/bin/bash", "-lc"] command = spec["Cmd"][2] - assert "scripts/upgrade_deployment_controller.sh" in command - assert "UPGRADE_DEPLOYMENT_CONTROLLER_V2" in command - assert revision in command - assert CONTROLLER_VERSION in command - assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command + assert "mktemp -d -p /tmp itworx-controller-artifact-promoter-" in command assert f"itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX" in command + assert "/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-" not in command assert "work='/mnt/cache" not in command assert "rm -rf -- \"$work\"" in command + assert "scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"" in command assert "chmod 0600 \"$work/controller.tar\"" in command assert "scripts/prepare_controller_hub_release_volume.py" in command + assert "scripts/run_repopulse_adoption.py" in command + assert "UPGRADE_DEPLOYMENT_CONTROLLER_V2" in command + assert revision in command + assert "--image-id" in command + assert CONTROLLER_VERSION in command + assert "version='2.1.14'" in command + assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command archive_start = command.index('git --git-dir="$repo" archive') archive_end = command.index('> "$work/controller.tar"', archive_start) assert "scripts/vendor" in command[archive_start:archive_end] assert "scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"" in command - assert f"--image-id '{image_id}'" in command + assert '--image-id "$image_id"' in command +def test_controller_promoter_bootstrap_child_spec_binds_legacy_target_identity(): + from backend.app.services.controller_artifact_promoter import helper_container_spec + + revision = "a" * 40 + image_id = "sha256:" + "b" * 64 + target_id = "c" * 64 + target_revision = "d" * 40 + target_tree = "e" * 40 + spec = helper_container_spec( + revision=revision, + image_id=image_id, + current_controller_id=target_id, + current_controller_revision=target_revision, + current_controller_tree=target_tree, + current_controller_version="2.1.12", + ) + + assert spec["Image"] == image_id + assert spec["Labels"] == { + "itworx.controller.promoter": "true", + "itworx.source.repository": "jens/unraid-appops-gateway", + "itworx.source.revision": revision, + "itworx.controller.version": "2.1.14", + "itworx.controller.bootstrap.target.id": target_id, + "itworx.controller.bootstrap.target.image": image_id, + "itworx.controller.bootstrap.target.revision": target_revision, + "itworx.controller.bootstrap.target.tree": target_tree, + "itworx.controller.bootstrap.target.version": "2.1.12", + } + + +def test_controller_promoter_rejects_partial_or_mutable_bootstrap_binding(): + from backend.app.services.controller_artifact_promoter import helper_container_spec + + base = {"revision": "a" * 40, "image_id": "sha256:" + "b" * 64} + with pytest.raises(ValueError): + helper_container_spec(**base, current_controller_id="c" * 64) + with pytest.raises(ValueError): + helper_container_spec( + **base, + current_controller_id="c" * 64, + current_controller_revision="../controller", + current_controller_tree="e" * 40, + current_controller_version="2.1.12", + ) + with pytest.raises(ValueError): + helper_container_spec( + **base, + current_controller_id="../controller", + current_controller_revision="d" * 40, + current_controller_tree="e" * 40, + current_controller_version="2.1.12", + ) + @pytest.mark.parametrize( "revision,image_id", [ diff --git a/backend/tests/test_controller_bootstrap_import_host.py b/backend/tests/test_controller_bootstrap_import_host.py index 2d85ea3..dc7aea2 100644 --- a/backend/tests/test_controller_bootstrap_import_host.py +++ b/backend/tests/test_controller_bootstrap_import_host.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib.util import json +import os from pathlib import Path import pytest @@ -41,3 +42,43 @@ def test_bootstrap_passes_only_on_stdin(monkeypatch): assert observed["input"] == b"sensitive-test-value" assert "sensitive-test-value" not in " ".join(observed["argv"]) assert observed["capture_output"] is True + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX metadata and symlinks") +def test_bootstrap_import_rejects_symlinks_and_multiple_links(tmp_path, monkeypatch): + monkeypatch.setattr(host_import.os, "geteuid", lambda: 0) + secret = tmp_path / "bootstrap" + secret.write_bytes(b"x" * 64) + secret.chmod(0o600) + link = tmp_path / "link" + link.symlink_to(secret) + with pytest.raises(ValueError): + host_import.read_bootstrap(link) + link.unlink() + os.link(secret, link) + with pytest.raises(ValueError): + host_import.read_bootstrap(secret) + + +def test_host_import_uses_protected_volume_configuration(): + assert host_import.BOOTSTRAP_FILE.as_posix() == ( + "/var/lib/docker/volumes/itworx-controller-hub-releases/_data/controller-config/" + "appops-enrollment-bootstrap" + ) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX mode checks") +def test_bootstrap_import_rechecks_private_mode_after_open(tmp_path, monkeypatch): + secret = tmp_path / "bootstrap" + secret.write_bytes(b"x" * 64) + secret.chmod(0o600) + monkeypatch.setattr(host_import.os, "geteuid", lambda: 0) + original_open = host_import.os.open + + def change_mode_before_open(path, flags): + secret.chmod(0o644) + return original_open(path, flags) + + monkeypatch.setattr(host_import.os, "open", change_mode_before_open) + with pytest.raises(ValueError, match="changed while opening"): + host_import.read_bootstrap(secret) diff --git a/backend/tests/test_controller_config_paths.py b/backend/tests/test_controller_config_paths.py index ab55257..a226c83 100644 --- a/backend/tests/test_controller_config_paths.py +++ b/backend/tests/test_controller_config_paths.py @@ -1,7 +1,15 @@ from __future__ import annotations +import ast import importlib.util +import json +import os +import subprocess +import sys from pathlib import Path +from types import SimpleNamespace + +import pytest ROOT = Path(__file__).resolve().parents[2] @@ -114,3 +122,64 @@ def test_status_reads_hub_state_through_the_fixed_pinned_adapter(monkeypatch, tm monkeypatch.setattr(mcp_hub_release_store, "read_hub_state_bytes", lambda: b'\xff') assert controller._deployment_state("itworx-mcp-hub") == {} + + +@pytest.mark.skipif(os.name == "nt", reason="the registration script uses POSIX flock") +@pytest.mark.parametrize("matching_aliases", [True, False]) +def test_embedded_registry_writer_uses_protected_aliases_and_preserves_code( + monkeypatch, tmp_path, matching_aliases +): + from backend.app.services.deployment_target_registry import _REGISTER_SCRIPT + + registry = tmp_path / "protected" / "projects.json" + registry.parent.mkdir(mode=0o755) + registry.write_text("{}\n") + registry.chmod(0o600) + readonly = registry if matching_aliases else tmp_path / "different-registry.json" + if not matching_aliases: + readonly.write_text("{}\n") + readonly.chmod(0o600) + code = tmp_path / "old-code" + code.mkdir() + old_registry = code / "projects.json" + old_registry.write_text('{"old":"must stay unchanged"}\n') + original = old_registry.read_bytes() + old_host_registry = tmp_path / "old-host-registry.json" + old_host_registry.write_text("{}\n") + repositories = tmp_path / "repositories" + bare = repositories / "jens" / "example.git" + bare.mkdir(parents=True) + paths = { + "/controller-config/projects.json": readonly, + "/controller-hub-releases/controller-config/projects.json": registry, + "/opt/gitea-deploy/projects.json": old_registry, + "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json": old_host_registry, + "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock": tmp_path / "deploy.lock", + "/gitea-repositories": repositories, + } + + def fixture_path(value): + return paths.get(str(value), Path(value)) + + revision = "a" * 40 + monkeypatch.setattr( + subprocess, "run", lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stdout=revision + "\n") + ) + project = {"branch": "main", "slug": "example", "compose_files": ["docker-compose.yml"]} + monkeypatch.setattr(sys, "argv", ["script", "jens/example", revision, "main", json.dumps(project)]) + tree = ast.parse(_REGISTER_SCRIPT) + # Redirect only filesystem paths into disposable fixtures. All production + # validation, locking, backup, atomic replace and readback code executes. + tree.body = [node for node in tree.body if not (isinstance(node, ast.ImportFrom) and node.module == "pathlib")] + if matching_aliases: + exec(compile(tree, "registry-writer", "exec"), {"Path": fixture_path}) + assert json.loads(registry.read_text()) == {"jens/example": project} + assert registry.stat().st_mode & 0o777 == 0o600 + assert list(registry.parent.glob("projects.json.appops-backup-*")) + else: + with pytest.raises(SystemExit) as error: + exec(compile(tree, "registry-writer", "exec"), {"Path": fixture_path}) + assert error.value.code == 36 + assert registry.read_text() == "{}\n" + assert old_registry.read_bytes() == original + assert old_host_registry.read_text() == "{}\n" diff --git a/backend/tests/test_controller_hub_release_volume.py b/backend/tests/test_controller_hub_release_volume.py index 065a28b..58eb82f 100644 --- a/backend/tests/test_controller_hub_release_volume.py +++ b/backend/tests/test_controller_hub_release_volume.py @@ -57,6 +57,109 @@ def test_only_the_exact_local_volume_is_accepted(): module.validate_volume(volume()) +def writer_inventory(inherited_labels): + identifier = "c" * 64 + spec = module._writer_spec(IMAGE_ID, REVISION, kind="config") + return { + "Id": identifier, "Name": f"/{module.WRITER_NAME}", "Image": IMAGE_ID, + "Config": { + "Image": IMAGE_ID, "User": "0:0", "Entrypoint": ["python3"], "Cmd": spec[-2:], + "Labels": {**inherited_labels, module.WRITER_LABEL: "true", + module.WRITER_TARGET_LABEL: "config", module.WRITER_REVISION_LABEL: REVISION}, + }, + "HostConfig": { + "NetworkMode": "none", "Binds": [], "Privileged": False, "ReadonlyRootfs": True, + "CpuQuota": module.WRITER_CPU_QUOTA, "CpuPeriod": module.WRITER_CPU_PERIOD, + "Memory": module.WRITER_MEMORY_BYTES, "PidsLimit": module.WRITER_PIDS_LIMIT, + "CapDrop": ["ALL"], "Tmpfs": {"/tmp": "rw,nosuid,nodev,size=16777216,mode=1777"}, + "SecurityOpt": ["no-new-privileges:true"], + }, + "Mounts": [{"Type": "volume", "Name": module.VOLUME, + "Source": module.MOUNTPOINT.as_posix(), "Destination": module.TARGET, "RW": True}], + } + + +@pytest.mark.parametrize("damage", [None, "changed_image_label", "extra_label", "reserved_image_label", "falsey_labels"]) +def test_writer_labels_are_bound_to_exact_image_inheritance(monkeypatch, damage): + inherited = {"org.opencontainers.image.revision": "d" * 40, "io.buildah.version": "1.40.0"} + item = writer_inventory(inherited) + if damage == "changed_image_label": + item["Config"]["Labels"]["org.opencontainers.image.revision"] = "e" * 40 + elif damage == "extra_label": + item["Config"]["Labels"]["unexpected"] = "value" + elif damage == "reserved_image_label": + inherited[module.WRITER_LABEL] = "inherited-marker" + elif damage == "falsey_labels": + inherited = [] + + def run(arguments, **kwargs): + if arguments == ["docker", "inspect", item["Id"]]: + return json.dumps([item]).encode() + assert arguments == ["docker", "image", "inspect", IMAGE_ID] + return json.dumps([{"Id": IMAGE_ID, "Config": {"Labels": inherited}}]).encode() + + monkeypatch.setattr(module, "_run", run) + if damage: + with pytest.raises(module.VolumeTransitionError, match="writer_.*invalid"): + module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config") + else: + assert module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config") == item + + +@pytest.mark.parametrize("damage", [None, "tmpfs_projection", "extra_bind", "wrong_volume", "read_only"]) +def test_writer_accepts_declared_tmpfs_without_inspect_mount_projection(monkeypatch, damage): + item = writer_inventory({}) + if damage == "tmpfs_projection": + item["Mounts"].append({"Type": "tmpfs", "Destination": "/tmp"}) + elif damage == "extra_bind": + item["Mounts"].append({"Type": "bind", "Destination": "/foreign"}) + elif damage == "wrong_volume": + item["Mounts"][0]["Name"] = "foreign-volume" + elif damage == "read_only": + item["Mounts"][0]["RW"] = False + + def run(arguments, **kwargs): + if arguments == ["docker", "inspect", item["Id"]]: + return json.dumps([item]).encode() + assert arguments == ["docker", "image", "inspect", IMAGE_ID] + return json.dumps([{"Id": IMAGE_ID, "Config": {"Labels": None}}]).encode() + + monkeypatch.setattr(module, "_run", run) + if damage in {"extra_bind", "wrong_volume", "read_only"}: + with pytest.raises(module.VolumeTransitionError, match="writer_mount_invalid"): + module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config") + else: + module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config") + + +@pytest.mark.parametrize("name", ["foreign-project", "", None, "${COMPOSE_PROJECT_NAME}"]) +def test_source_compose_name_cannot_be_overridden_by_fixed_project_flag(monkeypatch, name): + def never_compose(*args, **kwargs): + pytest.fail("Foreign source project reached Compose") + + monkeypatch.setattr(module, "_run", never_compose) + source = f"name: {json.dumps(name)}\nservices:\n deploy-control:\n image: immutable-controller\n".encode() + with pytest.raises(module.VolumeTransitionError, match="project_invalid"): + module.rendered_model(source, REVISION, IMAGE_ID) + + +def test_compose_without_explicit_name_retains_canonical_project(monkeypatch): + def run(arguments, *, data=None, **kwargs): + if arguments[:3] == ["docker", "image", "inspect"]: + return IMAGE_ID.encode() + normalized = model() + normalized["name"] = ( + arguments[arguments.index("--project-name") + 1] + if "--project-name" in arguments else "data" + ) + return json.dumps(normalized).encode() + + monkeypatch.setattr(module, "_run", run) + original = b"services:\n deploy-control:\n image: immutable-controller\n" + rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID)) + assert rendered["name"] == "gitea-deploy-runners" + + def model(): return { "name": "gitea-deploy-runners", @@ -146,12 +249,30 @@ def test_compose_uses_protected_code_and_two_config_views(compose): ] assert compose_calls assert all("--project-directory" in arguments for arguments in compose_calls) - assert all(module.MOUNTPOINT.as_posix() in arguments for arguments in compose_calls) + assert module.ROOT.as_posix() in compose_calls[0] + assert all(module.MOUNTPOINT.as_posix() in arguments for arguments in compose_calls[1:]) assert all("--env-file" in arguments and "/dev/null" in arguments for arguments in compose_calls) assert all("-f" in arguments and "-" in arguments for arguments in compose_calls) assert calls[0][1] == ORIGINAL_COMPOSE +def test_source_relative_controller_bind_uses_original_compose_directory(monkeypatch): + def run(arguments, *, data=None, **kwargs): + if arguments[:3] == ["docker", "image", "inspect"]: + return IMAGE_ID.encode() + normalized = model() + directory = arguments[arguments.index("--project-directory") + 1] + if arguments[-2:] == ["--format", "json"]: + normalized["services"][module.SERVICE]["volumes"][1]["source"] = directory + "/controller" + return json.dumps(normalized).encode() + + monkeypatch.setattr(module, "_run", run) + original = ORIGINAL_COMPOSE + b" volumes:\n - ./controller:/opt/gitea-deploy:ro\n" + rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID)) + mounts = rendered["services"][module.SERVICE]["volumes"] + assert next(m["source"] for m in mounts if m["target"] == module.EXECUTABLE_TARGET) == module._mount_source_for_revision(REVISION) + + @pytest.mark.parametrize("damage", ["target_collision", "foreign_volume", "graph_rw", "relative_bind", "env_file"]) def test_compose_rejects_unsafe_or_unpinned_activation_inputs(compose, damage): baseline, _ = compose @@ -191,6 +312,55 @@ def test_compose_tag_replacement_cannot_change_controller_startup_image(compose, ) +@pytest.mark.parametrize("damage", [None, "foreign_reference", "wrong_project", "wrong_service", "unhealthy", "digest_reference", "wrong_image", "normalized_reference", "interpolation"]) +def test_drifted_tag_requires_healthy_current_controller_provenance(compose, monkeypatch, damage): + baseline, calls = compose + reference = "registry.example/controller:stable" + if damage == "digest_reference": + reference = "registry.example/controller@sha256:" + "c" * 64 + baseline["services"][module.SERVICE]["image"] = reference + current = { + "Image": IMAGE_ID, + "Config": {"Image": reference, "Labels": { + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.service": module.SERVICE, + }}, + "State": {"Health": {"Status": "healthy"}}, + } + if damage == "foreign_reference": + current["Config"]["Image"] = "registry.example/foreign:stable" + elif damage == "wrong_project": + current["Config"]["Labels"]["com.docker.compose.project"] = "foreign" + elif damage == "wrong_service": + current["Config"]["Labels"]["com.docker.compose.service"] = "foreign" + elif damage == "unhealthy": + current["State"]["Health"]["Status"] = "unhealthy" + elif damage == "wrong_image": + current["Image"] = "sha256:" + "c" * 64 + original_run = module._run + + def drifted_tag(arguments, **kwargs): + if arguments[:3] == ["docker", "image", "inspect"]: + return ("sha256:" + "c" * 64).encode("ascii") + return original_run(arguments, **kwargs) + + monkeypatch.setattr(module, "_run", drifted_tag) + source_reference = reference + if damage == "normalized_reference": + source_reference = "registry.example/foreign:stable" + elif damage == "interpolation": + source_reference = "${CONTROLLER_IMAGE}" + original = f"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: {source_reference}\n".encode() + if damage: + with pytest.raises(module.VolumeTransitionError, match="image_reference_(changed|invalid)"): + module.rendered_model(original, REVISION, IMAGE_ID, current_container=current) + else: + rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID, current_container=current)) + assert rendered["services"][module.SERVICE]["image"] == IMAGE_ID + assert rendered["services"][module.SERVICE]["pull_policy"] == "never" + assert not any("up" in arguments or "pull" in arguments for arguments, _ in calls) + + def test_config_snapshot_has_exact_allowlist_and_broker_token(): entries = { "projects.json": (b"{}", 0o600), @@ -482,7 +652,7 @@ def test_installer_release_order_matches_renderer_contract(): renderer_spec.loader.exec_module(renderer) assert tuple(module.RELEASE_FILES) == tuple(renderer._ARTIFACT_FILES) assert module.RELEASE_FILES[:3] == ( - "controller_paths.py", "mcp_hub_release_store.py", "deploy.py" + "controller_paths.py", "mcp_hub_release_store.py", "hub_permission_preparation.py" ) @@ -566,11 +736,12 @@ def test_private_stage_rejects_symlinked_source_directory(): shutil.rmtree(work) -def test_source_compose_preflight_rejects_host_file_reads_before_compose(compose): +@pytest.mark.parametrize("directive", ["env_file", "label_file"]) +def test_source_compose_preflight_rejects_host_file_reads_before_compose(compose, directive): _baseline, calls = compose unsafe = ( b"name: gitea-deploy-runners\nservices:\n deploy-control:\n" - b" image: x\n env_file: /tmp/secret\n" + + f" image: x\n {directive}: /tmp/secret\n".encode() ) with pytest.raises(module.VolumeTransitionError, match="external_source_invalid"): module.rendered_model(unsafe, REVISION, IMAGE_ID) diff --git a/backend/tests/test_controller_state_reconciliation.py b/backend/tests/test_controller_state_reconciliation.py index d07bfe2..11d775e 100644 --- a/backend/tests/test_controller_state_reconciliation.py +++ b/backend/tests/test_controller_state_reconciliation.py @@ -2,6 +2,8 @@ from __future__ import annotations import importlib.util import os +import re +import shlex import stat from pathlib import Path @@ -211,7 +213,9 @@ def test_upgrade_prepares_and_validates_captured_optional_private_tls_before_sto def test_upgrade_rolls_back_state_before_restoring_or_starting_the_old_writer(): script = UPGRADE_PATH.read_text(encoding="utf-8") - restore = script[script.index("restore_previous() {") : script.index("cleanup() {")] + restore_start = script.index("restore_previous() {") + cleanup_start = script.index("cleanup() {", restore_start) + restore = script[restore_start:cleanup_start] stop = 'if ! python3 "${work}/${PREPARER_NAME}" rollback --backup "$backup" --stop-only; then' rollback = '--rollback' @@ -245,6 +249,12 @@ def test_wrapper_archives_the_fixed_operator_handoff_required_by_root_upgrade(): wrapper = WRAPPER_PATH.read_text(encoding="utf-8") upgrade = UPGRADE_PATH.read_text(encoding="utf-8") archive_line = next(line for line in wrapper.splitlines() if "git archive" in line) + source_paths_match = re.search(r"(?ms)^source_paths=\((.*?)^\)", upgrade) assert "scripts/run_repopulse_adoption.py" in archive_line - assert '"scripts/run_repopulse_adoption.py"' in upgrade + assert source_paths_match is not None + assert "scripts/run_repopulse_adoption.py" in shlex.split(source_paths_match.group(1)) + assert ( + 'install -o root -g root -m 0700 "${script_dir}/run_repopulse_adoption.py" ' + '"${work}/source/controller/run_repopulse_adoption.py"' + ) in upgrade diff --git a/backend/tests/test_deploy_broker.py b/backend/tests/test_deploy_broker.py index 58b74da..06dbe0f 100644 --- a/backend/tests/test_deploy_broker.py +++ b/backend/tests/test_deploy_broker.py @@ -247,6 +247,1429 @@ def _project(tmp_path: Path) -> dict[str, object]: } +def _appops_known_legacy_state() -> dict[str, object]: + old_revision = "fe0772fdf5eb7ad772312c9263560d70ebddb536" + failed_revision = "034c9f844d2455850ba37b31379b7edaa786583e" + image_id = "sha256:2b15f07e5be8e0e51075e282354ca8f9feeda1a4146c1f55313a7dd9e9b74fbe" + stale_container_id = "6632964b26056cb05832bd52cc19a32cb654cdc1e68ec9d569553271e88b1aea" + stale_container = { + "container_id": stale_container_id, + "image_id": image_id, + "image_ref": "unraid-appops-gateway:production", + "container_revision": old_revision, + "image_revision": old_revision, + "image_source": "jens/unraid-appops-gateway", + "image_tree": "89fae3afc35222c2f176cf50f1fcb19e535e2539", + "image_version": "1.8.0-platform-fe0772fdf5eb", + "image_created": "2026-09-29T00:00:00Z", + } + stale_artifacts = {"containers": {"Unraid-AppOps-Gateway": stale_container}} + stale_receipt = { + "schema": "gitea-deploy-receipt.v1", + "repository": "jens/unraid-appops-gateway", + "revision": old_revision, + "tree": stale_container["image_tree"], + "treeDigest": "657e133a772e839b1d917f081b5135f2b985f774aeb825cc7a00f29091b242a9", + "build": {"revision": old_revision, "tree": stale_container["image_tree"]}, + "containers": {"Unraid-AppOps-Gateway": stale_container}, + "live": None, + } + failure = "Existing production containers require a valid healthy live deployment state" + attempt_metadata = { + "builtAt": "2026-10-01T18:38:29Z", + "id": failed_revision[:12], + "repository": "jens/unraid-appops-gateway", + "revision": failed_revision, + "tree": "a" * 40, + "treeDigest": hashlib.sha256(f"git-tree:{'a' * 40}".encode()).hexdigest(), + } + return { + "schema_version": 3, + "repository": "jens/unraid-appops-gateway", + "slug": "unraid-appops-gateway", + "compose_project": "unraid-appops-gateway", + "revision": old_revision, + "live_revision": old_revision, + "live_status": "unknown", + "live_legacy_provenance": False, + "live_artifacts": copy.deepcopy(stale_artifacts), + "live_receipt": copy.deepcopy(stale_receipt), + "rollback_snapshot": { + "revision": old_revision, + "legacy_provenance": False, + "receipt": copy.deepcopy(stale_receipt), + }, + "status": "failed", + "message": failure, + "updated_at_epoch": 1, + "artifacts": copy.deepcopy(stale_artifacts), + "last_attempt": { + "revision": failed_revision, + "status": "failed", + "message": failure, + "updated_at_epoch": 1, + "metadata": attempt_metadata, + }, + "previous_revision": "", + "rollback_restored_revision": "", + } + + +def test_appops_legacy_state_anchor_accepts_only_known_preimage_or_current_green_attempt( + deploy_module, monkeypatch +): + state = _appops_known_legacy_state() + raw = (json.dumps(state, indent=2) + "\n").encode() + monkeypatch.setattr( + deploy_module, + "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", + hashlib.sha256(json.dumps( + state["last_attempt"]["metadata"], sort_keys=True, separators=(",", ":") + ).encode()).hexdigest(), + ) + monkeypatch.setattr( + deploy_module, + "APPOPS_LEGACY_STATE_SHA256", + hashlib.sha256(raw).hexdigest(), + ) + assert deploy_module.appops_legacy_state_anchor( + state, raw, "034c9f844d2455850ba37b31379b7edaa786583e", + protected_main_green=False, + ) + + current = "c" * 40 + rewritten = copy.deepcopy(state) + rewritten["message"] = deploy_module.APPOPS_LEGACY_FAILURE_MESSAGE + rewritten["updated_at_epoch"] = 2 + candidate_tree = "c" * 40 + rewritten["last_attempt"].update({ + "revision": current, "updated_at_epoch": 2, + "metadata": { + "builtAt": "2026-10-02T10:00:00Z", "id": current[:12], + "repository": "jens/unraid-appops-gateway", "revision": current, + "tree": candidate_tree, + "treeDigest": hashlib.sha256(f"git-tree:{candidate_tree}".encode()).hexdigest(), + }, + }) + expected = copy.deepcopy(rewritten) + expected.pop("updated_at_epoch") + expected.pop("last_attempt") + projection = hashlib.sha256( + json.dumps(expected, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_PROJECTION_SHA256", projection) + rewritten_raw = (json.dumps(rewritten, indent=2) + "\n").encode() + assert deploy_module.appops_legacy_state_anchor( + rewritten, rewritten_raw, current, + candidate_metadata=rewritten["last_attempt"]["metadata"], + protected_main_green=True, + ) + assert not deploy_module.appops_legacy_state_anchor( + rewritten, rewritten_raw, current, protected_main_green=False, + ) + + tampered = copy.deepcopy(rewritten) + tampered["rollback_snapshot"]["revision"] = "d" * 40 + tampered_raw = (json.dumps(tampered, indent=2) + "\n").encode() + assert not deploy_module.appops_legacy_state_anchor( + tampered, tampered_raw, current, + candidate_metadata=rewritten["last_attempt"]["metadata"], + protected_main_green=True, + ) + + +def test_appops_legacy_runtime_requires_pinned_source_image_and_resolved_model( + deploy_module, tmp_path +): + project = _project(tmp_path) + project.update({ + "branch": "main", "gitea_repository_id": 9, + "compose_project": "unraid-appops-gateway", + "containers": ["Unraid-AppOps-Gateway"], + "compose_files": ["docker-compose.yml"], + "environment": { + "APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216", + "APPOPS_PUBLISH_HOST": "198.18.10.150", + }, + }) + source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" + old_revision = "5553c04ce497c08a35ed301052dc5e0b02e5d9b4" + image_id = "sha256:9b5eddf32c8eb5361106fc35cc0e056c49d287a20d1fb26490662cc8447f09d4" + container = { + "Id": "72daa9f638defc4f8897d8aad23c4181054b2a1bc04c993078a5ac9597eeb0ec", + "Image": image_id, + "State": {"Running": True, "Health": {"Status": "healthy"}}, + "Config": { + "Image": "unraid-appops-gateway:platform-5553c04ce497", + "Hostname": "unraid-appops-gateway", + "Labels": { + "com.docker.compose.project": "unraid-appops-gateway", + "com.docker.compose.service": "appops-gateway", + "com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml", + "org.opencontainers.image.source": source, + "org.opencontainers.image.revision": old_revision, + "org.opencontainers.image.source-tree": "1624e1748053f998d8427c8c8a4203277d22f13c", + "org.opencontainers.image.version": "1.8.0-platform-5553c04ce497", + "org.opencontainers.image.created": "2026-10-01T16:32:01Z", + }, + }, + "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, + "Mounts": [ + {"Type": "bind", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", + "Destination": "/data", "RW": True}, + {"Type": "bind", "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models", + "Destination": "/mnt/privatemuse-comfyui-models", "RW": True}, + ], + "NetworkSettings": { + "Ports": {"1216/tcp": [{"HostIp": "198.18.10.150", "HostPort": "1216"}]}, + "Networks": { + "itworx-mcp-hub-npm": {}, + "unraid-appops-gateway_default": {}, + }, + }, + } + image = { + "Id": image_id, + "Config": {"Labels": container["Config"]["Labels"]}, + } + + observed = deploy_module.verify_appops_legacy_runtime(project, container, image) + assert observed["imageId"] == image_id + assert observed["imageRef"] == "unraid-appops-gateway:platform-5553c04ce497" + assert observed["sourceRepository"] == source + + wrong = copy.deepcopy(container) + wrong["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "127.0.0.1" + with pytest.raises(deploy_module.DeployError, match="fixed AppOps legacy runtime"): + deploy_module.verify_appops_legacy_runtime(project, wrong, image) + + wrong_image = copy.deepcopy(image) + wrong_image["Id"] = "sha256:" + "0" * 64 + with pytest.raises(deploy_module.DeployError, match="fixed AppOps legacy runtime"): + deploy_module.verify_appops_legacy_runtime(project, container, wrong_image) + + +def test_appops_legacy_intent_is_root_private_and_compare_and_swap_bound( + deploy_module, monkeypatch, tmp_path +): + root = tmp_path / "controller-config" / "appops-legacy-upgrade" + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", root) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_PATH", root / "unraid-appops-gateway.json") + revision = "c" * 40 + tree = "d" * 40 + metadata = { + "id": revision[:12], "repository": "jens/unraid-appops-gateway", + "revision": revision, "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": "2026-10-02T10:00:00.000Z", + } + source_proof = { + "legacyRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, + "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, + "candidateRevision": revision, "composeBlob": "3" * 40, + "dockerfileBlob": "4" * 40, + } + protected_main = { + "semanticSha256": "5" * 64, "revision": revision, + "source": "gitea_webhook", + "validationContext": "Managed validation / full (push)", + "validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000", + "bodySha256": "6" * 64, "validationSha256": "7" * 64, + } + runtime_proof = { + "resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64, + "runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64, + "healthSha256": "c" * 64, + } + backup_receipt = { + "schema": "itworx.appops-data-backup/v1", + "source": deploy_module.APPOPS_LEGACY_DATA_PATH, + "archive": "/srv/example/appdata/unraid-appops-gateway-backups/one/appdata.tar.gz", + "sha256": "d" * 64, "bytes": 12, "requestedRevision": revision, + "previousRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "createdAt": "2026-10-02T10:00:00.000Z", + } + backup = { + **backup_receipt, + "receiptSha256": deploy_module._canonical_json_sha256(backup_receipt), + } + snapshot = { + "revision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "legacy_provenance": True, "receipt": {}, + "compose_containers": ["Unraid-AppOps-Gateway"], + "containers": [{ + "name": "Unraid-AppOps-Gateway", + "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, + "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, + "image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF, + "restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", + "was_running": True, + }], + "captured_at_epoch": 1790884709, + } + payload = deploy_module._appops_legacy_intent_payload( + revision=revision, metadata=metadata, state_sha256="e" * 64, + observed_state_sha256="e" * 64, source_proof=source_proof, + protected_main=protected_main, runtime_proof=runtime_proof, + runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID}, + backup=backup, snapshot=snapshot, + ) + payload["status"] = "prepared" + digest = deploy_module.write_appops_legacy_intent(payload) + loaded, observed_digest = deploy_module.read_appops_legacy_intent() + assert loaded == payload + assert observed_digest == digest + assert root.stat().st_mode & 0o777 == 0o700 + assert deploy_module.APPOPS_LEGACY_INTENT_PATH.stat().st_mode & 0o777 == 0o600 + + captured_snapshot = copy.deepcopy(payload["rollbackSnapshot"]) + updated = deploy_module._appops_legacy_rebind_observed_state(payload, "f" * 64) + assert updated["stateObservedSha256"] == "f" * 64 + assert updated["rollbackSnapshot"] == captured_snapshot + updated["status"] = "snapshot_ready" + next_digest = deploy_module.write_appops_legacy_intent(updated, expected_sha256=digest) + assert next_digest != digest + with pytest.raises(deploy_module.DeployError, match="compare-and-swap"): + deploy_module.write_appops_legacy_intent(payload, expected_sha256=digest) + loaded, _ = deploy_module.read_appops_legacy_intent() + assert loaded["stateObservedSha256"] == "f" * 64 + assert loaded["rollbackSnapshot"] == captured_snapshot + tampered = copy.deepcopy(loaded) + tampered["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] = "f" * 64 + with pytest.raises(deploy_module.DeployError, match="intent and rollback snapshot differ"): + deploy_module.write_appops_legacy_intent(tampered, expected_sha256=next_digest) + + hardlink = root / "hardlink.json" + os.link(deploy_module.APPOPS_LEGACY_INTENT_PATH, hardlink) + with pytest.raises(deploy_module.DeployError, match="unsafe"): + deploy_module.read_appops_legacy_intent() + + +def test_appops_legacy_main_dispatch_uses_prepared_snapshot_backup_and_state_cas( + deploy_module, monkeypatch, tmp_path, capsys +): + repository = "jens/unraid-appops-gateway" + revision = "c" * 40 + project_config = { + "slug": "unraid-appops-gateway", "branch": "main", + "source_dir": str(tmp_path / "source"), + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": ["Unraid-AppOps-Gateway"], + } + projects = {repository: project_config} + metadata = { + "id": revision[:12], "repository": repository, "revision": revision, + "tree": "d" * 40, + "treeDigest": hashlib.sha256(f"git-tree:{'d' * 40}".encode()).hexdigest(), + "builtAt": "2026-10-02T10:00:00.000Z", + } + legacy_state = _appops_known_legacy_state() + raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode() + state_sha256 = hashlib.sha256(raw_state).hexdigest() + post_state = {"repository": repository, "live_revision": revision, "live_status": "healthy"} + events = [] + snapshot = { + "revision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "legacy_provenance": True, + "receipt": {}, + "containers": [{"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497"}], + } + backup = {"sha256": "e" * 64, "receiptSha256": "f" * 64} + intent = {"status": "deploying"} + migration = { + "intent": intent, "intent_sha256": "1" * 64, + "state_sha256": state_sha256, "state": legacy_state, + "raw_state": raw_state, "snapshot": snapshot, "backup": backup, + } + + @contextmanager + def project_lock(path, _description): + events.append(("project-lock", path)) + yield + + @contextmanager + def global_lock(): + events.append(("global-lock",)) + yield + + monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision]) + monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) + monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) + monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) + monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: metadata) + monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock) + monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock) + monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None) + monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state) + monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) + monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) + monkeypatch.setattr(deploy_module, "validate_managed_v1_runtime", lambda *_args: None) + monkeypatch.setattr(deploy_module, "ensure_docker_capacity", lambda: None) + monkeypatch.setattr(deploy_module, "capture_compose_rollback", lambda *_args, **_kwargs: pytest.fail("migration snapshot must be reused")) + monkeypatch.setattr(deploy_module, "capture_appops_data_backup", lambda *_args: pytest.fail("migration backup must be reused")) + + def read_preimage(_project): + events.append(("state-preimage",)) + return (legacy_state, raw_state, state_sha256) if events.count(("state-preimage",)) == 1 else ( + post_state, b"post-state", "2" * 64 + ) + + def prepare(project, _repository, _repo, _revision, _metadata, state, raw, digest): + assert events[-1] == ("state-preimage",) + assert state == legacy_state and raw == raw_state and digest == state_sha256 + assert events.count(("global-lock",)) == 1 + project["_appops_legacy_migration"] = migration + events.append(("prepare",)) + return migration + + monkeypatch.setattr(deploy_module, "read_appops_state_preimage", read_preimage) + monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_appops_legacy_reconcile_pending_rollback", lambda *_args, **_kwargs: False) + monkeypatch.setattr(deploy_module, "_appops_legacy_recover_interrupted_candidate", lambda *_args: False) + monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare) + monkeypatch.setattr(deploy_module, "write_appops_progress", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({}, "a" * 64, "b" * 64)) + monkeypatch.setattr(deploy_module, "deploy_project", lambda *_args: events.append(("deploy",))) + receipt = {"repository": repository, "revision": revision, "tree": metadata["tree"]} + monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt) + monkeypatch.setattr(deploy_module, "finish_compose_migration", lambda *_args: None) + + def write_state(*_args, **kwargs): + events.append(("write-state", kwargs)) + + def write_intent(payload, *, expected_sha256=None): + events.append(("write-intent", payload["status"], expected_sha256)) + return "3" * 64 + + monkeypatch.setattr(deploy_module, "write_state", write_state) + monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", write_intent) + monkeypatch.setattr(deploy_module, "cleanup_rollback_image_tags", lambda *_args, **_kwargs: None) + + assert deploy_module.main() == 0 + assert ("prepare",) in events and ("deploy",) in events + state_event = next(event for event in events if event[0] == "write-state") + assert state_event[1]["expected_state_sha256"] == state_sha256 + assert state_event[1]["rollback_snapshot"] == snapshot + assert events.index(("global-lock",)) < events.index(("prepare",)) < events.index(("deploy",)) + assert ("write-intent", "deployed", "1" * 64) in events + assert json.loads(capsys.readouterr().out)["status"] == "deployed" + + +def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256: str): + repository = "jens/unraid-appops-gateway" + tree = "d" * 40 + old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION + metadata = { + "id": revision[:12], "repository": repository, "revision": revision, + "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": "2026-10-02T10:00:00.000Z", + } + source_proof = { + "legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, + "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, + "candidateRevision": revision, "composeBlob": "3" * 40, + "dockerfileBlob": "4" * 40, + } + protected_main = { + "semanticSha256": "5" * 64, "revision": revision, + "source": "gitea_webhook", + "validationContext": "Managed validation / full (push)", + "validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000", + "bodySha256": "6" * 64, "validationSha256": "7" * 64, + } + runtime_proof = { + "resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64, + "runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64, + "healthSha256": "c" * 64, + } + backup_receipt = { + "schema": "itworx.appops-data-backup/v1", + "source": deploy_module.APPOPS_LEGACY_DATA_PATH, + "archive": "/tmp/appops-rollback-fixture/appdata.tar.gz", + "sha256": "d" * 64, "bytes": 12, "requestedRevision": revision, + "previousRevision": old_revision, "createdAt": "2026-10-02T10:00:00.000Z", + } + backup = { + **backup_receipt, + "receiptSha256": deploy_module._canonical_json_sha256(backup_receipt), + } + snapshot = { + "revision": old_revision, + "legacy_provenance": True, + "receipt": {}, + "compose_containers": ["Unraid-AppOps-Gateway"], + "containers": [{ + "name": "Unraid-AppOps-Gateway", + "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, + "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, + "image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF, + "restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "rollback_ref": ( + "gitea-deploy-rollback/unraid-appops-gateway-0:" + old_revision[:12] + ), + "was_running": True, + }], + "captured_at_epoch": 1790884709, + } + intent = deploy_module._appops_legacy_intent_payload( + revision=revision, metadata=metadata, state_sha256=capture_sha256, + observed_state_sha256=capture_sha256, source_proof=source_proof, + protected_main=protected_main, runtime_proof=runtime_proof, + runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID}, + backup=backup, snapshot=snapshot, status="deploying", + ) + return repository, metadata, source_proof, protected_main, backup, intent + + +def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt( + deploy_module, monkeypatch, tmp_path, capsys +): + repository = "jens/unraid-appops-gateway" + revision = "c" * 40 + old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION + state_root = tmp_path / "state" + state_root.mkdir() + intent_root = tmp_path / "appops-legacy-intent" + monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root) + monkeypatch.setattr( + deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json" + ) + capture_sha256 = hashlib.sha256(b"immutable legacy capture preimage\n").hexdigest() + repository, metadata, source_proof, protected_main, backup, intent = ( + _appops_rollback_intent_fixture(deploy_module, revision, capture_sha256) + ) + backup_root = tmp_path / "backups" + backup_dir = backup_root / "one" + backup_dir.mkdir(parents=True, mode=0o700) + backup_dir.chmod(0o700) + backup_root.chmod(0o700) + archive_path = backup_dir / "appdata.tar.gz" + archive_bytes = b"verified AppOps rollback backup fixture" + archive_path.write_bytes(archive_bytes) + archive_path.chmod(0o600) + monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", backup_root) + backup["archive"] = str(archive_path) + backup["sha256"] = hashlib.sha256(archive_bytes).hexdigest() + backup["bytes"] = len(archive_bytes) + backup_receipt = {key: value for key, value in backup.items() if key != "receiptSha256"} + backup["receiptSha256"] = deploy_module._canonical_json_sha256(backup_receipt) + receipt_path = backup_dir / "receipt.json" + receipt_path.write_text(json.dumps(backup_receipt, sort_keys=True), encoding="utf-8") + receipt_path.chmod(0o600) + intent["dataBackup"] = backup + intent["rollbackSnapshot"]["appops_legacy"]["dataBackupSha256"] = backup["sha256"] + snapshot = intent["rollbackSnapshot"] + project_config = { + "slug": "unraid-appops-gateway", "branch": "main", + "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": ["Unraid-AppOps-Gateway"], + "gitea_repository_id": 9, "verify_oci_revision": True, + "environment": { + "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, + "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, + }, + } + projects = {repository: project_config} + receipt = { + "repository": repository, "revision": revision, "tree": metadata["tree"], + "build": metadata, + } + candidate_artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "sha256:" + "e" * 64}}} + candidate_state = { + "schema_version": 3, "repository": repository, "slug": "unraid-appops-gateway", + "compose_project": "unraid-appops-gateway", "revision": revision, + "live_revision": revision, "live_status": "healthy", + "live_legacy_provenance": False, "live_artifacts": candidate_artifacts, + "live_receipt": receipt, "rollback_snapshot": snapshot, + "status": "success", "message": "verified candidate", + "updated_at_epoch": 100, "artifacts": candidate_artifacts, + "last_attempt": { + "revision": revision, "status": "success", "message": "verified candidate", + "updated_at_epoch": 100, "metadata": metadata, + }, + "previous_revision": old_revision, "rollback_restored_revision": "", + } + state_path = state_root / "unraid-appops-gateway.json" + state_path.write_text(json.dumps(candidate_state, indent=2) + "\n", encoding="utf-8") + state_path.chmod(0o644) + original_snapshot = copy.deepcopy(snapshot) + deploy_module.write_appops_legacy_intent(intent) + + @contextmanager + def no_lock(*_args): + yield + + monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", repository, revision]) + monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) + monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) + monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) + monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata) + monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) + monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) + monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: receipt) + monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) + monkeypatch.setattr( + deploy_module, + "validate_recorded_receipt", + lambda _project, observed, observed_repository, observed_revision: ( + metadata + if observed == receipt and observed_repository == repository + and observed_revision == revision + else pytest.fail("rollback receipt did not match the strict candidate fixture") + ), + ) + monkeypatch.setattr( + deploy_module, "inspect_image", + lambda _ref: {"Id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID}, + ) + monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: { + "containers": {"Unraid-AppOps-Gateway": { + "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, + "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, + "image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "container_revision": old_revision, + "image_revision": old_revision, + "image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0], + "image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE, + "image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION, + "image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED, + }}, + }) + restored = {"runtime": False} + + def restore(project, _snapshot, _repository, _failed_revision): + assert _snapshot == original_snapshot + restored["runtime"] = True + project["_appops_legacy_restored"] = True + + def inspect_restored_runtime(_project, *, require_original_container): + assert require_original_container is False + assert restored["runtime"] is True + return ({}, {}, {}, {}) + + runtime_kinds = [] + + def classify_runtime(_project, observed_intent): + assert observed_intent["rollbackSnapshot"] == original_snapshot + runtime_kinds.append("candidate") + return "candidate" + + monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", classify_runtime) + monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored_runtime) + monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) + monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) + original_verify_backup = deploy_module.verify_appops_data_backup + + def verify_backup(observed, requested, previous): + assert observed == backup + assert requested == revision and previous == old_revision + original_verify_backup(observed, requested, previous) + + monkeypatch.setattr(deploy_module, "verify_appops_data_backup", verify_backup) + monkeypatch.setattr( + deploy_module, "_appops_legacy_tag_snapshot", + lambda observed: observed == original_snapshot or pytest.fail("rollback snapshot changed"), + ) + + monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore) + original_write_intent = deploy_module.write_appops_legacy_intent + fail_after_state_write = {"enabled": True} + + def interrupt_completion(payload, **kwargs): + if payload.get("status") == "rolled_back" and fail_after_state_write["enabled"]: + fail_after_state_write["enabled"] = False + raise OSError("simulated crash after the state CAS") + return original_write_intent(payload, **kwargs) + + monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", interrupt_completion) + + # Reconcile the earlier state-write/intent-write gap against the real + # private intent file. Its rollback snapshot remains the original capture. + monkeypatch.setattr(deploy_module, "read_appops_state_preimage", deploy_module.read_appops_state_preimage) + deploy_module.reconcile_appops_deployed_intent( + _project(tmp_path) | {"_repository_name": repository, "branch": "main"}, + repository, tmp_path, revision, metadata, receipt, + ) + deployed_intent, _ = deploy_module.read_appops_legacy_intent() + candidate_preimage = deploy_module.read_appops_state_preimage(_project(tmp_path))[2] + assert deployed_intent["status"] == "deployed" + assert deployed_intent["stateObservedSha256"] == candidate_preimage + assert deployed_intent["rollbackSnapshot"] == original_snapshot + assert deployed_intent["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] == capture_sha256 + + candidate_raw = state_path.read_bytes() + drifted_state = json.loads(candidate_raw.decode("utf-8")) + drifted_state["message"] = "unexpected external state edit" + state_path.write_text(json.dumps(drifted_state, indent=2) + "\n", encoding="utf-8") + state_path.chmod(0o644) + with pytest.raises(deploy_module.DeployError, match="live state changed"): + deploy_module.main() + still_deployed, _ = deploy_module.read_appops_legacy_intent() + assert still_deployed["status"] == "deployed" + assert runtime_kinds == [] + state_path.write_bytes(candidate_raw) + state_path.chmod(0o644) + + # Verify backup assertions on rollback calls; deployment checkpoint + # reconciliation above is intentionally outside the rollback fixture. + monkeypatch.setattr( + deploy_module, "verify_appops_data_backup", + lambda observed, requested, previous: ( + observed == backup and requested == revision and previous == old_revision + ) or pytest.fail("rollback data backup did not match its immutable intent"), + ) + + with pytest.raises(OSError, match="simulated crash after the state CAS"): + deploy_module.main() + pending_intent, _ = deploy_module.read_appops_legacy_intent() + restored_state, restored_raw, restored_sha256 = deploy_module.read_appops_state_preimage( + _project(tmp_path) + ) + assert pending_intent["status"] == "rollback_pending" + assert restored_state["live_status"] == "healthy" + assert restored_state["live_revision"] == old_revision + assert restored_state["live_legacy_provenance"] is True + assert restored_state["live_receipt"] == {} + assert restored_state["rollback_snapshot"] == {} + assert pending_intent["stateObservedSha256"] == candidate_preimage + assert pending_intent["rollbackSnapshot"] == original_snapshot + assert hashlib.sha256(restored_raw).hexdigest() == restored_sha256 + + # The actual next CLI retry recognizes the exact controller-written + # legacy state and only closes the durable intent checkpoint. + assert deploy_module.main() == 0 + completed_intent, _ = deploy_module.read_appops_legacy_intent() + final_state, final_raw, final_sha256 = deploy_module.read_appops_state_preimage(_project(tmp_path)) + assert completed_intent["status"] == "rolled_back" + assert completed_intent["stateObservedSha256"] == final_sha256 + assert completed_intent["rollbackSnapshot"] == original_snapshot + assert completed_intent["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] == capture_sha256 + assert final_state["live_receipt"] == {} + assert final_state["rollback_snapshot"] == {} + assert hashlib.sha256(final_raw).hexdigest() == final_sha256 + assert runtime_kinds == ["candidate"] + assert json.loads(capsys.readouterr().out)["status"] == "rolled-back" + + +@pytest.mark.parametrize( + "runtime_kind", ["candidate", "absent", "legacy_partial", "unknown"] +) +def test_appops_candidate_crash_recovers_before_prepare_and_fails_forward( + deploy_module, monkeypatch, tmp_path, runtime_kind +): + repository = "jens/unraid-appops-gateway" + revision = "c" * 40 + old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION + state_root = tmp_path / "state" + state_root.mkdir() + intent_root = tmp_path / "appops-legacy-intent" + intent_root.mkdir(mode=0o700) + intent_root.chmod(0o700) + monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root) + monkeypatch.setattr( + deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json" + ) + legacy_state = _appops_known_legacy_state() + raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode() + state_sha256 = hashlib.sha256(raw_state).hexdigest() + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256) + monkeypatch.setattr( + deploy_module, + "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", + deploy_module._canonical_json_sha256(legacy_state["last_attempt"]["metadata"]), + ) + repository, metadata, source_proof, protected_main, backup, intent = ( + _appops_rollback_intent_fixture(deploy_module, revision, state_sha256) + ) + intent["status"] = "deploying" + state_path = state_root / "unraid-appops-gateway.json" + state_path.write_bytes(raw_state) + state_path.chmod(0o644) + deploy_module.write_appops_legacy_intent(intent) + + project_config = { + "slug": "unraid-appops-gateway", "branch": "main", + "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": ["Unraid-AppOps-Gateway"], + "gitea_repository_id": 9, "verify_oci_revision": True, + "environment": { + "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, + "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, + }, + } + projects = {repository: project_config} + original_snapshot = copy.deepcopy(intent["rollbackSnapshot"]) + restored = {"runtime": False, "snapshot": None} + classification_calls = [] + + @contextmanager + def no_lock(*_args): + yield + + def classify(_project, observed_intent): + assert observed_intent["targetRevision"] == revision + assert observed_intent["rollbackSnapshot"] == original_snapshot + classification_calls.append(runtime_kind) + return runtime_kind + + def restore(project, observed_snapshot, _repository, _failed_revision): + assert observed_snapshot == original_snapshot + restored["runtime"] = True + restored["snapshot"] = observed_snapshot + project["_appops_legacy_restored"] = True + + def inspect_restored(_project, *, require_original_container): + assert require_original_container is False + assert restored["runtime"] is True + return ({}, {}, {}, {}) + + old_artifacts = {"containers": {"Unraid-AppOps-Gateway": { + "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, + "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, + "image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "container_revision": old_revision, + "image_revision": old_revision, + "image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0], + "image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE, + "image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION, + "image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED, + }}} + monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision]) + monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) + monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) + monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) + monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata) + monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) + monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) + monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None) + monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state) + monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) + monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) + monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None) + monkeypatch.setattr( + deploy_module, "prepare_appops_legacy_migration", + lambda *_args: pytest.fail("partial AppOps runtime must recover before migration preparation"), + ) + monkeypatch.setattr(deploy_module, "write_appops_progress", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: old_artifacts) + monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", classify) + monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored) + monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) + monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) + monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda observed: ( + observed == original_snapshot or pytest.fail("recovery snapshot was not intent-owned") + )) + monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore) + + if runtime_kind == "unknown": + with pytest.raises(deploy_module.AppOpsLegacyNoMutation): + deploy_module.main() + untouched_intent, _ = deploy_module.read_appops_legacy_intent() + untouched_state, untouched_raw, untouched_sha256 = deploy_module.read_appops_state_preimage( + _project(tmp_path) + ) + assert untouched_intent["status"] == "deploying" + assert untouched_state == legacy_state + assert untouched_raw == raw_state + assert untouched_sha256 == state_sha256 + assert restored["runtime"] is False + assert classification_calls == [runtime_kind] + return + + with pytest.raises(deploy_module.AppOpsLegacyRecoveryComplete): + deploy_module.main() + + rolled_state, rolled_raw, rolled_sha256 = deploy_module.read_appops_state_preimage( + _project(tmp_path) + ) + recovered_intent, _ = deploy_module.read_appops_legacy_intent() + assert classification_calls == [runtime_kind, runtime_kind] + assert restored["snapshot"] == original_snapshot + assert recovered_intent["status"] == "rolled_back" + assert recovered_intent["stateObservedSha256"] == rolled_sha256 + assert recovered_intent["rollbackSnapshot"] == original_snapshot + assert rolled_state["live_revision"] == old_revision + assert rolled_state["live_status"] == "healthy" + assert rolled_state["live_legacy_provenance"] is True + assert rolled_state["live_receipt"] == {} + assert rolled_state["rollback_snapshot"] == {} + assert hashlib.sha256(rolled_raw).hexdigest() == rolled_sha256 + + retry_prepared = [] + runtime_proof = dict(recovered_intent["migrationProof"]["runtime"]) + + def prepare_same_candidate(*_args): + retry_prepared.append(True) + raise deploy_module.AppOpsLegacyNoMutation("same candidate reached safe preparation") + + monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({}, "a" * 64, "b" * 64)) + monkeypatch.setattr(deploy_module, "appops_legacy_runtime_hashes", lambda *_args: runtime_proof) + monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare_same_candidate) + monkeypatch.setattr(deploy_module, "restore_live_source", lambda *_args: None) + with pytest.raises(deploy_module.AppOpsLegacyNoMutation, match="same candidate reached safe preparation"): + deploy_module.main() + assert retry_prepared == [True] + + +def test_appops_forward_failure_rollback_is_pending_before_state_commit_and_resumes( + deploy_module, monkeypatch, tmp_path +): + repository = "jens/unraid-appops-gateway" + revision = "c" * 40 + old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION + missing_docker_root = tmp_path / "missing-docker-root" + assert not missing_docker_root.exists() + monkeypatch.setattr(deploy_module, "DOCKER_ROOT", missing_docker_root) + state_root = tmp_path / "state" + state_root.mkdir() + intent_root = tmp_path / "appops-legacy-intent" + intent_root.mkdir(mode=0o700) + intent_root.chmod(0o700) + monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root) + monkeypatch.setattr( + deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json" + ) + legacy_state = _appops_known_legacy_state() + raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode() + state_sha256 = hashlib.sha256(raw_state).hexdigest() + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256) + monkeypatch.setattr( + deploy_module, + "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", + deploy_module._canonical_json_sha256(legacy_state["last_attempt"]["metadata"]), + ) + repository, metadata, source_proof, protected_main, backup, intent = ( + _appops_rollback_intent_fixture(deploy_module, revision, state_sha256) + ) + intent["status"] = "deploying" + state_path = state_root / "unraid-appops-gateway.json" + state_path.write_bytes(raw_state) + state_path.chmod(0o644) + intent_digest = deploy_module.write_appops_legacy_intent(intent) + + project_config = { + "slug": "unraid-appops-gateway", "branch": "main", + "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": ["Unraid-AppOps-Gateway"], + "gitea_repository_id": 9, "verify_oci_revision": True, + "environment": { + "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, + "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, + }, + } + projects = {repository: project_config} + snapshot = copy.deepcopy(intent["rollbackSnapshot"]) + old_artifacts = {"containers": {"Unraid-AppOps-Gateway": { + "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, + "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, + "image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "container_revision": old_revision, + "image_revision": old_revision, + "image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0], + "image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE, + "image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION, + "image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED, + }}} + runtime_kinds = iter(["legacy", "candidate"]) + restored = {"runtime": False} + progress = [] + + @contextmanager + def no_lock(*_args): + yield + + def prepare(project, _repository, _repo, _revision, _metadata, state, raw, digest): + assert digest == state_sha256 + assert raw == raw_state and state == legacy_state + migration = { + "intent": intent, + "intent_sha256": intent_digest, + "state_sha256": digest, + "state": state, + "raw_state": raw, + "snapshot": snapshot, + "backup": backup, + } + project["_appops_legacy_migration"] = migration + project["_compose_rollback"] = snapshot + project["_appops_data_backup"] = backup + return migration + + def restore(project, observed_snapshot, _repository, _failed_revision): + assert observed_snapshot == snapshot + pending_intent, _ = deploy_module.read_appops_legacy_intent() + assert pending_intent["status"] == "rollback_pending" + restored["runtime"] = True + project["_rollback_restored_revision"] = old_revision + project["_appops_legacy_restored"] = True + + def inspect_restored(_project, *, require_original_container): + assert require_original_container is False + assert restored["runtime"] is True + return ({}, {}, {}, {}) + + monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision]) + monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) + monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) + monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) + monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata) + monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) + monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) + monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None) + monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state) + monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) + monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) + monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None) + monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare) + monkeypatch.setattr(deploy_module, "write_appops_progress", lambda _revision, phase, **_kwargs: progress.append(phase)) + monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: old_artifacts) + monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", lambda *_args: next(runtime_kinds)) + monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored) + monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) + monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) + monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda observed: ( + observed == snapshot or pytest.fail("forward failure did not use the intent-owned snapshot") + )) + monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore) + + capacity_checks = [] + + def skip_docker_capacity_for_rollback_test(): + assert not deploy_module.DOCKER_ROOT.exists() + capacity_checks.append(True) + + monkeypatch.setattr(deploy_module, "ensure_docker_capacity", skip_docker_capacity_for_rollback_test) + deploy_calls = [] + + def fail_deploy(*_args): + deploy_calls.append(revision) + raise RuntimeError("simulated candidate failure") + + monkeypatch.setattr(deploy_module, "deploy_project", fail_deploy) + + original_write_intent = deploy_module.write_appops_legacy_intent + + def crash_before_terminal_intent(payload, **kwargs): + if payload.get("status") == "rolled_back": + observed_state, _raw, _digest = deploy_module.read_appops_state_preimage( + _project(tmp_path) + ) + assert observed_state["status"] == "success" + assert observed_state["live_revision"] == old_revision + raise OSError("simulated crash after rollback state CAS") + return original_write_intent(payload, **kwargs) + + monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", crash_before_terminal_intent) + with pytest.raises(deploy_module.AppOpsLegacyRecoveryPending): + deploy_module.main() + assert capacity_checks == [True] + assert deploy_calls == [revision] + + pending_intent, _ = deploy_module.read_appops_legacy_intent() + pending_state, pending_raw, pending_sha256 = deploy_module.read_appops_state_preimage( + _project(tmp_path) + ) + assert pending_intent["status"] == "rollback_pending" + assert pending_intent["stateObservedSha256"] == state_sha256 + assert pending_intent["rollbackSnapshot"] == snapshot + assert pending_state["status"] == "success" + assert pending_state["live_revision"] == old_revision + assert pending_state["live_status"] == "healthy" + assert pending_state["live_receipt"] == {} + assert hashlib.sha256(pending_raw).hexdigest() == pending_sha256 + assert progress[-2:] == ["rolling_back", "outcome_unknown"] + + monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", original_write_intent) + with pytest.raises(deploy_module.AppOpsLegacyRecoveryComplete): + deploy_module.main() + completed_intent, _ = deploy_module.read_appops_legacy_intent() + assert completed_intent["status"] == "rolled_back" + assert completed_intent["stateObservedSha256"] == pending_sha256 + + +def test_appops_durable_rollback_requires_completed_intent_and_verified_backup( + deploy_module, monkeypatch, tmp_path +): + revision = "c" * 40 + tree = "d" * 40 + repository = "jens/unraid-appops-gateway" + metadata = { + "id": revision[:12], "repository": repository, "revision": revision, + "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": "2026-10-02T10:00:00.000Z", + } + source_proof = { + "legacyRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, + "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, + "candidateRevision": revision, "composeBlob": "3" * 40, + "dockerfileBlob": "4" * 40, + } + protected_main = { + "semanticSha256": "5" * 64, "revision": revision, + "source": "gitea_webhook", + "validationContext": "Managed validation / full (push)", + "validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000", + "bodySha256": "6" * 64, "validationSha256": "7" * 64, + } + runtime_proof = { + "resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64, + "runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64, + "healthSha256": "c" * 64, + } + backup_receipt = { + "schema": "itworx.appops-data-backup/v1", + "source": deploy_module.APPOPS_LEGACY_DATA_PATH, + "archive": "/srv/example/appdata/unraid-appops-gateway-backups/one/appdata.tar.gz", + "sha256": "d" * 64, "bytes": 12, "requestedRevision": revision, + "previousRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "createdAt": "2026-10-02T10:00:00.000Z", + } + backup = { + **backup_receipt, + "receiptSha256": deploy_module._canonical_json_sha256(backup_receipt), + } + snapshot = { + "revision": deploy_module.APPOPS_LEGACY_OLD_REVISION, + "legacy_provenance": True, "receipt": {}, + "compose_containers": ["Unraid-AppOps-Gateway"], + "containers": [{ + "name": "Unraid-AppOps-Gateway", + "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, + "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, + "image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF, + "restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", + "was_running": True, + }], + "captured_at_epoch": 1790884709, + } + intent = deploy_module._appops_legacy_intent_payload( + revision=revision, metadata=metadata, state_sha256="e" * 64, + observed_state_sha256="e" * 64, source_proof=source_proof, + protected_main=protected_main, runtime_proof=runtime_proof, + runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID}, + backup=backup, snapshot=snapshot, status="deployed", + ) + project = _project(tmp_path) + project.update({"branch": "main", "_repository_name": repository}) + current_receipt = { + "repository": repository, "revision": revision, "tree": tree, + "build": metadata, + } + state = { + "schema_version": 3, "repository": repository, + "revision": revision, "live_revision": revision, + "live_status": "healthy", "live_legacy_provenance": False, + "live_receipt": current_receipt, "rollback_snapshot": intent["rollbackSnapshot"], + } + monkeypatch.setattr(deploy_module, "read_state", lambda _project: state) + monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "1" * 64)) + monkeypatch.setattr(deploy_module, "validate_recorded_receipt", lambda *_args: metadata) + backup_checks = [] + monkeypatch.setattr( + deploy_module, + "verify_appops_data_backup", + lambda observed, requested, previous: backup_checks.append((observed, requested, previous)), + ) + monkeypatch.setattr( + deploy_module, + "inspect_image", + lambda _ref: {"Id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID}, + ) + + assert deploy_module.durable_rollback_snapshot(project, repository, revision) == intent["rollbackSnapshot"] + assert backup_checks == [(backup, revision, deploy_module.APPOPS_LEGACY_OLD_REVISION)] + + intent["status"] = "rolled_back" + with pytest.raises(deploy_module.DeployError, match="completed migration intent"): + deploy_module.durable_rollback_snapshot(project, repository, revision) + + +def test_appops_candidate_crash_reconcile_writes_only_from_exact_anchored_state( + deploy_module, monkeypatch, tmp_path +): + repository = "jens/unraid-appops-gateway" + revision = "c" * 40 + tree = "d" * 40 + metadata = { + "id": revision[:12], "repository": repository, "revision": revision, + "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": "2026-10-02T10:00:00.000Z", + } + state = _appops_known_legacy_state() + raw = (json.dumps(state, indent=2) + "\n").encode() + state_sha256 = hashlib.sha256(raw).hexdigest() + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", hashlib.sha256( + json.dumps(state["last_attempt"]["metadata"], sort_keys=True, separators=(",", ":")).encode() + ).hexdigest()) + protected_main = {"revision": revision, "delivery": "same-green-main"} + source_proof = {"candidateRevision": revision, "databaseTree": "1" * 40} + intent = { + "status": "deploying", "targetRevision": revision, + "targetMetadata": metadata, "stateObservedSha256": state_sha256, + "migrationProof": {"protectedMain": protected_main, "source": source_proof}, + "rollbackSnapshot": {}, "dataBackup": {}, + } + project = _project(tmp_path) + project["_repository_path"] = str(tmp_path) + receipt = { + "repository": repository, "revision": revision, "tree": tree, + "build": metadata, + } + writes = [] + intent_writes = [] + monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "a" * 64)) + monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) + monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) + monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) + monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({"name": project["compose_project"]}, "b" * 64, "c" * 64)) + monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_validate_appops_legacy_snapshot", lambda *_args: None) + monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda *_args: None) + monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt) + monkeypatch.setattr(deploy_module, "write_state", lambda *args, **kwargs: writes.append((args, kwargs))) + monkeypatch.setattr(deploy_module, "read_appops_state_preimage", lambda _project: (state, raw, state_sha256)) + monkeypatch.setattr( + deploy_module, + "write_appops_legacy_intent", + lambda payload, **kwargs: intent_writes.append((payload, kwargs)) or "d" * 64, + ) + + observed = deploy_module.reconcile_appops_legacy_candidate( + project, repository, tmp_path, revision, metadata, state, raw, state_sha256, + ) + assert observed == receipt + assert len(writes) == 1 and writes[0][1]["expected_state_sha256"] == state_sha256 + assert writes[0][1]["rollback_snapshot"] == {} + assert intent_writes[0][0]["status"] == "deployed" + writes.clear() + + changed_raw = raw + b" " + assert deploy_module.reconcile_appops_legacy_candidate( + project, repository, tmp_path, revision, metadata, state, changed_raw, + hashlib.sha256(changed_raw).hexdigest(), + ) is None + assert writes == [] + + +def test_appops_state_write_rejects_compare_and_swap_drift(deploy_module, monkeypatch, tmp_path): + state_root = tmp_path / "state" + state_root.mkdir() + monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) + project = _project(tmp_path) + path = state_root / "unraid-appops-gateway.json" + original = json.dumps({"schema_version": 3, "repository": "jens/unraid-appops-gateway"}).encode() + path.write_bytes(original) + path.chmod(0o644) + + with pytest.raises(deploy_module.DeployError, match="compare-and-swap"): + deploy_module.write_state( + "jens/unraid-appops-gateway", project, "c" * 40, "failed", "blocked", + expected_state_sha256="0" * 64, + ) + assert path.read_bytes() == original + + +def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint( + deploy_module, tmp_path, monkeypatch +): + repository = tmp_path / "source.git" + working = tmp_path / "work" + working.mkdir() + subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True) + subprocess.run(["git", "init", str(working)], check=True, capture_output=True) + subprocess.run(["git", "-C", str(working), "config", "user.name", "Fixture"], check=True) + subprocess.run(["git", "-C", str(working), "config", "user.email", "fixture@example.invalid"], check=True) + db = working / "backend" / "app" / "db" + db.mkdir(parents=True) + entrypoint = working / "docker" / "entrypoint.sh" + entrypoint.parent.mkdir(parents=True) + (db / "models.py").write_text("MODEL = 1\n") + (db / "init_db.py").write_text("INIT = 1\n") + (db / "session.py").write_text("SESSION = 1\n") + entrypoint.write_text("#!/bin/sh\nexec app\n") + subprocess.run(["git", "-C", str(working), "add", "."], check=True) + subprocess.run(["git", "-C", str(working), "commit", "-m", "legacy"], check=True, capture_output=True) + old_revision = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip() + subprocess.run(["git", "-C", str(working), "remote", "add", "origin", str(repository)], check=True) + subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True) + (working / "backend" / "app" / "core").mkdir(parents=True) + (working / "backend" / "app" / "core" / "config.py").write_text("STATS_KEY = 1\n") + subprocess.run(["git", "-C", str(working), "add", "."], check=True) + subprocess.run(["git", "-C", str(working), "commit", "-m", "non-db stats config"], check=True, capture_output=True) + candidate = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip() + subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", old_revision) + monkeypatch.setattr( + deploy_module, + "APPOPS_LEGACY_OLD_TREE", + subprocess.check_output( + ["git", "-C", str(working), "rev-parse", f"{old_revision}^{{tree}}"], + text=True, + ).strip(), + ) + assert deploy_module.verify_appops_schema_compatibility(repository, candidate)["databaseTree"] + + (db / "migrations").mkdir() + (db / "migrations" / "001.sql").write_text("DROP TABLE app;\n") + subprocess.run(["git", "-C", str(working), "add", "."], check=True) + subprocess.run(["git", "-C", str(working), "commit", "-m", "schema migration"], check=True, capture_output=True) + changed = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip() + subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True) + with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"): + deploy_module.verify_appops_schema_compatibility(repository, changed) + + +def test_appops_migration_requires_private_exact_full_green_delivery( + deploy_module, monkeypatch, tmp_path +): + revision = "c" * 40 + state_root = tmp_path / "state" + inbox = state_root / "webhook-inbox" + inbox.mkdir(parents=True, mode=0o700) + inbox.chmod(0o700) + monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) + semantic = deploy_module._canonical_json_sha256({ + "gitea_repository_id": 9, "ref": "refs/heads/main", "revision": revision, + }) + record = { + "schema": "gitea-webhook-inbox.v2", "semantic_sha256": semantic, + "body_sha256": "a" * 64, "delivery": "12345678-1234-4234-8234-123456789012", + "gitea_repository_id": 9, "ref": "refs/heads/main", + "repository": "jens/unraid-appops-gateway", "revision": revision, + "source": "typed_self_upgrade", "status": "deploying", + "validation": { + "validation_gate_version": 1, + "required_contexts": ["Managed validation / full (push)"], + "timeout_seconds": 1800, "poll_interval_seconds": 5, + "max_poll_interval_seconds": 30, "last_result": "pass", + "observed": [{"context": "Managed validation / full (push)", "status": "success"}], + }, + "audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"}], + } + marker = inbox / f"{semantic}.json" + marker.write_text(json.dumps(record), encoding="utf-8") + marker.chmod(0o600) + proof = deploy_module.verify_appops_green_delivery(revision) + assert proof["validationContext"] == "Managed validation / full (push)" + + # The webhook worker atomically makes this same immutable record terminal + # after deploy.py has already committed the success state. A restarted + # broker must still be able to finish the protected intent checkpoint. + record["status"] = "deployed" + marker.write_text(json.dumps(record), encoding="utf-8") + marker.chmod(0o600) + assert deploy_module.verify_appops_green_delivery(revision) == proof + + record["validation"]["observed"][0]["status"] = "pending" + marker.write_text(json.dumps(record), encoding="utf-8") + marker.chmod(0o600) + with pytest.raises(deploy_module.DeployError, match="protected full-green"): + deploy_module.verify_appops_green_delivery(revision) + + record["status"] = "failed" + record["validation"]["observed"][0]["status"] = "success" + marker.write_text(json.dumps(record), encoding="utf-8") + marker.chmod(0o600) + with pytest.raises(deploy_module.DeployError, match="protected full-green"): + deploy_module.verify_appops_green_delivery(revision) + + marker.unlink() + marker.symlink_to(tmp_path / "untrusted.json") + with pytest.raises(deploy_module.DeployError, match="unsafe"): + deploy_module.verify_appops_green_delivery(revision) + + +def test_appops_retry_completes_pending_intent_after_delivery_becomes_deployed( + deploy_module, monkeypatch, tmp_path +): + repository = "jens/unraid-appops-gateway" + revision = "c" * 40 + tree = "d" * 40 + metadata = { + "id": revision[:12], "repository": repository, "revision": revision, + "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": "2026-10-02T10:00:00.000Z", + } + protected_main = {"revision": revision, "delivery": "green-main-delivery"} + source_proof = {"candidateRevision": revision, "databaseTree": "1" * 40} + snapshot = {"revision": deploy_module.APPOPS_LEGACY_OLD_REVISION} + receipt = {"repository": repository, "revision": revision, "tree": tree, "build": metadata} + state = { + "repository": repository, "live_revision": revision, "live_status": "healthy", + "live_legacy_provenance": False, "live_receipt": receipt, + "rollback_snapshot": snapshot, + } + state_sha256 = "a" * 64 + intent = { + "status": "deploying", "targetRevision": revision, "targetMetadata": metadata, + "migrationProof": {"protectedMain": protected_main, "source": source_proof}, + "rollbackSnapshot": snapshot, "dataBackup": {"sha256": "b" * 64}, + } + project = _project(tmp_path) + project.update({"_repository_name": repository, "branch": "main"}) + intent_writes = [] + + monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "c" * 64)) + monkeypatch.setattr( + deploy_module, "read_appops_state_preimage", + lambda _project: (state, b"controller-committed-success", state_sha256), + ) + monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) + monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) + monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) + monkeypatch.setattr( + deploy_module, "write_appops_legacy_intent", + lambda payload, **kwargs: intent_writes.append((payload, kwargs)) or "d" * 64, + ) + + # The state-write/intention-write crash leaves the original deploying + # intent, while the webhook worker subsequently marks its durable green + # record deployed. The current-receipt retry must only close that gap. + deploy_module.reconcile_appops_deployed_intent( + project, repository, tmp_path, revision, metadata, receipt, + ) + + assert len(intent_writes) == 1 + assert intent_writes[0][0]["status"] == "deployed" + assert intent_writes[0][0]["stateObservedSha256"] == state_sha256 + assert intent_writes[0][1]["expected_sha256"] == "c" * 64 + + def _workmesh_product_project(tmp_path: Path) -> dict[str, object]: return { "slug": "workmesh", @@ -3476,6 +4899,7 @@ def test_health_is_served_while_initial_runner_maintenance_is_blocked( return super().serve_forever(*args, **kwargs) monkeypatch.setattr(status_module, "RunnerFleetService", BlockingFleet) + monkeypatch.setattr(status_module, "runner_fleet_specialization_state", lambda: "legacy") monkeypatch.setattr(status_module, "BoundedThreadingHTTPServer", ObservedServer) monkeypatch.setattr( status_module, @@ -5544,9 +6968,12 @@ def test_controller_rejects_a_group_writable_projects_registry(status_module, tm status_module._load_projects() -def test_repopulse_capability_requires_fixed_readonly_and_rw_mount_aliases(status_module): - readonly_root = status_module._repopulse_adoption_store.CONTROLLER_READONLY_ROOT - rw_root = status_module._repopulse_adoption_store.CONTROLLER_RW_ROOT +def test_repopulse_capability_requires_fixed_readonly_and_rw_mount_aliases(status_module, monkeypatch): + # Exercise the installed aliases, rather than the shared source-tree fallback. + readonly_root = Path("/controller-config") + rw_root = Path("/controller-hub-releases/controller-config") + monkeypatch.setattr(status_module._repopulse_adoption_store, "CONTROLLER_READONLY_ROOT", readonly_root) + monkeypatch.setattr(status_module._repopulse_adoption_store, "CONTROLLER_RW_ROOT", rw_root) valid_mountinfo = ( f"25 1 0:2 / {readonly_root.as_posix()} ro - ext4 /dev/test ro\n" f"26 1 0:2 / {rw_root.as_posix()} rw - ext4 /dev/test rw\n" @@ -5783,7 +7210,7 @@ def test_direct_appops_hub_record_is_denied_before_project_or_deploy_access( assert record["reason"] == "canonical_hub_push_required" -def test_hub_protected_push_worker_remains_the_fixed_deploy_path( +def test_hub_protected_push_waits_for_candidate_before_fixed_deploy_path( status_module, tmp_path, monkeypatch ): repository = "jens/itworx_mcp_hub" @@ -5837,8 +7264,15 @@ def test_hub_protected_push_worker_remains_the_fixed_deploy_path( ), ) + monkeypatch.setattr(status_module, "_prepare_hub_candidate_delivery", lambda *_args: False) status_module._process_webhook_record(marker) + waiting = json.loads(marker.read_text(encoding="utf-8")) + assert waiting["status"] == "validation_passed" + assert "deployment_started" not in [entry["stage"] for entry in waiting["audit"]] + assert calls == [] + monkeypatch.setattr(status_module, "_prepare_hub_candidate_delivery", lambda *_args: True) + status_module._process_webhook_record(marker) assert json.loads(marker.read_text(encoding="utf-8"))["status"] == "deployed" assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", repository, revision]] diff --git a/backend/tests/test_deployment_target_registry.py b/backend/tests/test_deployment_target_registry.py index c372b79..3b7a2a4 100644 --- a/backend/tests/test_deployment_target_registry.py +++ b/backend/tests/test_deployment_target_registry.py @@ -87,7 +87,7 @@ def test_derives_bounded_single_container_topology_and_uses_fixed_exec(): assert '"source_dir":"/mnt/cache/appdata/gitea-deploy-runners/sources/certscope"' in command[6] assert '"containers":["certscope"]' in command[6] script = command[2] - assert '/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json' in script + assert '/controller-hub-releases/controller-config/projects.json' in script assert '/srv/example/appdata/gitea-deploy-runners/deploy.lock' not in script assert '/mnt/cache/appdata/gitea-deploy-runners/deploy.lock' in script assert 'os.path.samefile(path, host_path)' in script diff --git a/backend/tests/test_gitea_actions.py b/backend/tests/test_gitea_actions.py index 44791dc..eda5b8c 100644 --- a/backend/tests/test_gitea_actions.py +++ b/backend/tests/test_gitea_actions.py @@ -95,6 +95,63 @@ def test_runner_snapshot_recommends_only_approval_gated_restart_for_stalled_queu assert "fixture-secret" not in repr(result) +def test_pending_jobs_do_not_trigger_runnable_queue_diagnostics(monkeypatch): + _configure(monkeypatch) + monkeypatch.setitem( + settings.apis, + "GITEA_RUNNER_TARGETS_JSON", + '{"itworx-ci-build":"itworx-security-runner"}', + ) + connector = GiteaActionsConnector() + created_at = (datetime.now(UTC) - timedelta(minutes=15)).isoformat() + requested_statuses = [] + + def fake_request(path, params=None): + if path.endswith("/runners"): + return { + "total_count": 1, + "runners": [ + { + "id": 1, + "name": "itworx-security-runner", + "status": "online", + "busy": False, + "disabled": False, + "labels": ["linux-validation"], + } + ], + } + status = params["status"] + requested_statuses.append(status) + if status == "pending": + return { + "total_count": 1, + "jobs": [ + { + "id": 171, + "name": "conditional release full", + "status": "requested", + "created_at": created_at, + "labels": ["itworx-ci-build"], + } + ], + } + return {"total_count": 0, "jobs": []} + + monkeypatch.setattr(connector, "_request", fake_request) + result = connector.snapshot(refresh=True) + + assert requested_statuses == ["queued", "pending", "in_progress"] + assert result["severity"] == "healthy" + assert result["queue"]["queued"] == 0 + assert result["queue"]["oldestQueuedAgeSeconds"] is None + assert result["queue"]["sampleTruncated"] is False + assert result["queue"]["blockedByRunnerLabels"] == [] + assert result["queue"]["staleAssignmentRunnerLabels"] == [] + assert result["queue"]["items"] == [] + assert result["recommendedAction"] is None + + def test_runner_snapshot_is_cached_and_healthy_when_queue_is_clear(monkeypatch): _configure(monkeypatch) connector = GiteaActionsConnector() diff --git a/backend/tests/test_host_guard_memory_threshold_installer.py b/backend/tests/test_host_guard_memory_threshold_installer.py new file mode 100644 index 0000000..2228279 --- /dev/null +++ b/backend/tests/test_host_guard_memory_threshold_installer.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import contextlib +import hashlib +import importlib.util +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +INSTALLER_PATH = ( + ROOT + / "ops" + / "gitea-deploy-broker" + / "tune_host_guard_memory_thresholds.py" +) +SPEC = importlib.util.spec_from_file_location( + "host_guard_memory_threshold_installer_test", INSTALLER_PATH +) +assert SPEC and SPEC.loader +installer = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(installer) + + +@pytest.fixture +def guard_script() -> bytes: + return b"""#!/usr/bin/env bash +set -euo pipefail +readonly MEM_PAUSE_KB=4194304 +readonly MEM_RESUME_KB=6291456 +readonly SWAP_CRITICAL_MEMORY_KB=8388608 +readonly SWAP_PAUSE_KBPS=1024 +readonly SWAP_RESUME_KBPS=256 +readonly SWAP_CRITICAL_SAMPLES=3 +if [ \"$mem_kb\" -lt \"$MEM_PAUSE_KB\" ]; then + runner_critical=1 +fi +""" + + +def _baseline(monkeypatch, payload: bytes) -> None: + monkeypatch.setattr(installer, "BASELINE_SHA256", hashlib.sha256(payload).hexdigest()) + + +def _assignment_values(payload: bytes) -> dict[str, int]: + rows = re.findall(rb"(?m)^readonly ([A-Z0-9_]+)=([0-9]+)$", payload) + return {name.decode("ascii"): int(value) for name, value in rows} + + +def _fixture_install(monkeypatch, tmp_path, guard_script, *, fsync_fails=False): + target = tmp_path / "script" + backup = tmp_path / ".script.backup" + lock = tmp_path / ".lock" + target.write_bytes(guard_script) + _baseline(monkeypatch, guard_script) + monkeypatch.setattr(installer, "TARGET_PATH", target) + monkeypatch.setattr(installer, "BACKUP_PATH", backup) + monkeypatch.setattr(installer, "LOCK_PATH", lock) + monkeypatch.setattr(installer, "_assert_fixed_environment", lambda: target.stat()) + monkeypatch.setattr(installer, "_exclusive_lock", contextlib.nullcontext) + monkeypatch.setattr( + installer, + "_read_private_backup", + lambda: backup.read_bytes() if backup.exists() else None, + ) + + events: list[str] = [] + + def assert_private_file(path, *, allow_missing=False): + if path == backup: + events.append("backup_identity") + elif path == target: + events.append("target_identity") + return path.lstat() if path.exists() else None + + def write_backup(path, payload): + events.append("backup_create") + with path.open("xb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + + def write_stage(payload): + stage = tmp_path / f".stage-{len(events)}" + stage.write_bytes(payload) + return stage + + def fsync_directory(_path): + events.append("directory_fsync") + if fsync_fails: + raise OSError("fixture directory fsync failure") + + def atomic_replace(stage, destination): + events.append("replace") + os.replace(stage, destination) + events.append("post_replace_fsync") + + original_read_bytes = Path.read_bytes + + def traced_read_bytes(path): + payload = original_read_bytes(path) + if path == backup: + events.append("backup_readback") + return payload + + monkeypatch.setattr(installer, "_assert_private_regular_file", assert_private_file) + monkeypatch.setattr(installer, "_write_private_file_exclusive", write_backup) + monkeypatch.setattr(installer, "_write_stage", write_stage) + monkeypatch.setattr(installer, "_bash_syntax_check", lambda _path: None) + monkeypatch.setattr(installer, "_fsync_directory", fsync_directory) + monkeypatch.setattr(installer, "_atomic_replace", atomic_replace) + monkeypatch.setattr(Path, "read_bytes", traced_read_bytes) + return target, backup, events + + +def test_fixture_patch_changes_only_reviewed_memory_literals(guard_script, monkeypatch): + _baseline(monkeypatch, guard_script) + + candidate = installer._render_candidate(guard_script) + + before = _assignment_values(guard_script) + after = _assignment_values(candidate) + assert after == { + **before, + "MEM_PAUSE_KB": 2_097_152, + "MEM_RESUME_KB": 4_194_304, + "SWAP_CRITICAL_MEMORY_KB": 4_194_304, + } + changed = [ + (old.decode("ascii"), new.decode("ascii")) + for old, new in zip(guard_script.splitlines(), candidate.splitlines()) + if old != new + ] + assert changed == [ + ("readonly MEM_PAUSE_KB=4194304", "readonly MEM_PAUSE_KB=2097152"), + ("readonly MEM_RESUME_KB=6291456", "readonly MEM_RESUME_KB=4194304"), + ( + "readonly SWAP_CRITICAL_MEMORY_KB=8388608", + "readonly SWAP_CRITICAL_MEMORY_KB=4194304", + ), + ] + + +def test_candidate_passes_real_bash_syntax(guard_script, monkeypatch, tmp_path): + bash = shutil.which("bash") + if not bash: + pytest.skip("bash is required for this fixture syntax check") + _baseline(monkeypatch, guard_script) + candidate = installer._render_candidate(guard_script) + fixture = tmp_path / "Host_Resource_Guard.script" + fixture.write_bytes(candidate) + + fixture_argument = str(fixture) + if os.name == "nt": + # Windows bash.exe here is the WSL launcher; translate C:\... to /mnt/c/... + drive = fixture.drive.rstrip(":").lower() + fixture_argument = f"/mnt/{drive}/{fixture.as_posix()[3:]}" + completed = subprocess.run( + [bash, "-n", fixture_argument], check=False, capture_output=True, timeout=15 + ) + + assert completed.returncode == 0, completed.stderr.decode("utf-8", errors="replace") + + +def test_baseline_fingerprint_mismatch_refuses_candidate(guard_script, monkeypatch): + _baseline(monkeypatch, guard_script) + changed = guard_script + b"# unreviewed source change\n" + + with pytest.raises(installer.InstallError, match="baseline_fingerprint_mismatch"): + installer._render_candidate(changed) + + +def test_duplicate_or_changed_memory_literal_refuses_candidate(guard_script, monkeypatch): + duplicate = guard_script + b"readonly MEM_PAUSE_KB=4194304\n" + _baseline(monkeypatch, duplicate) + + with pytest.raises(installer.InstallError, match="threshold_shape_invalid:MEM_PAUSE_KB"): + installer._render_candidate(duplicate) + + wrong_rate = guard_script.replace( + b"readonly SWAP_RESUME_KBPS=256", b"readonly SWAP_RESUME_KBPS=512" + ) + _baseline(monkeypatch, wrong_rate) + with pytest.raises(installer.InstallError, match="threshold_shape_invalid:SWAP_RESUME_KBPS"): + installer._render_candidate(wrong_rate) + + +def test_backup_is_readback_verified_and_directory_synced_before_replace( + guard_script, monkeypatch, tmp_path +): + target, backup, events = _fixture_install( + monkeypatch, tmp_path, guard_script + ) + + status, after_sha = installer.install() + + assert status == "applied" + assert hashlib.sha256(target.read_bytes()).hexdigest() == after_sha + assert backup.read_bytes() == guard_script + replace_at = events.index("replace") + assert events.index("backup_create") < events.index("backup_readback") + assert events.index("backup_readback") < events.index("directory_fsync") + assert events.index("directory_fsync") < max( + index + for index, event in enumerate(events[:replace_at]) + if event == "target_identity" + ) < replace_at + assert events.index("post_replace_fsync") > replace_at + + +def test_backup_directory_sync_failure_leaves_target_unmodified( + guard_script, monkeypatch, tmp_path +): + target, backup, events = _fixture_install( + monkeypatch, tmp_path, guard_script, fsync_fails=True + ) + + with pytest.raises(installer.InstallError, match="backup_directory_sync_failed"): + installer.install() + + assert target.read_bytes() == guard_script + assert backup.read_bytes() == guard_script + assert "replace" not in events diff --git a/backend/tests/test_hub_candidate_delivery.py b/backend/tests/test_hub_candidate_delivery.py new file mode 100644 index 0000000..1395308 --- /dev/null +++ b/backend/tests/test_hub_candidate_delivery.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +import types +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def controller(monkeypatch): + path = ROOT / "ops/gitea-deploy-broker/status_server.py" + spec = importlib.util.spec_from_file_location("hub_candidate_delivery_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "_gitea_validation_settings", + lambda: ("https://gitea.example.com/api/v1", b"x" * 40)) + return module + + +def test_candidate_api_reads_only_fixed_hub_paths(controller, monkeypatch): + opened = [] + + def run(argv, **kwargs): + opened.append((argv, kwargs)) + assert argv == [sys.executable, str(controller.ROOT / "hub_candidate_transport.py"), "json"] + assert kwargs["timeout"] == 20 + assert kwargs["pass_fds"] == () + assert kwargs["stderr"] == controller.subprocess.DEVNULL + assert set(kwargs["env"]) == {"PATH", "PYTHONDONTWRITEBYTECODE", "PYTHONIOENCODING"} + return types.SimpleNamespace(returncode=0, stdout=b'{"workflows":[]}') + + monkeypatch.setattr(controller.subprocess, "run", run) + prefix = "/repos/jens/itworx_mcp_hub" + assert controller._hub_candidate_api_json(prefix + "/actions/workflows", query={}) == {"workflows": []} + payload = json.loads(opened[0][1]["input"]) + assert payload["url"] == "https://gitea.example.com/api/v1" + prefix + "/actions/workflows" + assert payload["outputFd"] is None + assert payload["headers"]["Authorization"] == "token " + "x" * 40 + assert all("x" * 40 not in item for item in opened[0][0]) + for path, query in ( + ("https://other.invalid/api/v1", {}), + (prefix + "/../other/actions/workflows", {}), + (prefix + "/actions/runs", {"event": "pull_request"}), + (prefix + "/actions/runs/1/jobs", {"page": "11", "limit": "100"}), + (prefix + "/actions/runs/1/artifacts", {"name": "other-artifact"}), + ): + with pytest.raises(controller._HubCandidateTransportError): + controller._hub_candidate_api_json(path, query=query) + assert len(opened) == 1 + + +@pytest.mark.parametrize("raw", [b'{"id":1,"id":2}', b"x" * (2 * 1024 * 1024 + 1)]) +def test_candidate_metadata_is_bounded_and_rejects_duplicate_keys(controller, monkeypatch, raw): + monkeypatch.setattr(controller.subprocess, "run", lambda *_args, **_kwargs: + types.SimpleNamespace(returncode=0, stdout=raw)) + with pytest.raises(controller._HubCandidateTransportError): + controller._hub_candidate_api_json("/repos/jens/itworx_mcp_hub/branches/main", query={}) + + +def test_candidate_transport_deadline_is_parent_enforced(controller, monkeypatch): + def blocked(argv, **kwargs): + assert kwargs["timeout"] == 20 + raise controller.subprocess.TimeoutExpired(argv, kwargs["timeout"], output=b"private") + + monkeypatch.setattr(controller.subprocess, "run", blocked) + with pytest.raises(controller._HubCandidateTransportError, match="read_unavailable") as error: + controller._hub_candidate_api_json("/repos/jens/itworx_mcp_hub/branches/main", query={}) + assert "private" not in str(error.value) + + +def test_artifact_transport_writes_only_parent_owned_fd(controller, monkeypatch, tmp_path): + import os + controller.STATE_ROOT = tmp_path + parent = tmp_path / "mcp-hub-release-imports" + parent.mkdir(mode=0o700) + destination = parent / "candidate.zip" + + def run(argv, **kwargs): + assert argv[-1] == "zip" + assert kwargs["timeout"] == 1800 + payload = json.loads(kwargs["input"]) + assert set(payload) == {"url", "headers", "maxBytes", "outputFd"} + assert str(destination) not in kwargs["input"].decode() + assert kwargs["pass_fds"] == (payload["outputFd"],) + os.write(payload["outputFd"], b"zip data") + return types.SimpleNamespace(returncode=0, stdout=b'{"bytes":8}\n') + + monkeypatch.setattr(controller.subprocess, "run", run) + controller._hub_candidate_download_artifact(123, destination, 1024) + assert destination.read_bytes() == b"zip data" + assert destination.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize("receipt", [b'{"bytes":9}', b'{"bytes":true}', b'{"bytes":0}']) +def test_artifact_receipt_must_match_actual_file(controller, monkeypatch, tmp_path, receipt): + controller.STATE_ROOT = tmp_path + parent = tmp_path / "mcp-hub-release-imports" + parent.mkdir(mode=0o700) + monkeypatch.setattr(controller.subprocess, "run", lambda *_args, **_kwargs: + types.SimpleNamespace(returncode=0, stdout=receipt)) + with pytest.raises(controller._HubCandidateTransportError): + controller._hub_candidate_download_artifact(123, parent / "candidate.zip", 1024) + + +def _delivery(controller, monkeypatch, tmp_path, outcome): + record = {"repository": controller.CANONICAL_HUB_REPOSITORY, "revision": "a" * 40, + "status": "validation_passed", "audit": []} + marker = tmp_path / "inbox.json" + monkeypatch.setattr(controller, "_atomic_inbox_update", + lambda _marker, item: marker.write_text(json.dumps(item))) + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True) + calls = [] + + def prepare(*args, **kwargs): + calls.append((args, kwargs)) + return {"revision": record["revision"], "status": outcome, + "candidateSha256": "b" * 64} + + monkeypatch.setitem(sys.modules, "hub_candidate_import", types.SimpleNamespace( + prepare_candidate=prepare, CandidateRejected=type("CandidateRejected", (ValueError,), {}), + CandidateDeferred=type("CandidateDeferred", (RuntimeError,), {}))) + return marker, record, calls + + +@pytest.mark.parametrize("outcome", ["awaiting_candidate", "pending_review"]) +def test_candidate_wait_never_marks_deployment_started_and_throttles(controller, monkeypatch, tmp_path, outcome): + marker, record, calls = _delivery(controller, monkeypatch, tmp_path, outcome) + assert not controller._prepare_hub_candidate_delivery(marker, record, {}) + assert record["status"] == "validation_passed" + assert "deployment_started" not in [entry["stage"] for entry in record["audit"]] + assert not controller._prepare_hub_candidate_delivery(marker, record, {}) + assert len(calls) == 1 + + +def test_candidate_ready_is_hash_bound_before_deployment(controller, monkeypatch, tmp_path): + marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready") + assert controller._prepare_hub_candidate_delivery(marker, record, {}) + assert record["status"] == "validation_passed" + assert record["candidate_preparation"]["candidate_sha256"] == "b" * 64 + assert len(calls) == 1 + + +def test_candidate_wait_expires_without_production_failure(controller, monkeypatch, tmp_path): + marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "awaiting_candidate") + monkeypatch.setattr(controller.time, "time", lambda: 12000) + record["candidate_preparation"] = {"started_at": 1, "deadline_at": 10801, + "next_check_at": 2, "status": "awaiting_candidate"} + assert not controller._prepare_hub_candidate_delivery(marker, record, {}) + assert record["status"] == "validation_timeout" + assert record["reason"] == "hub_candidate_timeout" + assert calls == [] + + +def test_candidate_preparation_does_not_import_superseded_head(controller, monkeypatch, tmp_path): + marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready") + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: False) + assert not controller._prepare_hub_candidate_delivery(marker, record, {}) + assert record["status"] == "superseded" + assert calls == [] + + +def test_candidate_does_not_deploy_when_head_advances_during_import(controller, monkeypatch, tmp_path): + marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready") + heads = iter((True, False)) + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: next(heads)) + assert not controller._prepare_hub_candidate_delivery(marker, record, {}) + assert len(calls) == 1 + assert record["status"] == "superseded" + assert "candidate_sha256" not in record["candidate_preparation"] + assert "deployment_started" not in [entry["stage"] for entry in record["audit"]] + + +def test_candidate_import_cannot_overrun_the_delivery_deadline(controller, monkeypatch, tmp_path): + marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready") + clock = [1] + monkeypatch.setattr(controller.time, "time", lambda: clock[0]) + prepare = sys.modules["hub_candidate_import"].prepare_candidate + + def slow(*args, **kwargs): + result = prepare(*args, **kwargs) + clock[0] = 10802 + return result + + monkeypatch.setattr(sys.modules["hub_candidate_import"], "prepare_candidate", slow) + assert not controller._prepare_hub_candidate_delivery(marker, record, {}) + assert len(calls) == 1 + assert record["status"] == "validation_timeout" + assert record["reason"] == "hub_candidate_timeout" + assert "candidate_sha256" not in record["candidate_preparation"] diff --git a/backend/tests/test_hub_candidate_import.py b/backend/tests/test_hub_candidate_import.py new file mode 100644 index 0000000..69d1110 --- /dev/null +++ b/backend/tests/test_hub_candidate_import.py @@ -0,0 +1,837 @@ +from __future__ import annotations + +import hashlib +import importlib +import io +import json +import contextlib +import stat +import struct +import sys +import tarfile +import types +import zipfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +BROKER_DIR = ROOT / "ops" / "gitea-deploy-broker" + + +@pytest.fixture +def modules(monkeypatch): + if "fcntl" not in sys.modules: + shim = types.ModuleType("fcntl") + shim.LOCK_EX, shim.LOCK_UN, shim.LOCK_NB = 2, 8, 4 + shim.flock = lambda *_args: None + monkeypatch.setitem(sys.modules, "fcntl", shim) + monkeypatch.syspath_prepend(str(BROKER_DIR)) + for name in ("deploy", "prepare_mcp_hub_candidate", "hub_candidate_import"): + sys.modules.pop(name, None) + controller = importlib.import_module("deploy") + prepare = importlib.import_module("prepare_mcp_hub_candidate") + importer = importlib.import_module("hub_candidate_import") + monkeypatch.setattr(prepare, "_is_host_root", lambda: True) + yield types.SimpleNamespace( + controller=controller, prepare=prepare, importer=importer + ) + for name in ("hub_candidate_import", "prepare_mcp_hub_candidate", "deploy"): + sys.modules.pop(name, None) + + +def _zip_stream_member(name: str, payload: bytes, offset: int) -> tuple[bytes, bytes]: + encoded_name = name.encode("ascii") + crc = zipfile.crc32(payload) & 0xFFFFFFFF + local = struct.pack( + " bytearray: + names = sorted(importer._expected_zip_members()) + if extra_member: + names.append("unexpected.txt") + members = [(name, b"{}" if name != "images.tar" else b"tar-data") for name in names] + output = bytearray() + centrals: list[bytes] = [] + for name, payload in members: + local, central = _zip_stream_member(name, payload, len(output)) + output.extend(local) + centrals.append(central) + central_offset = len(output) + for central in centrals: + output.extend(central) + central_size = len(output) - central_offset + output.extend( + struct.pack( + " None: + path.write_bytes(data) + path.chmod(0o600) + + +def _synthetic_scan_reports( + services, images, *, created_at: str, finding_service: str | None = None +) -> dict[str, bytes]: + report_data: dict[str, bytes] = {} + for service in services: + image_id = images[service]["imageId"] + vulnerabilities = ( + [ + { + "Severity": "HIGH", + "PkgName": "package", + "VulnerabilityID": "CVE-2099-0001", + "InstalledVersion": "1.0", + } + ] + if service == finding_service + else [] + ) + vulnerability = { + "SchemaVersion": 2, + "Metadata": { + "ImageID": image_id, + "ImageConfig": {"os": "linux", "architecture": "amd64"}, + }, + "CreatedAt": created_at, + "Trivy": {"Version": "0.65.0"}, + "Results": [ + { + "Packages": [{"Name": "package", "Version": "1.0"}], + "Vulnerabilities": vulnerabilities, + } + ], + } + secret = { + "ImageID": image_id, + "CreatedAt": created_at, + "TrivyVersion": "0.65.0", + "FindingCount": 0, + "SensitiveFindingDetailsPersisted": False, + "Results": [{"FindingCount": 0}], + } + report_data[f"vulnerability-{service}.json"] = json.dumps( + vulnerability + ).encode() + report_data[f"secret-{service}.json"] = json.dumps(secret).encode() + report_data["trivy-db-metadata.json"] = json.dumps( + {"UpdatedAt": created_at, "DownloadedAt": created_at} + ).encode() + return report_data + + +def test_run_identity_requires_exact_protected_push_and_no_pull_request(modules): + importer = modules.importer + revision = "a" * 40 + run = { + "path": importer.RUN_PATH, + "event": "push", + "head_branch": "main", + "head_sha": revision, + "repository": {"full_name": "Jens/ITWorx_MCP_Hub"}, + "head_repository": None, + "pull_requests": [], + } + assert importer._run_matches(run, revision) + for change in ( + {"event": "workflow_dispatch"}, + {"path": "other.yml@refs/heads/main"}, + {"head_repository": {"full_name": "attacker/fork"}}, + {"pull_requests": [{"number": 12}]}, + {"head_sha": "b" * 40}, + {"repository": {"full_name": "attacker/repository"}}, + ): + assert not importer._run_matches({**run, **change}, revision) + + +def test_selected_run_and_jobs_accept_the_live_gitea_job_shape(modules): + importer = modules.importer + revision = "a" * 40 + now = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ) + run = { + "id": 21531, + "path": importer.RUN_PATH, + "event": "push", + "head_branch": "main", + "head_sha": revision, + "repository": {"full_name": "Jens/ITWorx_MCP_Hub"}, + "head_repository": None, + "pull_requests": [], + "created_at": now, + "run_attempt": 1, + "status": "completed", + "conclusion": "success", + } + job_rows = [ + { + "id": 37145 + index, + "name": name, + "run_id": 21531, + "status": "completed", + "conclusion": "success", + "runner_id": 22, + "started_at": now, + "completed_at": now, + } + for index, name in enumerate(("full", "release-images")) + ] + calls = [] + + def api(path, *, query): + calls.append((path, query)) + if path.endswith("/actions/runs"): + return {"total_count": 1, "workflow_runs": [run]} + if path.endswith("/actions/runs/21531/jobs"): + return {"total_count": 2, "jobs": job_rows} + raise AssertionError(f"unexpected fixed API path: {path}") + + selected, built_at = importer._selected_run(api, revision) + assert selected == run + assert built_at.endswith("Z") + assert importer._jobs_ready(api, revision, selected) + assert calls[0][1] == { + "event": "push", + "branch": "main", + "head_sha": revision, + "page": "1", + "limit": "100", + } + assert calls[1][1] == {"page": "1", "limit": "100"} + + +def test_required_candidate_jobs_must_both_succeed_and_bind_run(modules): + importer = modules.importer + run = { + "id": 5, + "status": "completed", + "conclusion": "success", + "run_attempt": 1, + } + rows = [ + {"name": name, "run_id": 5, "status": "completed", "conclusion": "success"} + for name in ("full", "release-images") + ] + + def api(_path, *, query): + assert query == {"page": "1", "limit": "100"} + return {"total_count": len(rows), "jobs": rows} + + assert importer._jobs_ready(api, "a" * 40, run) + rows[1] = {**rows[1], "conclusion": "failure"} + with pytest.raises(importer.CandidateRejected): + importer._jobs_ready(api, "a" * 40, run) + + +def test_recheck_rejects_a_new_attempt_for_the_same_workflow_run(modules, monkeypatch): + importer = modules.importer + revision = "a" * 40 + original = {"id": 21531, "run_attempt": 1} + restarted = {"id": 21531, "run_attempt": 2} + artifact = {"id": 730} + monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: "b" * 40) + monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None) + monkeypatch.setattr( + importer, + "_selected_run", + lambda *_args: (restarted, "2026-10-03T10:00:00.000Z"), + ) + monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True) + with pytest.raises(importer.CandidateRejected, match="run changed"): + importer._recheck_protected_artifact( + lambda *_args, **_kwargs: None, + revision=revision, + tree="b" * 40, + initial_run=original, + initial_built_at="2026-10-03T10:00:00.000Z", + initial_artifact=artifact, + ) + + +def test_unfixable_vulnerability_finding_is_bound_to_fixed_image_reference(modules): + importer = modules.importer + controller = modules.controller + created_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace( + "+00:00", "Z" + ) + images = { + service: { + "imageId": f"sha256:{index:064x}", + "imageRef": f"registry.example/{service}:fixed", + } + for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1) + } + report_data = _synthetic_scan_reports( + controller.MCP_HUB_SERVICES, + images, + created_at=created_at, + finding_service=controller.MCP_HUB_SERVICES[0], + ) + + _reports, findings = importer._validate_reports(images, report_data) + assert len(findings) == 1 + assert findings[0]["imageRef"] == images[controller.MCP_HUB_SERVICES[0]]["imageRef"] + summary = importer._candidate_findings_summary(findings, images) + assert summary["unresolvedCriticalOrHigh"][0]["affectedImages"] == [ + findings[0]["imageRef"] + ] + + +@pytest.mark.parametrize("finding_present", [False, True]) +@pytest.mark.parametrize("rerun_during_load", [False, True]) +def test_fake_local_import_publishes_candidate_and_requires_review_for_findings( + modules, monkeypatch, tmp_path, finding_present, rerun_during_load +): + importer = modules.importer + prepare = modules.prepare + controller = modules.controller + revision, tree, epoch = "a" * 40, "b" * 40, "1790985600" + built_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace( + "+00:00", "Z" + ) + build = { + "id": revision[:12], + "repository": importer.REPOSITORY, + "revision": revision, + "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": built_at, + } + compose_sha = "c" * 64 + model_services = {} + images = {} + for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1): + external = importer.EXTERNAL_IMAGES.get(service) + image_ref = ( + external[0] + if external is not None + else f"registry.example/itworx/{service}:{revision[:12]}" + ) + archive_ref = external[1] if external is not None else image_ref + model_services[service] = {"image": image_ref} + images[service] = { + "imageRef": image_ref, + "archiveRef": archive_ref, + "imageId": f"sha256:{index:064x}", + } + model = {"services": model_services} + created_at = built_at + finding_service = controller.MCP_HUB_SERVICES[0] if finding_present else None + report_data = _synthetic_scan_reports( + controller.MCP_HUB_SERVICES, + images, + created_at=created_at, + finding_service=finding_service, + ) + _reports, findings = importer._validate_reports(images, report_data) + summary = importer._candidate_findings_summary(findings, images) + digests = { + "images": { + service: { + "imageId": images[service]["imageId"], + "tag": images[service]["imageRef"], + } + for service in controller.MCP_HUB_SERVICES + } + } + exception_rows = [ + { + "package": finding["package"], + "cve": finding["cve"], + "installedVersion": finding["installedVersion"], + "severity": finding["severity"], + "affectedImageDigests": { + finding["imageRef"]: images_by_ref["imageId"] + for images_by_ref in [ + next( + row + for row in images.values() + if row["imageRef"] == finding["imageRef"] + ) + ] + }, + } + for finding in findings + ] + draft = {"schemaVersion": "1.0", "exceptions": exception_rows} + state_root = tmp_path / "state" + state_root.mkdir(mode=0o700) + state_root.chmod(0o700) + releases_root = tmp_path / "releases" + source = releases_root / revision + source.mkdir(parents=True) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases_root) + monkeypatch.setattr(controller, "GLOBAL_DEPLOY_LOCK", state_root / "deploy.lock") + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda *_args: None) + monkeypatch.setattr(controller, "_root_owned", lambda *_args: True) + + def create_private_directory(path, *, create=False, root=None): + del root + if create: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + path.chmod(0o700) + if path.is_symlink() or not path.is_dir(): + raise controller.DeployError("test private directory missing") + + def candidate_directory(candidate_revision): + directory = state_root / "mcp-hub-release-candidates" / candidate_revision + for path in (directory.parent, directory): + if path.is_symlink() or not path.is_dir(): + raise controller.DeployError("test candidate directory missing") + return directory + + monkeypatch.setattr(controller, "_mcp_hub_private_directory", create_private_directory) + monkeypatch.setattr(controller, "_mcp_hub_candidate_directory", candidate_directory) + project, repo = {"branch": "main"}, tmp_path / "repo" + monkeypatch.setattr(prepare, "_fixed_target", lambda *_args: (project, repo)) + monkeypatch.setattr(controller, "deployment_metadata", lambda *_args, **_kwargs: build) + monkeypatch.setattr(controller, "mcp_hub_source_date_epoch", lambda *_args: epoch) + monkeypatch.setattr(controller, "mcp_hub_stage_release_source", lambda *_args: source) + monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True) + monkeypatch.setattr(controller, "_mcp_hub_verify_release_tree_metadata", lambda *_args: None) + monkeypatch.setattr( + prepare, + "_render", + lambda *_args, **_kwargs: (model, compose_sha), + ) + monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: tree) + monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None) + run = {"id": 21531, "run_attempt": 1} + run_state = {"attempt": 1} + monkeypatch.setattr( + importer, + "_selected_run", + lambda *_args: ({**run, "run_attempt": run_state["attempt"]}, built_at), + ) + monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True) + artifact_bytes = b"bounded fake Gitea artifact" + artifact = { + "id": 730, + "size_in_bytes": len(artifact_bytes), + "name": importer.ARTIFACT_PREFIX + revision, + "workflow_run": {"id": run["id"]}, + } + monkeypatch.setattr(importer, "_candidate_artifact", lambda *_args: artifact) + monkeypatch.setattr(importer, "_docker_root", lambda: tmp_path) + monkeypatch.setattr(importer, "_check_disk_headroom", lambda **_kwargs: None) + monkeypatch.setattr(importer, "_parse_artifact_zip", lambda *_args, **_kwargs: ( + {}, + report_data, + hashlib.sha256(artifact_bytes).hexdigest(), + "d" * 64, + "e" * 64, + 123, + )) + monkeypatch.setattr( + importer, + "_validate_manifest", + lambda *_args, **_kwargs: (images, {}), + ) + monkeypatch.setattr(importer, "_validate_docker_save", lambda *_args, **_kwargs: None) + monkeypatch.setattr(importer, "_bounded_global_deploy_lock", contextlib.nullcontext) + monkeypatch.setattr(importer, "_image_preflight", lambda *_args: None) + + def fake_fixed_docker(*args, **_kwargs): + if rerun_during_load and args and args[0] == "load": + run_state["attempt"] = 2 + return object() + + monkeypatch.setattr(importer, "_fixed_docker", fake_fixed_docker) + monkeypatch.setattr(importer, "_verify_loaded_images", lambda *_args, **_kwargs: {}) + monkeypatch.setattr( + controller, + "_mcp_hub_check_candidate_exceptions", + lambda *_args: None, + ) + monkeypatch.setattr( + prepare, + "_draft_pending_exceptions", + lambda *_args: draft, + ) + + def fake_plan(_repository, _revision, *, built_at, staging_directory, **_kwargs): + Path(staging_directory).mkdir(mode=0o700) + frozen = { + "schema": prepare.BUILD_SCHEMA, + "repository": importer.REPOSITORY, + "revision": revision, + "build": build, + "sourceDateEpoch": epoch, + "composeSha256": compose_sha, + } + controller._mcp_hub_write_private_new( + Path(staging_directory) / "build.json", prepare._json_bytes(frozen) + ) + return {"status": "planned", "revision": revision, "composeSha256": compose_sha} + + monkeypatch.setattr(prepare, "plan_candidate", fake_plan) + + def fake_capture(_repository, _revision, *, staging_directory, **_kwargs): + image_rows = { + service: { + "imageRef": images[service]["imageRef"], + "imageId": images[service]["imageId"], + "vulnerabilitySha256": hashlib.sha256( + report_data[f"vulnerability-{service}.json"] + ).hexdigest(), + "secretSha256": hashlib.sha256( + report_data[f"secret-{service}.json"] + ).hexdigest(), + } + for service in controller.MCP_HUB_SERVICES + } + candidate = { + "schema": prepare.CANDIDATE_SCHEMA, + "repository": importer.REPOSITORY, + "revision": revision, + "build": build, + "sourceDateEpoch": epoch, + "composeSha256": compose_sha, + "images": image_rows, + "databaseMetadataSha256": hashlib.sha256( + report_data["trivy-db-metadata.json"] + ).hexdigest(), + "summarySha256": hashlib.sha256(prepare._json_bytes(summary)).hexdigest(), + "digestsSha256": hashlib.sha256(prepare._json_bytes(digests)).hexdigest(), + } + outputs = { + "candidate.json": candidate, + "scan-summary.json": summary, + "image-digests.json": digests, + "exceptions.json": draft, + } + for filename, value in outputs.items(): + controller._mcp_hub_write_private_new( + Path(staging_directory) / filename, prepare._json_bytes(value) + ) + candidate_sha = hashlib.sha256(prepare._json_bytes(candidate)).hexdigest() + exceptions_sha = hashlib.sha256(prepare._json_bytes(draft)).hexdigest() + return { + "candidateSha256": candidate_sha, + "exceptionsSha256": exceptions_sha, + } + + monkeypatch.setattr(prepare, "capture_candidate", fake_capture) + + def download_artifact(_artifact_id, destination, _maximum): + _private_file(destination, artifact_bytes) + return None + + candidate_directory = state_root / "mcp-hub-release-candidates" / revision + if rerun_during_load: + with pytest.raises(importer.CandidateRejected, match="run changed"): + importer.prepare_candidate( + importer.REPOSITORY, + revision, + api_json=lambda *_args, **_kwargs: None, + download_artifact=download_artifact, + ) + assert not candidate_directory.exists() + return + + first = importer.prepare_candidate( + importer.REPOSITORY, + revision, + api_json=lambda *_args, **_kwargs: None, + download_artifact=download_artifact, + ) + assert candidate_directory.is_dir() + assert stat.S_IMODE(candidate_directory.stat().st_mode) == 0o700 + candidate_before_review = (candidate_directory / "candidate.json").read_bytes() + receipt_before_review = (candidate_directory / "import-receipt.json").read_bytes() + collision_stage = tmp_path / "second-candidate" + collision_stage.mkdir(mode=0o700) + with pytest.raises(importer.CandidateRejected, match="never overwrites"): + importer._publish_candidate(collision_stage, candidate_directory) + assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review + if finding_present: + assert first["status"] == "pending_review" + reviewed = { + **draft, + "reviewer": "release-operator", + "exceptions": [ + {**row, "justification": "Reviewed unfixable finding"} + for row in draft["exceptions"] + ], + } + exceptions_path = candidate_directory / "exceptions.json" + exceptions_path.unlink() + controller._mcp_hub_write_private_new( + exceptions_path, prepare._json_bytes(reviewed) + ) + approval = { + "schema": importer.APPROVAL_SCHEMA, + "candidateSha256": first["candidateSha256"], + "exceptionsSha256": hashlib.sha256( + prepare._json_bytes(reviewed) + ).hexdigest(), + } + controller._mcp_hub_write_private_new( + candidate_directory / "approval.json", prepare._json_bytes(approval) + ) + final = importer.prepare_candidate( + importer.REPOSITORY, + revision, + api_json=lambda *_args, **_kwargs: None, + download_artifact=download_artifact, + ) + assert final["status"] == "ready" + else: + assert first["status"] == "ready" + final = first + assert final["candidateSha256"] == hashlib.sha256(candidate_before_review).hexdigest() + assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review + assert (candidate_directory / "import-receipt.json").read_bytes() == receipt_before_review + + +def test_candidate_artifact_is_exact_bounded_and_unexpired(modules): + importer = modules.importer + revision = "c" * 40 + run_id = 11 + artifact = { + "id": 22, + "name": importer.ARTIFACT_PREFIX + revision, + "size_in_bytes": 4096, + "expired": False, + "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)) + .isoformat() + .replace("+00:00", "Z"), + "workflow_run": {"id": run_id}, + } + result = importer._candidate_artifact( + lambda _path, *, query: {"artifacts": [artifact]}, run_id, revision + ) + assert result == artifact + for changed in ( + {"expired": True}, + {"name": "hub-release-candidate-" + "d" * 40}, + {"size_in_bytes": importer.MAX_ARTIFACT_ZIP_BYTES + 1}, + {"workflow_run": {"id": run_id + 1}}, + ): + with pytest.raises(importer.CandidateRejected): + importer._candidate_artifact( + lambda _path, *, query: {"artifacts": [{**artifact, **changed}]}, + run_id, + revision, + ) + + +def test_streamed_uploader_zip_descriptors_are_verified(modules, tmp_path): + importer = modules.importer + payload = _streamed_zip_bytes(importer) + zip_path = tmp_path / "candidate.zip" + tar_path = tmp_path / "images.tar" + _private_file(zip_path, bytes(payload)) + ( + manifest, + reports, + zip_sha, + manifest_sha, + tar_sha, + tar_bytes, + ) = importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(payload)) + assert manifest == {} + assert len(reports) == 35 + assert zip_sha == hashlib.sha256(payload).hexdigest() + assert manifest_sha == hashlib.sha256(b"{}").hexdigest() + assert tar_sha == hashlib.sha256(b"tar-data").hexdigest() + assert tar_bytes == len(b"tar-data") + assert tar_path.read_bytes() == b"tar-data" + + tampered = bytearray(payload) + name_size = len("candidate-manifest.json") + descriptor_offset = 30 + name_size + 2 + tampered[descriptor_offset + 4] ^= 0x01 + _private_file(zip_path, bytes(tampered)) + with pytest.raises(importer.CandidateRejected, match="descriptor"): + importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(tampered)) + + +def test_zip_central_directory_is_bounded_before_zipfile_parse(modules, tmp_path, monkeypatch): + importer = modules.importer + payload = _streamed_zip_bytes(importer, extra_member=True) + path = tmp_path / "too-many.zip" + _private_file(path, bytes(payload)) + monkeypatch.setattr( + importer.zipfile, + "ZipFile", + lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized member inventory"), + ) + with pytest.raises(importer.CandidateRejected, match="central directory"): + importer._zip_central_directory_preflight(path, len(payload)) + + +def test_zip_oversized_central_directory_is_rejected_without_allocating_it( + modules, tmp_path, monkeypatch +): + importer = modules.importer + payload = struct.pack( + "= 0: + os.close(write_fd) + os.close(read_fd) + + +def test_redirect_strips_authorization_and_rejects_other_hosts(): + request = transport.Request( + "https://gitea.example.com/api/v1/artifact", + headers={"Accept": "application/zip", "Authorization": "token secret", "User-Agent": transport.USER_AGENT}, + ) + handler = transport._SameHostArtifactRedirect() + redirected = handler.redirect_request( + request, None, 302, "", {}, "https://gitea.example.com/attachments/opaque?signature=opaque" + ) + assert redirected is not None + assert {key.lower(): value for key, value in redirected.header_items()} == { + "accept": "application/zip", "user-agent": transport.USER_AGENT, + } + for url in ( + "http://gitea.example.com/attachments/file", + "https://other.invalid/attachments/file", + "https://gitea.example.com:444/attachments/file", + "https://user@gitea.example.com/attachments/file", + "https://gitea.example.com/attachments/file#fragment", + ): + assert handler.redirect_request(request, None, 302, "", {}, url) is None + + +def test_deadline_stops_a_continuous_read1_trickle(): + clock = [0.0] + + class Trickle(Response): + def read1(self, _limit): + clock[0] += 2 + return b" " + + value = _request("https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/branches/main") + with pytest.raises(transport.TransportError, match="deadline"): + transport.execute(value, "json", opener_factory=lambda _mode: Opener(Trickle([])), + monotonic=lambda: clock[0]) + + +def test_cli_errors_are_generic_and_zip_marker_is_compact(): + errors = io.BytesIO() + stdout = io.BytesIO() + assert transport.main(["wrong"], stdin=io.BytesIO(b""), stdout=stdout, stderr=errors) == 1 + assert errors.getvalue() == transport._GENERIC_ERROR + assert b"wrong" not in errors.getvalue() + + read_fd, write_fd = os.pipe() + try: + value = _request( + "https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/artifacts/12/zip", + mode="zip", output_fd=write_fd, + ) + response = Response([b"zip"]) + payload = json.dumps(value, separators=(",", ":")).encode() + stdout = io.BytesIO() + errors = io.BytesIO() + result = transport.main(["zip"], stdin=io.BytesIO(payload), stdout=stdout, + stderr=errors, opener_factory=lambda _mode: Opener(response)) + assert result == 0 + assert stdout.getvalue() == b'{"bytes":3}\n' + assert errors.getvalue() == b"" + os.close(write_fd) + write_fd = -1 + assert os.read(read_fd, 32) == b"zip" + finally: + if write_fd >= 0: + os.close(write_fd) + os.close(read_fd) diff --git a/backend/tests/test_hub_permission_preparation.py b/backend/tests/test_hub_permission_preparation.py new file mode 100644 index 0000000..07430bc --- /dev/null +++ b/backend/tests/test_hub_permission_preparation.py @@ -0,0 +1,775 @@ +from __future__ import annotations + +import fcntl +import hashlib +import importlib.util +import io +import json +import os +import stat +import sys +import time +from email.message import Message +from pathlib import Path + +import pytest +from sqlmodel import Session, SQLModel, create_engine, select + +from backend.app.db.models import AuditEvent, JobRun +from backend.app.services import action_execution, deployment_broker +from backend.app.services.action_execution import ( + approval_execution_supported, + mcp_execution_supported, +) +from backend.app.services.action_registry import ACTION_RISK, ACTIONS, action_default_target + +ROOT = Path(__file__).resolve().parents[2] +CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" +ACTION = "hub.migration.prepare_permissions" +TARGET = "tower/itworx-mcp-hub-permissions" +STATE_PATH = "/mnt/cache/appdata/gitea-deploy-runners/state/itworx-mcp-hub.json" +LEGACY_BACKUP_PATH = "/srv/example/appdata/itworx-mcp-hub/backups" + + +@pytest.fixture +def hub(): + path = CONTROLLER / "hub_permission_preparation.py" + assert path.is_file(), "the protected controller permission helper is missing" + spec = importlib.util.spec_from_file_location("hub_permission_preparation_test", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def controller(monkeypatch): + monkeypatch.syspath_prepend(str(CONTROLLER)) + spec = importlib.util.spec_from_file_location( + "hub_permission_controller_test", CONTROLLER / "status_server.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def permission_fixture(hub, tmp_path, monkeypatch): + if os.name != "posix" or not hasattr(os, "geteuid") or os.geteuid() != 0: + pytest.skip("permission ownership fixture requires Linux root") + state_dir = tmp_path / "cache" / "appdata" / "gitea-deploy-runners" / "state" + state = state_dir / "itworx-mcp-hub.json" + lock = state_dir.parent / "deploy.lock" + state_dir.mkdir(parents=True) + + state_bytes = b'{"fixture":"hub state"}\n'.ljust(4096, b" ") + state.write_bytes(state_bytes) + lock.write_bytes(b"") + os.chown(state, 0, 0) + os.chmod(state, 0o644) + os.chown(lock, 0, 0) + os.chmod(lock, 0o644) + + monkeypatch.setattr(hub, "STATE_FILE", str(state)) + monkeypatch.setattr(hub, "LOCK_FILE", str(lock)) + monkeypatch.setattr(hub, "LOCK_TIMEOUT_SECONDS", 0.25) + monkeypatch.setattr(hub, "LOCK_POLL_INTERVAL_SECONDS", 0.01) + digest = hashlib.sha256(state_bytes).hexdigest() + return state, lock, state_bytes, digest + + +def state_target(state: Path, *, mode: str = "0600", content: bytes | None = None) -> dict: + payload = content or b'{"fixture":"hub state"}\n'.ljust(4096, b" ") + return { + "name": "state_file", + "path": str(state), + "uid": 0, + "gid": 0, + "mode": mode, + "desired_mode": "0600", + "would_change": mode != "0600", + "acl_present": False, + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + + +def test_permission_action_is_high_risk_and_has_one_fixed_target(): + assert ACTION in ACTIONS + assert ACTION_RISK[ACTION] == "high" + assert action_default_target(ACTION) == TARGET + assert approval_execution_supported(ACTION) + assert mcp_execution_supported(ACTION) + + +def test_broker_posts_only_the_bounded_reason_to_fixed_signed_path(monkeypatch): + calls = [] + broker = deployment_broker.DeploymentBrokerClient() + monkeypatch.setattr(broker, "_request", lambda *args, **kwargs: calls.append((args, kwargs)) or {}) + + broker.hub_migration_prepare_permissions(reason="prepare fixed Hub permissions" * 40) + + assert calls[0][0] == ("POST", "/v1/appops/hub-migration/prepare-permissions") + assert calls[0][1]["payload"] == {"reason": ("prepare fixed Hub permissions" * 40)[:1000]} + + +def test_governed_execution_blocks_wrong_target_before_job_or_broker(tmp_path, monkeypatch): + engine = create_engine(f"sqlite:///{tmp_path / 'blocked.db'}") + SQLModel.metadata.create_all(engine) + monkeypatch.setattr( + deployment_broker.DeploymentBrokerClient, + "hub_migration_prepare_permissions", + lambda *args, **kwargs: pytest.fail("wrong target must not call the controller"), + ) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, + action=ACTION, + target="tower/other", + source="chatgpt-mcp", + risk="high", + reason="prepare fixed Hub permissions", + original_policy_decision="allowed", + ) + assert result["policyDecision"] == "blocked" + assert session.exec(select(JobRun)).all() == [] + + +@pytest.mark.parametrize( + "status,success", + [("prepared", True), ("already_compliant", True), ("partial", False), ("blocked", False)], +) +def test_governed_execution_audits_single_state_file_outcome(tmp_path, monkeypatch, status, success): + engine = create_engine(f"sqlite:///{tmp_path / f'{status}.db'}") + SQLModel.metadata.create_all(engine) + result_payload = { + "status": status, + "revision": "a" * 40, + "changedTargets": [STATE_PATH] if status in {"prepared", "partial"} else [], + "changedTargetDetails": [], + "uncertainTargets": [STATE_PATH] if status == "partial" else [], + } + if status == "partial": + result_payload["changedTargetDetails"] = [ + { + "name": "state_file", + "path": STATE_PATH, + "observedMode": "0600", + "sameIdentityAtPath": True, + } + ] + if status in {"prepared", "already_compliant"}: + result_payload["verifiedTargets"] = [ + { + "name": "state_file", + "path": STATE_PATH, + "uid": 0, + "gid": 0, + "mode": "0600", + "aclPresent": False, + } + ] + monkeypatch.setattr( + deployment_broker.DeploymentBrokerClient, + "hub_migration_prepare_permissions", + lambda self, **kwargs: result_payload, + ) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, + action=ACTION, + target=TARGET, + source="chatgpt-mcp", + risk="high", + reason="prepare fixed Hub permissions", + original_policy_decision="allowed", + ) + job = session.get(JobRun, result["jobId"]) + audit = session.get(AuditEvent, result["auditId"]) + assert result["success"] is success + assert job.status == ("succeeded" if success else "failed") + assert audit.result == ("success" if success else "failed") + assert audit.target == TARGET + assert audit.after_state["status"] == status + assert audit.after_state["changedTargets"] == result_payload["changedTargets"] + assert audit.after_state["changedTargetDetails"] == result_payload["changedTargetDetails"] + + +@pytest.mark.parametrize("legacy_status", ["prepared", "already_compliant", "partial"]) +def test_governed_execution_rejects_legacy_two_target_reports_as_unknown( + tmp_path, monkeypatch, legacy_status +): + engine = create_engine(f"sqlite:///{tmp_path / f'legacy-{legacy_status}.db'}") + SQLModel.metadata.create_all(engine) + result_payload = { + "status": legacy_status, + "revision": "a" * 40, + "changedTargets": [LEGACY_BACKUP_PATH] if legacy_status != "already_compliant" else [], + "changedTargetDetails": [], + "uncertainTargets": [], + "verifiedTargets": [ + { + "name": "backup_directory", + "path": LEGACY_BACKUP_PATH, + "uid": 99, + "gid": 100, + "mode": "0700", + "aclPresent": False, + }, + { + "name": "state_file", + "path": STATE_PATH, + "uid": 0, + "gid": 0, + "mode": "0600", + "aclPresent": False, + }, + ], + } + monkeypatch.setattr( + deployment_broker.DeploymentBrokerClient, + "hub_migration_prepare_permissions", + lambda self, **kwargs: result_payload, + ) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, + action=ACTION, + target=TARGET, + source="chatgpt-mcp", + risk="high", + reason="prepare fixed Hub permissions", + original_policy_decision="allowed", + ) + audit = session.get(AuditEvent, result["auditId"]) + assert result["success"] is False + assert result["status"] == "outcome_unknown" + assert result["result"] == "outcome_unknown" + assert result["uncertainTargets"] == [STATE_PATH] + assert audit.result == "outcome_unknown" + assert audit.after_state["uncertainTargets"] == [STATE_PATH] + + +@pytest.mark.parametrize( + "failure", ["broker_exception", "unreachable", "malformed_success", "malformed_partial"] +) +def test_governed_execution_marks_unresolved_controller_results_unknown( + tmp_path, monkeypatch, failure +): + engine = create_engine(f"sqlite:///{tmp_path / f'{failure}.db'}") + SQLModel.metadata.create_all(engine) + calls = [] + + def broker_call(self, **_kwargs): + calls.append("called") + if failure == "broker_exception": + raise RuntimeError("request ended without a controller result") + if failure == "unreachable": + return {"status": "unreachable"} + if failure == "malformed_partial": + return { + "status": "partial", + "revision": "a" * 40, + "changedTargets": [LEGACY_BACKUP_PATH], + "changedTargetDetails": [], + "uncertainTargets": [], + } + return { + "status": "prepared", + "revision": "a" * 40, + "changedTargets": [STATE_PATH], + "changedTargetDetails": [], + "uncertainTargets": [], + } + + monkeypatch.setattr( + deployment_broker.DeploymentBrokerClient, + "hub_migration_prepare_permissions", + broker_call, + ) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, + action=ACTION, + target=TARGET, + source="chatgpt-mcp", + risk="high", + reason="prepare fixed Hub permissions", + original_policy_decision="allowed", + ) + job = session.get(JobRun, result["jobId"]) + audit = session.get(AuditEvent, result["auditId"]) + + assert calls == ["called"] + assert result["success"] is False + assert result["status"] == "outcome_unknown" + assert result["result"] == "outcome_unknown" + assert result["uncertainTargets"] == [STATE_PATH] + assert job.status == "failed" + assert "outcome is unknown" in job.result_summary + assert audit.result == "outcome_unknown" + assert audit.after_state["status"] == "outcome_unknown" + assert audit.after_state["changedTargets"] == [] + + +def test_preparation_changes_only_fixed_state_file_and_is_idempotent(hub, tmp_path, monkeypatch): + state, lock, state_bytes, digest = permission_fixture(hub, tmp_path, monkeypatch) + state_identity = (state.stat().st_dev, state.stat().st_ino) + original_open = hub._open_pinned_object + opened = [] + changed_objects = [] + + # Keep a legacy backup path with an intentionally irrelevant mode. The + # helper must neither open nor chmod it. + legacy_backup = tmp_path / "user" / "appdata" / "itworx-mcp-hub" / "backups" + legacy_backup.mkdir(parents=True) + os.chmod(legacy_backup, 0o750) + monkeypatch.setattr(hub, "BACKUP_DIRECTORY", str(legacy_backup), raising=False) + + def record_open(name, path): + opened.append((name, path)) + return original_open(name, path) + + real_set_mode = hub._set_mode + + def record_change(fd, mode): + st = os.fstat(fd) + changed_objects.append((st.st_dev, st.st_ino)) + real_set_mode(fd, mode) + + monkeypatch.setattr(hub, "_open_pinned_object", record_open) + monkeypatch.setattr(hub, "_set_mode", record_change) + + code, report = hub.prepare_fixed_permissions() + + assert code == 0 + assert report["status"] == "applied" + assert report["changed_targets"] == [str(state)] + assert [target["path"] for target in report["targets"]] == [str(state)] + assert stat.S_IMODE(state.stat().st_mode) == 0o600 + assert stat.S_IMODE(lock.stat().st_mode) == 0o644 + assert stat.S_IMODE(legacy_backup.stat().st_mode) == 0o750 + assert state.read_bytes() == state_bytes + assert hashlib.sha256(state.read_bytes()).hexdigest() == digest + assert (state.stat().st_dev, state.stat().st_ino) == state_identity + assert (state.stat().st_uid, state.stat().st_gid) == (0, 0) + assert report["lock"]["acquired"] is True + assert opened == [("deploy_lock", str(lock)), ("state_file", str(state))] + assert changed_objects == [state_identity] + + code, report = hub.prepare_fixed_permissions() + assert code == 0 + assert report["status"] == "already_compliant" + assert report["changed_targets"] == [] + assert [target["path"] for target in report["targets"]] == [str(state)] + assert stat.S_IMODE(legacy_backup.stat().st_mode) == 0o750 + + +def test_dry_run_does_not_change_state(hub, tmp_path, monkeypatch): + state, lock, before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + + code, report = hub.execute() + + assert code == 0 + assert report["status"] == "ready" + assert [item["path"] for item in report["targets"]] == [str(state)] + assert stat.S_IMODE(state.stat().st_mode) == 0o644 + assert stat.S_IMODE(lock.stat().st_mode) == 0o644 + assert state.read_bytes() == before + + +def test_preparation_rechecks_fresh_state_digest_before_changing_mode(hub, tmp_path, monkeypatch): + state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + real_apply = hub._apply_changes + + def change_before_apply(lock, pinned_state, mode, expected_digest): + state.write_bytes(b"changed after inspection".ljust(4096, b" ")) + return real_apply(lock, pinned_state, mode, expected_digest) + + monkeypatch.setattr(hub, "_apply_changes", change_before_apply) + + code, report = hub.prepare_fixed_permissions() + + assert code == 2 + assert report["status"] == "failed" + assert "digest mismatch" in report["error"] + assert stat.S_IMODE(state.stat().st_mode) == 0o644 + + +def test_symlinks_wrong_owner_mode_hardlinks_and_acl_fail_closed(hub, tmp_path, monkeypatch): + state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + + real_state = state.with_name("state-real.json") + state.rename(real_state) + state.symlink_to(real_state) + code, report = hub.execute() + assert code == 2 and report["status"] == "blocked" + state.unlink() + real_state.rename(state) + + os.chown(state, 99, 100) + code, report = hub.execute() + assert code == 2 and "owner mismatch" in report["error"] + os.chown(state, 0, 0) + + os.link(state, state.with_name("unexpected-hardlink")) + code, report = hub.execute() + assert code == 2 and report["status"] == "blocked" + assert "link-count mismatch" in report["error"] + state.with_name("unexpected-hardlink").unlink() + + os.chmod(state, 0o640) + code, report = hub.execute() + assert code == 2 and "mode mismatch" in report["error"] + os.chmod(state, 0o644) + + state_identity = (state.stat().st_dev, state.stat().st_ino) + actual_has_acl = hub._has_acl + monkeypatch.setattr( + hub, + "_has_acl", + lambda fd: (os.fstat(fd).st_dev, os.fstat(fd).st_ino) == state_identity + or actual_has_acl(fd), + ) + code, report = hub.execute() + assert code == 2 and "POSIX ACL is present" in report["error"] + assert stat.S_IMODE(state.stat().st_mode) == 0o644 + + +def test_unsafe_deploy_lock_metadata_blocks_without_state_mutation(hub, tmp_path, monkeypatch): + state, lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + os.chmod(lock, 0o600) + + code, report = hub.prepare_fixed_permissions() + + assert code == 2 and report["status"] == "blocked" + assert "mode mismatch" in report["error"] + assert stat.S_IMODE(state.stat().st_mode) == 0o644 + assert stat.S_IMODE(lock.stat().st_mode) == 0o600 + + +def test_replaced_state_path_reports_uncertain_outcome(hub, tmp_path, monkeypatch): + state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + moved_state = state.with_name("state-pinned.json") + real_set_mode = hub._set_mode + replaced = False + + def replace_after_mode_change(fd, mode): + nonlocal replaced + real_set_mode(fd, mode) + if not replaced: + replaced = True + state.rename(moved_state) + state.write_bytes(b"replacement") + os.chown(state, 0, 0) + os.chmod(state, 0o644) + + monkeypatch.setattr(hub, "_set_mode", replace_after_mode_change) + + code, report = hub.prepare_fixed_permissions() + + assert code == 2 + assert report["status"] == "partial" + assert report["changed_targets"] == [ + { + "name": "state_file", + "path": str(state), + "observed_mode": "0600", + "same_identity_at_path": False, + } + ] + assert report["uncertain_targets"] == [str(state)] + assert stat.S_IMODE(moved_state.stat().st_mode) == 0o600 + assert stat.S_IMODE(state.stat().st_mode) == 0o644 + + +def test_fsync_failure_reports_changed_and_uncertain_state(hub, tmp_path, monkeypatch): + state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + monkeypatch.setattr(hub, "_fsync", lambda _fd: (_ for _ in ()).throw(OSError("fsync failed"))) + + code, report = hub.prepare_fixed_permissions() + + assert code == 2 + assert report["status"] == "partial" + assert report["changed_targets"] == [ + { + "name": "state_file", + "path": str(state), + "observed_mode": "0600", + "same_identity_at_path": True, + } + ] + assert report["uncertain_targets"] == [str(state)] + + +def test_bounded_deploy_lock_contention_fails_without_mutation(hub, tmp_path, monkeypatch): + state, lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch) + fd = os.open(lock, os.O_RDONLY) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + started = time.monotonic() + code, report = hub.prepare_fixed_permissions() + elapsed = time.monotonic() - started + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + + assert code == 2 + assert report["status"] == "blocked" + assert "bounded wait" in report["error"] + assert elapsed < 1.0 + assert stat.S_IMODE(state.stat().st_mode) == 0o644 + + +def controller_handler(controller, monkeypatch, payload=None): + instance = object.__new__(controller.Handler) + body = json.dumps(payload or {"reason": "Prepare fixed Hub permissions"}).encode() + instance.headers = Message() + instance.headers["Content-Length"] = str(len(body)) + instance.headers["Content-Type"] = "application/json" + instance.rfile = io.BytesIO(body) + instance.path = controller.HUB_PERMISSION_PREPARATION_PATH + responses = [] + signatures = [] + monkeypatch.setattr(instance, "_trusted_client", lambda: True) + monkeypatch.setattr(instance, "_send_json", lambda code, data: responses.append((code, json.loads(data)))) + monkeypatch.setattr( + controller, + "_self_upgrade_signature_is_valid", + lambda *args, **kwargs: signatures.append(kwargs) or True, + ) + monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {"revision": "a" * 40}) + monkeypatch.setattr( + controller, + "_require_controller_release", + lambda: { + "sourceRevision": "a" * 40, + "sourceRepository": controller.SELF_UPGRADE_REPOSITORY, + }, + ) + monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda revision: {"status": "already_deployed"}) + monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}}) + monkeypatch.setattr(controller, "_repository_path_for_delivery", lambda *args: Path("fixed-repo.git")) + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: True) + return instance, responses, signatures + + +def controller_success_report(state_path: str, *, already: bool = False) -> dict: + return { + "status": "already_compliant" if already else "applied", + "changed_targets": [] if already else [state_path], + "uncertain_targets": [], + "targets": [state_target(Path(state_path))], + } + + +def test_controller_route_uses_only_signed_protected_validated_current_main( + controller, monkeypatch +): + instance, responses, signatures = controller_handler(controller, monkeypatch) + calls = [] + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: calls.append("prepare") + or (0, controller_success_report(STATE_PATH)), + ) + + instance.do_POST() + + assert signatures == [{"method": "POST", "path": controller.HUB_PERMISSION_PREPARATION_PATH}] + assert calls == ["prepare"] + assert responses == [ + ( + 200, + { + "status": "prepared", + "revision": "a" * 40, + "changedTargets": [STATE_PATH], + "changedTargetDetails": [], + "uncertainTargets": [], + "verifiedTargets": [ + { + "name": "state_file", + "path": STATE_PATH, + "uid": 0, + "gid": 0, + "mode": "0600", + "aclPresent": False, + } + ], + }, + ) + ] + + +def test_controller_route_rejects_legacy_two_target_success_report(controller, monkeypatch): + instance, responses, _signatures = controller_handler(controller, monkeypatch) + legacy = controller_success_report(STATE_PATH) + legacy["changed_targets"] = [LEGACY_BACKUP_PATH, STATE_PATH] + legacy["targets"].insert( + 0, + { + "name": "backup_directory", + "path": LEGACY_BACKUP_PATH, + "uid": 99, + "gid": 100, + "mode": "0700", + "desired_mode": "0700", + "would_change": False, + "acl_present": False, + }, + ) + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: (0, legacy), + ) + + instance.do_POST() + + assert responses == [ + ( + 200, + { + "status": "outcome_unknown", + "revision": "a" * 40, + "changedTargets": [], + "changedTargetDetails": [], + "uncertainTargets": [STATE_PATH], + }, + ) + ] + + +def test_controller_route_rejects_success_without_full_helper_projection(controller, monkeypatch): + instance, responses, _signatures = controller_handler(controller, monkeypatch) + malformed = controller_success_report(STATE_PATH, already=True) + malformed.pop("changed_targets") + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: (0, malformed), + ) + + instance.do_POST() + + assert responses == [ + ( + 200, + { + "status": "outcome_unknown", + "revision": "a" * 40, + "changedTargets": [], + "changedTargetDetails": [], + "uncertainTargets": [STATE_PATH], + }, + ) + ] + + +@pytest.mark.parametrize( + "payload", + [ + {"reason": "Prepare fixed Hub permissions", "path": "/"}, + {"reason": "Prepare fixed Hub permissions", "mode": "0777"}, + {"reason": "Prepare fixed Hub permissions", "command": "chmod -R 700 /"}, + ], +) +def test_controller_route_rejects_caller_selected_effects(controller, monkeypatch, payload): + instance, responses, _signatures = controller_handler(controller, monkeypatch, payload) + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: pytest.fail("invalid request must not inspect or mutate permissions"), + ) + + instance.do_POST() + + assert responses == [(400, {"status": "invalid_request"})] + + +def test_controller_route_does_not_mutate_before_full_push_validation(controller, monkeypatch): + instance, responses, _signatures = controller_handler(controller, monkeypatch) + monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: {"status": "preflight_rejected"}) + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: pytest.fail("pending validation must block permission changes"), + ) + + instance.do_POST() + + assert responses == [(409, {"status": "preflight_rejected", "revision": "a" * 40})] + + +@pytest.mark.parametrize("failure", ["helper_exception", "malformed_report", "malformed_partial"]) +def test_controller_route_reports_unknown_state_after_unresolved_helper_call( + controller, monkeypatch, failure +): + instance, responses, _signatures = controller_handler(controller, monkeypatch) + if failure == "helper_exception": + def helper_failure(): + raise RuntimeError("helper returned no result") + + monkeypatch.setattr( + controller.hub_permission_preparation, "prepare_fixed_permissions", helper_failure + ) + elif failure == "malformed_report": + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: (0, {"status": "applied", "changed_targets": "unexpected"}), + ) + else: + monkeypatch.setattr( + controller.hub_permission_preparation, + "prepare_fixed_permissions", + lambda: ( + 2, + { + "status": "partial", + "changed_targets": [LEGACY_BACKUP_PATH], + "uncertain_targets": [], + }, + ), + ) + + instance.do_POST() + + assert responses == [ + ( + 200, + { + "status": "outcome_unknown", + "revision": "a" * 40, + "changedTargets": [], + "changedTargetDetails": [], + "uncertainTargets": [STATE_PATH], + }, + ) + ] + + +def test_controller_artifact_manifests_include_new_fixed_helper(controller): + renderer_spec = importlib.util.spec_from_file_location( + "hub_permission_artifact_renderer", ROOT / "scripts/prepare_controller_artifact.py" + ) + assert renderer_spec is not None and renderer_spec.loader is not None + renderer = importlib.util.module_from_spec(renderer_spec) + renderer_spec.loader.exec_module(renderer) + volume_spec = importlib.util.spec_from_file_location( + "hub_permission_volume_preparer", ROOT / "scripts/prepare_controller_hub_release_volume.py" + ) + assert volume_spec is not None and volume_spec.loader is not None + volume = importlib.util.module_from_spec(volume_spec) + volume_spec.loader.exec_module(volume) + installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8") + + assert "hub_permission_preparation.py" in controller._CONTROLLER_ARTIFACT_FILES + assert "hub_permission_preparation.py" in renderer._ARTIFACT_FILES + assert tuple(volume.RELEASE_FILES) == tuple(renderer._ARTIFACT_FILES) + assert "ops/gitea-deploy-broker/hub_permission_preparation.py" in installer + assert "hub_permission_preparation.py" in installer diff --git a/backend/tests/test_mcp_hub_candidate_preparation.py b/backend/tests/test_mcp_hub_candidate_preparation.py index b4f8585..7159cff 100644 --- a/backend/tests/test_mcp_hub_candidate_preparation.py +++ b/backend/tests/test_mcp_hub_candidate_preparation.py @@ -272,6 +272,122 @@ def test_plan_freezes_protected_metadata_without_candidate_or_approval(harness): assert h.state.mutations == [] +def test_plan_candidate_freezes_the_selected_workflow_creation_timestamp( + harness, monkeypatch +): + h = harness + revision = h.revision + built_at = "2026-10-03T12:34:56.789Z" + observed = [] + metadata = { + "id": revision[:12], + "repository": h.controller.MCP_HUB_REPOSITORY, + "revision": revision, + "tree": "d" * 40, + "treeDigest": hashlib.sha256(("git-tree:" + "d" * 40).encode()).hexdigest(), + "builtAt": built_at, + } + monkeypatch.setattr(h.controller, "output", lambda _args: revision) + monkeypatch.setattr(h.controller, "validate_commit", lambda _repo, _branch, commit: commit) + + def deployment_metadata(_repo, supplied_revision, repository, *, built_at=None): + observed.append(built_at) + assert supplied_revision == revision + assert repository == h.controller.MCP_HUB_REPOSITORY + return dict(metadata) + + monkeypatch.setattr(h.controller, "deployment_metadata", deployment_metadata) + monkeypatch.setattr(h.controller, "mcp_hub_stage_release_source", lambda *_args: h.state.source) + monkeypatch.setattr(h.controller, "exact_source_matches_revision", lambda *_args: True) + compose_calls = [] + + def fixed_compose_modeler(project, revision, repository): + compose_calls.append((project, revision, repository)) + return { + "services": { + service: {"image": f"fixture/{service}:candidate"} + for service in h.controller.MCP_HUB_SERVICES + } + } + + monkeypatch.setattr( + h.controller, + "mcp_hub_compose_model", + lambda *_args: pytest.fail("plan used the ambient Compose renderer"), + ) + + staging = ( + h.controller.STATE_ROOT + / "mcp-hub-release-imports" + / revision + / ("1" * 32) + / revision + ) + h.utility.plan_candidate( + h.controller.MCP_HUB_REPOSITORY, + revision, + built_at=built_at, + staging_directory=staging, + compose_modeler=fixed_compose_modeler, + ) + + plan = json.loads( + (staging / "build.json").read_text() + ) + assert observed == [built_at] + assert plan["build"]["builtAt"] == built_at + assert len(compose_calls) == 1 + + +def test_candidate_preparation_rejects_staging_path_outside_private_import_anchor( + harness, +): + h = harness + revision = h.revision + staging = h.controller.STATE_ROOT / "outside-imports" / revision + with pytest.raises(h.controller.DeployError, match="private anchor|staging path"): + h.utility.plan_candidate( + h.controller.MCP_HUB_REPOSITORY, + revision, + staging_directory=staging, + ) + assert not staging.exists() + + +def test_candidate_preparation_rejects_symlinked_staging_destination(harness): + h = harness + revision = h.revision + anchor = h.controller.STATE_ROOT / "mcp-hub-release-imports" / revision + token = anchor / ("2" * 32) + token.mkdir(parents=True, mode=0o700) + target = h.controller.STATE_ROOT / "untrusted-target" + target.mkdir(mode=0o700) + staging = token / revision + staging.symlink_to(target, target_is_directory=True) + with pytest.raises(h.controller.DeployError): + h.utility.plan_candidate( + h.controller.MCP_HUB_REPOSITORY, + revision, + staging_directory=staging, + ) + + +def test_candidate_preparation_rejects_nonprivate_staging_parent(harness): + h = harness + revision = h.revision + anchor = h.controller.STATE_ROOT / "mcp-hub-release-imports" / revision + token = anchor / ("3" * 32) + token.mkdir(parents=True, mode=0o700) + token.chmod(0o755) + staging = token / revision + with pytest.raises(h.controller.DeployError, match="private|safe|canonical"): + h.utility.plan_candidate( + h.controller.MCP_HUB_REPOSITORY, + revision, + staging_directory=staging, + ) + + def test_wrong_repository_and_nonroot_fail_before_controller_access( modules, monkeypatch ): @@ -354,6 +470,37 @@ def test_capture_emits_candidate_and_pending_only_envelope_without_apply_calls( } +def test_capture_can_use_the_importers_fixed_image_inspector(harness, monkeypatch): + h = harness + inspected = [] + + def fixed_inspector(reference): + inspected.append(reference) + return h.state.images.get(reference) + + monkeypatch.setattr( + h.controller, + "inspect_image", + lambda _reference: pytest.fail("capture used the ambient Docker context"), + ) + monkeypatch.setattr( + h.controller, + "mcp_hub_compose_model", + lambda *_args: pytest.fail("capture used the ambient Compose renderer"), + ) + result = h.utility.capture_candidate( + h.controller.MCP_HUB_REPOSITORY, + h.revision, + image_inspector=fixed_inspector, + compose_modeler=lambda *_args: h.model, + ) + assert result["status"] == "pending_review" + assert len(inspected) == 17 + assert set(inspected) == { + row["image"] for row in h.model["services"].values() + } + + @pytest.mark.parametrize("mutation", ["label", "root"]) def test_capture_rejects_contextforge_provenance_drift_without_candidate( harness, mutation diff --git a/backend/tests/test_mcp_hub_governed_topology.py b/backend/tests/test_mcp_hub_governed_topology.py index 66a26ca..5df1638 100644 --- a/backend/tests/test_mcp_hub_governed_topology.py +++ b/backend/tests/test_mcp_hub_governed_topology.py @@ -48,6 +48,138 @@ def _legacy_target(controller): "source_dir": (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix(), } + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="backup ownership contract requires root-owned POSIX fixtures", +) +def test_hub_backup_child_is_created_with_controller_owner_and_private_mode( + controller, tmp_path, monkeypatch +): + state_root = tmp_path / "controller-state" + backup_root = state_root / "mcp-hub-backups" + state_root.mkdir(mode=0o700) + backup_root.mkdir(mode=0o700) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + + attempt = backup_root / "attempt" + controller._mcp_hub_private_directory(attempt, create=True, root=backup_root) + + metadata = attempt.stat() + assert (metadata.st_uid, metadata.st_gid) == (0, 0) + assert stat.S_IMODE(metadata.st_mode) == 0o700 + controller._mcp_hub_private_directory(attempt, root=backup_root) + + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="backup root contract requires fixed-owner POSIX fixtures", +) +def test_hub_backup_root_rejects_group_readable_root(controller, tmp_path, monkeypatch): + state_root = tmp_path / "controller-state" + backup_root = state_root / "mcp-hub-backups" + state_root.mkdir(mode=0o700) + backup_root.mkdir(mode=0o750) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + + with pytest.raises(controller.DeployError, match="not private and canonical"): + controller._mcp_hub_private_directory(backup_root, root=backup_root) + + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="fixed Hub backup owner contract requires root-owned POSIX fixtures", +) +def test_hub_backup_root_rejects_runtime_owner_even_when_it_matches_parent( + controller, tmp_path, monkeypatch +): + state_root = tmp_path / "controller-state" + backup_root = state_root / "mcp-hub-backups" + state_root.mkdir(mode=0o700) + backup_root.mkdir(mode=0o700) + os.chown(state_root, 99, 100) + os.chown(backup_root, 99, 100) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + + with pytest.raises(controller.DeployError, match="owner differs"): + controller._mcp_hub_private_directory(backup_root, root=backup_root) + + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="descriptor-relative private storage requires root-owned POSIX fixtures", +) +def test_hub_backup_anchor_is_created_under_the_pinned_controller_state( + controller, tmp_path, monkeypatch +): + state_root = tmp_path / "controller-state" + state_root.mkdir(mode=0o755) + backup_root = state_root / "mcp-hub-backups" + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + + controller._mcp_hub_private_directory(backup_root, create=True) + + metadata = backup_root.stat() + assert (metadata.st_uid, metadata.st_gid) == (0, 0) + assert stat.S_IMODE(metadata.st_mode) == 0o700 + assert stat.S_IMODE(state_root.stat().st_mode) == 0o755 + + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="descriptor-relative path replacement requires root-owned POSIX fixtures", +) +def test_hub_backup_child_rejects_parent_replacement_during_creation( + controller, tmp_path, monkeypatch +): + state_root = tmp_path / "controller-state" + state_root.mkdir(mode=0o700) + backup_root = state_root / "mcp-hub-backups" + backup_root.mkdir(mode=0o700) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + real_mkdir = controller.os.mkdir + + def replace_parent(path, mode=0o777, *, dir_fd=None): + real_mkdir(path, mode, dir_fd=dir_fd) + if path == "attempt" and dir_fd is not None: + backup_root.rename(state_root / "replaced-backups") + real_mkdir(backup_root, 0o700) + + monkeypatch.setattr(controller.os, "mkdir", replace_parent) + + with pytest.raises(controller.DeployError, match="could not be created safely"): + controller._mcp_hub_private_directory(backup_root / "attempt", create=True) + assert not (backup_root / "attempt").exists() + + +@pytest.mark.skipif(os.name == "nt", reason="release receipt storage requires POSIX mode and ACL metadata") +def test_hub_backup_receipt_directory_is_guarded_by_its_release_volume_parent( + controller, tmp_path, monkeypatch +): + volume_root = tmp_path / "controller-hub-releases" + state_root = volume_root / "state" + receipt_root = state_root / "backup-receipts" + receipt_root.mkdir(mode=0o700, parents=True) + volume_root.chmod(0o700) + state_root.chmod(0o700) + monkeypatch.setattr(controller, "MCP_HUB_RELEASE_STORE_STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_RECEIPT_ROOT", receipt_root) + + controller._mcp_hub_private_directory(receipt_root, root=state_root) + with pytest.raises(controller.DeployError, match="escapes its fixed parent"): + controller._mcp_hub_private_directory(receipt_root, root=controller.STATE_ROOT) + + def _valid_model(controller): services = {} for service_name in controller.MCP_HUB_SERVICES: @@ -159,14 +291,11 @@ def test_installed_legacy_registry_shape_is_adapted_only_in_memory(controller): "docker-compose.prod.yml", "docker-compose.autodeploy.yml", ], - "exact_source": None, - "verify_oci_revision": None, "manual": False, "deploy_script": "", "archive_deploy_script": "", "live_probe": None, "managed_plan_version": "", - "clean_untracked_paths": None, "env_source": "/srv/example/appdata/itworx-mcp-hub/deploy.env", } original = copy.deepcopy(target) @@ -182,8 +311,15 @@ def test_installed_legacy_registry_shape_is_adapted_only_in_memory(controller): for field, value in ( ("source_dir", "/srv/example/appdata/other/repository"), ("compose_files", ["docker-compose.yml", "unexpected.yml"]), + ("exact_source", None), + ("exact_source", False), ("exact_source", True), + ("verify_oci_revision", None), ("verify_oci_revision", False), + ("verify_oci_revision", True), + ("clean_untracked_paths", None), + ("clean_untracked_paths", False), + ("clean_untracked_paths", []), ("env_source", "/tmp/untrusted.env"), ): changed = {**original, field: value} @@ -921,14 +1057,28 @@ def test_database_size_query_is_limited_to_two_fixed_database_names(controller, assert "pg_database_size" in query assert captured["timeout"] == 30 + +@pytest.mark.skipif( + os.name != "nt" and (not hasattr(os, "geteuid") or os.geteuid() != 0), + reason="backup cleanup ownership fixture requires root on POSIX", +) def test_failed_backup_attempt_removes_only_its_unique_private_directory( controller, monkeypatch, tmp_path ): - backup_root = tmp_path / "backups" + appdata_root = tmp_path / "hub-appdata" + backup_root = appdata_root / "backups" + appdata_root.mkdir(mode=0o700) backup_root.mkdir(mode=0o700) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) - monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", tmp_path) - monkeypatch.setattr(controller, "_mcp_hub_private_directory", lambda *_args, **_kwargs: None) + monkeypatch.setattr(controller, "STATE_ROOT", appdata_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata_root) + private_directory = controller._mcp_hub_private_directory + monkeypatch.setattr( + controller, + "_mcp_hub_private_directory", + lambda path, **kwargs: private_directory(path, **{"root": backup_root, **kwargs}), + ) monkeypatch.setattr(controller, "mcp_hub_validate_secret_files", lambda _model: 21) monkeypatch.setattr( controller, diff --git a/backend/tests/test_mcp_hub_legacy_registry_compatibility.py b/backend/tests/test_mcp_hub_legacy_registry_compatibility.py new file mode 100644 index 0000000..ab1b71f --- /dev/null +++ b/backend/tests/test_mcp_hub_legacy_registry_compatibility.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import pytest + +from backend.tests.test_mcp_hub_governed_topology import _legacy_target, _load_controller + + +LEGACY_SOURCE = "/srv/example/appdata/itworx-mcp-hub/deployment/repository" + + +def observed_legacy_target(controller): + project = _legacy_target(controller) + project.pop("exact_source") + project.pop("verify_oci_revision") + project["source_dir"] = LEGACY_SOURCE + project["compose_dir"] = "infrastructure/compose" + project["compose_files"] = list(controller.MCP_HUB_COMPOSE_FILES[:3]) + project["env_source"] = "/srv/example/appdata/itworx-mcp-hub/deploy.env" + return project + + +@pytest.mark.parametrize("legacy", [False, True]) +def test_observed_hub_legacy_source_is_only_normalized_in_memory(legacy): + controller = _load_controller() + project = observed_legacy_target(controller) if legacy else _legacy_target(controller) + canonical = (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix() + original = LEGACY_SOURCE if legacy else canonical + project["source_dir"] = original + + effective = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project) + + assert effective["source_dir"] == canonical + assert project["source_dir"] == original + assert effective["containers"] == list(controller.MCP_HUB_CONTAINERS) + assert effective["compose_files"] == list(controller.MCP_HUB_COMPOSE_FILES) + assert effective["_mcp_hub_contract"] == controller.MCP_HUB_TOPOLOGY_CONTRACT + + +def test_observed_legacy_registry_omissions_are_upgraded_only_in_memory(): + controller = _load_controller() + project = observed_legacy_target(controller) + effective = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project) + assert effective["exact_source"] is True + assert effective["verify_oci_revision"] is True + assert "exact_source" not in project + assert "verify_oci_revision" not in project + + +@pytest.mark.parametrize( + "field,value", + [ + ("source_dir", "/mnt/cache/appdata/gitea-deploy-runners/sources/itworx-mcp-hub"), + ("compose_dir", "."), + ("compose_files", ["other.yml"]), + ("env_source", "/tmp/deploy.env"), + ("exact_source", None), + ("exact_source", False), + ("exact_source", True), + ("verify_oci_revision", None), + ("verify_oci_revision", False), + ("verify_oci_revision", True), + ("clean_untracked_paths", None), + ("clean_untracked_paths", False), + ("clean_untracked_paths", []), + ("clean_untracked_paths", ["local.env"]), + ], +) +def test_legacy_registry_omissions_do_not_widen_other_target_shapes(field, value): + controller = _load_controller() + project = observed_legacy_target(controller) + project[field] = value + with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): + controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project) + + +@pytest.mark.parametrize("field", ["exact_source", "verify_oci_revision"]) +def test_fully_declared_legacy_target_rejects_explicit_false_verification(field): + controller = _load_controller() + project = observed_legacy_target(controller) + project[field] = False + with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): + controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project) + + +@pytest.mark.parametrize( + "source", + [ + None, + True, + {}, + LEGACY_SOURCE + "/", + LEGACY_SOURCE + "/../repository", + "/mnt/cache/appdata/itworx-mcp-hub/deployment/repository", + "/srv/example/appdata/another-hub/deployment/repository", + "/tmp/repository", + ], +) +def test_hub_legacy_source_compatibility_rejects_other_spellings_and_types(source): + controller = _load_controller() + project = _legacy_target(controller) + project["source_dir"] = source + + with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): + controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project) diff --git a/backend/tests/test_mcp_hub_release_store.py b/backend/tests/test_mcp_hub_release_store.py index 818d058..0e88d53 100644 --- a/backend/tests/test_mcp_hub_release_store.py +++ b/backend/tests/test_mcp_hub_release_store.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import hashlib import importlib.util import json import os @@ -106,6 +107,91 @@ def test_fixed_hub_state_reader_is_bounded_and_rejects_symlink_or_missing_state( assert attacker_state.read_bytes() == b'{"status":"attacker"}' +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="Hub state reader verifies root-owned fixed storage", +) +@pytest.mark.parametrize("state_file_mode", [0o640, 0o644, 0o660, 0o664, 0o666]) +def test_fixed_hub_state_reader_rejects_other_file_modes( + store, tmp_path, monkeypatch, state_file_mode +): + controller_root = tmp_path / "controller" + state_root = controller_root / "state" + state_root.mkdir(parents=True, mode=0o700) + controller_root.chmod(0o700) + state_file = state_root / store.HUB_STATE_FILENAME + state_file.write_bytes(b'{"status":"success"}') + state_file.chmod(state_file_mode) + monkeypatch.setattr(store, "CONTROLLER_STATE_PARENT", controller_root) + monkeypatch.setattr(store, "CONTROLLER_STATE_DIRECTORY", state_root) + + with pytest.raises(store.ReleaseStoreError, match="unavailable or untrusted"): + store.read_hub_state_bytes(missing_ok=False) + + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="external backup owner checks require root-owned POSIX fixtures", +) +def test_descriptor_helpers_verify_an_exact_external_owner(store, tmp_path): + expected_owner = (99, 100) + backup_root = tmp_path / "backups" + backup_root.mkdir(mode=0o700) + backup_root.chmod(0o700) + os.chown(backup_root, *expected_owner) + root_fd = os.open(backup_root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + attempt_fd = None + try: + attempt_fd = store.open_child_directory( + root_fd, + "attempt", + create=True, + mode=0o700, + expected_owner=expected_owner, + ) + artifact_fd = os.open( + "manifest.json", + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=attempt_fd, + ) + try: + os.write(artifact_fd, b"private manifest") + os.fchmod(artifact_fd, 0o600) + os.fchown(artifact_fd, *expected_owner) + finally: + os.close(artifact_fd) + + assert store.read_at( + attempt_fd, + "manifest.json", + max_bytes=64, + expected_mode=0o600, + expected_owner=expected_owner, + ) == b"private manifest" + digest, size = store.sha256_at( + attempt_fd, + "manifest.json", + max_bytes=64, + expected_mode=0o600, + expected_owner=expected_owner, + ) + assert digest == hashlib.sha256(b"private manifest").hexdigest() + assert size == len(b"private manifest") + with pytest.raises(store.ReleaseStoreError, match="metadata is unsafe"): + store.read_at( + attempt_fd, + "manifest.json", + max_bytes=64, + expected_mode=0o600, + expected_owner=(0, 0), + ) + finally: + if attempt_fd is not None: + os.close(attempt_fd) + os.close(root_fd) + + @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="Hub state reader verifies root-owned fixed storage", diff --git a/backend/tests/test_media_codec_dashboard_deployment_reconciliation.py b/backend/tests/test_media_codec_dashboard_deployment_reconciliation.py index f259b54..29ef77b 100644 --- a/backend/tests/test_media_codec_dashboard_deployment_reconciliation.py +++ b/backend/tests/test_media_codec_dashboard_deployment_reconciliation.py @@ -69,9 +69,8 @@ def test_reconciliation_derives_the_verified_live_revision_from_runtime_evidence "Media-Codec-Dashboard", ] script = command[2] - # The controller registry must be the canonical physical cache bind, not - # the user-share alias that can resolve to a different inode. - assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script + # The mutable registry must identify the protected volume's fixed config. + assert "/controller-hub-releases/controller-config/projects.json" in script assert "/srv/example/appdata/gitea-deploy-runners/controller/projects.json" not in script assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script diff --git a/backend/tests/test_portainer_restart_timeout.py b/backend/tests/test_portainer_restart_timeout.py index 887bbf1..0a20a83 100644 --- a/backend/tests/test_portainer_restart_timeout.py +++ b/backend/tests/test_portainer_restart_timeout.py @@ -1,8 +1,47 @@ from types import SimpleNamespace +import pytest + from backend.app.connectors.portainer import PortainerConnector from backend.app.core.config import settings +CONTROLLER_OWNED_CI_NAMES = ( + "itworx-ci-checks", + "itworx-ci-backend", + "itworx-ci-frontend", + "itworx-ci-security", + "itworx-ci-browser", + "itworx-ci-build", + "itworx-security-runner", + "itworx-validation-runner-2", + "itworx-validation-runner-3", + "itworx-docker-runner", + "itworx-browser-runner", + "itworx-validation-runner-burst-1", + "gitea-runner-dind", +) + + +def _connector_with_containers(monkeypatch, containers): + monkeypatch.setitem(settings.apis, "PORTAINER_URL", "https://portainer.example.test") + monkeypatch.setitem(settings.apis, "PORTAINER_API_KEY", "test-key") + monkeypatch.setitem(settings.apis, "PORTAINER_ENDPOINT_ID", "9") + + connector = PortainerConnector() + monkeypatch.setattr( + connector, + "list_containers", + lambda: {"status": "ok", "containers": containers}, + ) + calls = [] + + def request(method: str, path: str, **kwargs): + calls.append((method, path, kwargs)) + return SimpleNamespace(status_code=204) + + monkeypatch.setattr(connector, "_request", request) + return connector, calls + def test_restart_request_allows_docker_grace_period_without_shortening_container_timeout(monkeypatch): previous = dict(settings.apis) @@ -17,7 +56,7 @@ def test_restart_request_allows_docker_grace_period_without_shortening_container "list_containers", lambda: { "status": "ok", - "containers": [{"id": "runner-id", "name": "itworx-validation-runner-2", "appId": "itworx-validation-runner-2"}], + "containers": [{"id": "user-container-id", "name": "jellyfin", "appId": "jellyfin"}], }, ) calls: list[tuple[str, str, dict]] = [] @@ -28,7 +67,7 @@ def test_restart_request_allows_docker_grace_period_without_shortening_container monkeypatch.setattr(connector, "_request", request) try: - result = connector.restart_container("itworx-validation-runner-2") + result = connector.restart_container("jellyfin") finally: settings.apis.clear() settings.apis.update(previous) @@ -36,6 +75,67 @@ def test_restart_request_allows_docker_grace_period_without_shortening_container assert result["status"] == "ok" assert calls == [( "POST", - "/api/endpoints/9/docker/containers/runner-id/restart", + "/api/endpoints/9/docker/containers/user-container-id/restart", + {"timeout": 30.0}, + )] + + +@pytest.mark.parametrize("container_name", CONTROLLER_OWNED_CI_NAMES) +def test_restart_blocks_fixed_controller_owned_ci_containers_by_name(monkeypatch, container_name): + connector, calls = _connector_with_containers( + monkeypatch, + [{"id": "0123456789abcdef0123456789abcdef", "name": container_name, "appId": "ci-stack"}], + ) + + result = connector.restart_container(container_name) + + assert result == {"status": "blocked", "reason": "controller_owned_ci_lifecycle"} + assert calls == [] + + +@pytest.mark.parametrize( + "target", + ( + "itworx-ci-backend", + "0123456789abcdef0123456789abcdef", + "01234567", + "backend-ci-stack", + ), +) +def test_restart_blocks_controller_owned_container_selected_by_alias(monkeypatch, target): + connector, calls = _connector_with_containers( + monkeypatch, + [{ + "id": "0123456789abcdef0123456789abcdef", + "name": "itworx-ci-backend", + "appId": "backend-ci-stack", + }], + ) + + result = connector.restart_container(target) + + assert result == {"status": "blocked", "reason": "controller_owned_ci_lifecycle"} + assert calls == [] + + +@pytest.mark.parametrize("container_name", ("itworx-unraid-deploy", "jellyfin")) +def test_restart_still_allows_unrelated_deploy_and_user_containers(monkeypatch, container_name): + container_id = f"id-{container_name}" + connector, calls = _connector_with_containers( + monkeypatch, + [{"id": container_id, "name": container_name, "appId": container_name}], + ) + + result = connector.restart_container(container_name) + + assert result == { + "status": "ok", + "container": container_id, + "target": container_name, + "action": "container.restart", + } + assert calls == [( + "POST", + f"/api/endpoints/9/docker/containers/{container_id}/restart", {"timeout": 30.0}, )] diff --git a/backend/tests/test_portfolio_deployment_enrollment.py b/backend/tests/test_portfolio_deployment_enrollment.py index 022c15e..697364e 100644 --- a/backend/tests/test_portfolio_deployment_enrollment.py +++ b/backend/tests/test_portfolio_deployment_enrollment.py @@ -55,9 +55,9 @@ def test_portfolio_enrollment_contract_is_fixed_and_paired(): def test_portfolio_enrollment_registry_alias_matches_controller_cache_bind_source(): - assert 'registry_path = Path("/opt/gitea-deploy/projects.json")' in _ENROLL_SCRIPT + assert 'registry_path = Path("/controller-config/projects.json")' in _ENROLL_SCRIPT assert ( - 'host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")' + 'host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")' in _ENROLL_SCRIPT ) assert "os.path.samefile(registry_path, host_registry_path)" in _ENROLL_SCRIPT diff --git a/backend/tests/test_privatemuse_deployment_enrollment.py b/backend/tests/test_privatemuse_deployment_enrollment.py index fa91c97..5fd5220 100644 --- a/backend/tests/test_privatemuse_deployment_enrollment.py +++ b/backend/tests/test_privatemuse_deployment_enrollment.py @@ -75,7 +75,7 @@ def test_enrollment_uses_fixed_private_muse_contract_and_controller_exec(): assert "appops_managed_target" not in canonical_payload script = command[2] - assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script + assert "/controller-hub-releases/controller-config/projects.json" in script assert "/srv/example/appdata/gitea-deploy-runners/controller/projects.json" not in script assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script diff --git a/backend/tests/test_privatemuse_webhook_repair.py b/backend/tests/test_privatemuse_webhook_repair.py index 013e986..87e2bd5 100644 --- a/backend/tests/test_privatemuse_webhook_repair.py +++ b/backend/tests/test_privatemuse_webhook_repair.py @@ -81,7 +81,7 @@ def test_privatemuse_webhook_repair_executes_only_fixed_contract(): def test_privatemuse_webhook_repair_script_never_returns_credentials(): assert "runner_guard_admin_token" in _REPAIR_SCRIPT - assert 'secret_root = Path("/opt/gitea-deploy/webhook-secrets")' in _REPAIR_SCRIPT + assert 'secret_root = Path("/controller-config/webhook-secrets")' in _REPAIR_SCRIPT assert 'secret_path = secret_root / "jens__privatemuse"' in _REPAIR_SCRIPT assert "secretMaterialReturned" not in _REPAIR_SCRIPT assert 'print(f"privatemuse_runnerless_webhook_ready hook_id={hook_id}"' in _REPAIR_SCRIPT diff --git a/backend/tests/test_repopulse_target_attestation.py b/backend/tests/test_repopulse_target_attestation.py index d00a010..41d4cab 100644 --- a/backend/tests/test_repopulse_target_attestation.py +++ b/backend/tests/test_repopulse_target_attestation.py @@ -5,11 +5,15 @@ from __future__ import annotations import hashlib import hmac import importlib +import importlib.util import json import sqlite3 +import sys from datetime import datetime, timedelta, timezone from pathlib import Path +import pytest + def _module(monkeypatch): root = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker" @@ -17,6 +21,52 @@ def _module(monkeypatch): return importlib.import_module("repopulse_target_attestation") +@pytest.mark.parametrize("code_root", [Path("/opt/gitea-deploy"), Path("/private/controller-stage")]) +def test_bootstrap_producer_and_verifier_share_the_config_path(monkeypatch, code_root): + root = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker" + monkeypatch.syspath_prepend(str(root)) + spec = importlib.util.spec_from_file_location("config_path_attestation_test", root / "repopulse_target_attestation.py") + verifier = importlib.util.module_from_spec(spec) + verifier.__file__ = str(code_root / "repopulse_target_attestation.py") + exec(compile((root / "repopulse_target_attestation.py").read_bytes(), str(spec.origin), "exec"), verifier.__dict__) + expected = ( + Path("/controller-hub-releases/controller-config") + if code_root == Path("/opt/gitea-deploy") else code_root + ) / "appops-enrollment-bootstrap" + assert verifier.BOOTSTRAP_PATH == expected + monkeypatch.setitem(sys.modules, "repopulse_target_attestation", verifier) + producer_spec = importlib.util.spec_from_file_location("config_path_producer_test", root / "stage_repopulse_target_attestation.py") + producer = importlib.util.module_from_spec(producer_spec) + producer_spec.loader.exec_module(producer) + observed = [] + + def unavailable_secret(path, _maximum): + observed.append(path) + raise OSError("fixture stops before any external operation") + + monkeypatch.setattr(producer.os, "geteuid", lambda: 0, raising=False) + monkeypatch.setattr(producer.sys, "argv", ["stage_repopulse_target_attestation.py"]) + monkeypatch.setattr(producer, "_private_bytes", unavailable_secret) + assert producer.main() == 2 + assert observed == [expected] + observed.clear() + + def verifier_file(path, *, maximum): + if path == verifier.ATTESTATION_PATH: + return json.dumps({"schema": verifier.ENVELOPE_SCHEMA, "claims": {}, "signatureSha256": "0" * 64}).encode() + return unavailable_secret(path, maximum) + + monkeypatch.setattr(verifier, "_private_file", verifier_file) + required = { + "repository", "applicationId", "profileId", "profileVersion", + "planSha256", "intentId", "targetSecretRef", "controllerTlsCertificateSha256", + } + assert verifier.FixedTargetAttestationVerifier().verify_target_attestation( + expected={key: "fixture" for key in required} + ) is None + assert observed == [expected] + + def _sample(now: datetime) -> dict: return { "schema": "itworx.repopulse-target-attestation/v1", diff --git a/backend/tests/test_runner_admission_controller.py b/backend/tests/test_runner_admission_controller.py new file mode 100644 index 0000000..31459a0 --- /dev/null +++ b/backend/tests/test_runner_admission_controller.py @@ -0,0 +1,1640 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +import sys +import time +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from backend.app.services.runner_statistics import sanitize_statistics + +ROOT = Path(__file__).resolve().parents[2] +CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker" + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def controller_module(monkeypatch): + profiles = _load( + "appops_runner_profiles_for_admission_controller_test", + CONTROLLER_ROOT / "runner_profiles.py", + ) + monkeypatch.setitem(sys.modules, "runner_profiles", profiles) + policy = _load( + "appops_runner_admission_policy_for_controller_test", + CONTROLLER_ROOT / "runner_admission_policy.py", + ) + monkeypatch.setitem(sys.modules, "runner_admission_policy", policy) + return _load( + "appops_runner_admission_controller_test", + CONTROLLER_ROOT / "runner_admission_controller.py", + ) + + +def _public_observation(module, now: float): + rows = {} + containers = {} + profile_runner_ids = {} + runner_profile_ids = {} + active = {} + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + runner_id = 100 + index + rows[profile.container_name] = { + "id": runner_id, + "name": profile.container_name, + "status": "offline", + "busy": False, + "disabled": True, + "labels": {profile.runner_label, *profile.legacy_labels}, + } + containers[profile.container_name] = {"running": False, "health": "none"} + profile_runner_ids[profile.id] = runner_id + runner_profile_ids[runner_id] = profile.id + active[runner_id] = [] + return { + "specialization": "active", + "at": now, + "runnerRows": rows, + "profileRunnerIds": profile_runner_ids, + "runnerProfileIds": runner_profile_ids, + "containers": containers, + "claims": [], + "queue": [ + { + "id": "private-job-id", + "labels": ["linux-validation"], + "queued_at": now - 44, + } + ], + "activeJobs": active, + "resourceProof": { + "ready": True, + "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, + "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, + "activeClaims": 0, + "heavyClaims": 0, + "runningRunnerContainers": 0, + "cgroups": {}, + "digest": "a" * 64, + }, + "resourceReason": None, + "hostReason": None, + "capacity": { + "host": { + "availableMemoryBytes": 12 * 1024**3, + "cacheFreeBytes": 40 * 1024**3, + "loadRatio": 0.25, + } + }, + } + + +def _install_gitea_observation( + monkeypatch, + controller, + module, + *, + queued_rows=(), + pending_rows=(), + active_rows=(), +): + runner_rows = {} + by_id = {} + profile_runner_ids = {} + runner_profile_ids = {} + containers = {} + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + runner_id = 100 + index + row = { + "id": runner_id, + "name": profile.container_name, + "status": "offline", + "busy": False, + "disabled": True, + "labels": {profile.runner_label, *profile.legacy_labels}, + } + runner_rows[profile.container_name] = row + by_id[runner_id] = row + profile_runner_ids[profile.id] = runner_id + runner_profile_ids[runner_id] = profile.id + containers[profile.container_name] = { + "id": f"{index:064x}", + "running": False, + "health": "none", + } + containers[module.DIND_NAME] = { + "id": "f" * 64, + "running": True, + "health": "healthy", + } + proof = { + "ready": True, + "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, + "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, + "activeClaims": 0, + "heavyClaims": 0, + "runningRunnerContainers": 0, + "cgroups": {}, + "digest": "e" * 64, + } + + def job_rows(status): + if status == "queued": + return list(queued_rows), True + if status == "pending": + return list(pending_rows), True + if status == "in_progress": + return list(active_rows), True + return [], True + + monkeypatch.setattr(controller, "_specialization_state", lambda: "active") + monkeypatch.setattr(controller, "_runner_inventory", lambda: []) + monkeypatch.setattr(controller, "_normalized_runner_rows", lambda _rows: (runner_rows, by_id)) + monkeypatch.setattr(controller, "_job_rows", job_rows) + monkeypatch.setattr( + controller, + "_active_claims", + lambda *_args: ([], profile_runner_ids, runner_profile_ids), + ) + monkeypatch.setattr(controller, "_container", lambda name: containers.get(name)) + monkeypatch.setattr(controller, "_verify_profile_task_images", lambda *_args: True) + monkeypatch.setattr(controller, "_resource_attestation", lambda *_args: proof) + monkeypatch.setattr( + controller, + "_host_evidence", + lambda: ( + "clear", + None, + { + "host": { + "availableMemoryBytes": 8 * 1024**3, + "cacheFreeBytes": 20 * 1024**3, + "loadRatio": 0.25, + } + }, + ), + ) + + +def _lease_fixture(module, *, enabled=("backend", "frontend"), busy=(), queue=()): + controller = module.RunnerAdmissionController(clock=lambda: time.time()) + state = module._initial_state() + rows = {} + containers = {} + profile_runner_ids = {} + active_jobs = {} + inventory_rows = [] + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + runner_id = 100 + index + is_enabled = profile.id in enabled + is_busy = profile.id in busy + labels = [profile.runner_label, *profile.legacy_labels] + rows[profile.container_name] = { + "id": runner_id, + "name": profile.container_name, + "status": "online" if is_enabled else "offline", + "busy": is_busy, + "disabled": not is_enabled, + "labels": set(labels), + } + containers[profile.container_name] = { + "id": f"{index:064x}", + "running": is_enabled, + "health": "healthy" if is_enabled else "none", + } + profile_runner_ids[profile.id] = runner_id + state["registrationIds"][profile.id] = runner_id + state["profiles"][profile.id].update( + observed=True, + enabledByController=is_enabled, + operatorDisabled=False, + operatorEnabled=False, + ) + inventory_rows.append( + { + "name": profile.display_name, + "id": runner_id, + "status": "online" if is_enabled else "offline", + "busy": is_busy, + "disabled": not is_enabled, + "labels": labels, + } + ) + if is_busy: + active_jobs[runner_id] = [ + { + "id": f"active-{profile.id}", + "labels": labels, + "startedAt": module._utc_iso(), + "attempt": 1, + } + ] + by_id = {row["id"]: row for row in rows.values()} + claims, _profile_ids, _runner_profiles = module.RunnerAdmissionController._active_claims( + rows, by_id, active_jobs + ) + observation = { + "runnerRows": rows, + "containers": containers, + "activeJobs": active_jobs, + "profileRunnerIds": profile_runner_ids, + "claims": claims, + "queue": list(queue), + } + return controller, state, observation, inventory_rows, containers + + +def _raw_job(module, job, *, runner_id=None, active=False): + row = { + "id": job["id"], + "labels": list(job["labels"]), + "attempt": 1, + } + if active: + row.update( + runner_id=runner_id, + started_at=module._utc_iso(), + status="in_progress", + ) + else: + row["created_at"] = module._utc_iso(float(job["queued_at"])) + return row + + +def _install_lease_io( + monkeypatch, + controller, + module, + inventory_rows, + containers, + *, + queue_rows=(), + active_rows=(), + saved=None, +): + import copy + + def save(state): + if saved is not None: + saved.append(copy.deepcopy(state)) + + monkeypatch.setattr(controller, "_save_state", save) + + def job_rows(status): + if status == "queued" or status == "pending": + return list(queue_rows), True + if status == "in_progress": + return list(active_rows), True + return [], True + + monkeypatch.setattr( + controller, + "_job_rows", + job_rows, + ) + monkeypatch.setattr(controller, "_runner_inventory", lambda: [dict(row) for row in inventory_rows]) + monkeypatch.setattr(controller, "_container", lambda name: dict(containers[name])) + monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("clear", "")) + + def stop_container(args, **_kwargs): + assert args[:2] == ["docker", "stop"] + containers[args[2]]["running"] = False + containers[args[2]]["health"] = "none" + return "" + + monkeypatch.setattr(controller, "_docker_run", stop_container) + + +def _complete_state(module, now: float): + state = module._initial_state() + state["terminalHistoryComplete"] = True + state["terminalHistoryReason"] = None + state["historySamples"] = [ + { + "at": module._utc_iso(now - 30), + "activeJobs": 0, + "queuedJobs": 2, + "cpuUsed": 1.25, + "memoryUsedBytes": 2048, + }, + { + "at": module._utc_iso(now), + "activeJobs": 0, + "queuedJobs": 1, + "cpuUsed": 1.5, + "memoryUsedBytes": 4096, + }, + ] + state["terminalHistory"] = [ + { + "jobId": str(index), + "attempt": 1, + "profileId": "checks", + "result": "success", + "createdAt": module._utc_iso(now - 300 + index * 30), + "startedAt": module._utc_iso(now - 290 + index * 30), + "finishedAt": module._utc_iso(now - 250 + index * 30), + "waitSeconds": index * 10, + "durationSeconds": index * 20, + } + for index in range(1, 4) + ] + return state + + +def test_public_projection_matches_fixed_v1_and_uses_only_observed_statistics( + controller_module, +): + module = controller_module + now = time.time() + controller = module.RunnerAdmissionController(clock=lambda: now) + payload = controller._public_snapshot( + _public_observation(module, now), _complete_state(module, now) + ) + + assert set(payload) == { + "schemaVersion", + "status", + "generatedAt", + "scope", + "budget", + "host", + "profiles", + "window", + "history", + } + assert payload["scope"] == "ci-fleet" + assert payload["status"] == "healthy" + assert payload["budget"] == { + "cpuLimit": 8, + "memoryLimitBytes": 16 * 1024**3, + "activeJobLimit": 2, + "heavyJobLimit": 1, + "reservedCpu": 1, + "reservedMemoryBytes": 2 * 1024**3, + "activeJobs": 0, + "heavyJobs": 0, + } + assert payload["host"]["cpuBusyPercent"] == 18.75 + assert payload["profiles"][0]["cpuLimit"] is None + assert payload["profiles"][0]["memoryLimitBytes"] is None + backend = next(row for row in payload["profiles"] if row["id"] == "backend") + frontend = next(row for row in payload["profiles"] if row["id"] == "frontend") + assert backend["queueDepth"] == frontend["queueDepth"] == 1 + checks = payload["profiles"][0] + assert checks["completedJobs"] == 3 + assert checks["p50WaitSeconds"] == 20 + assert checks["p95WaitSeconds"] == 30 + assert checks["p50DurationSeconds"] == 40 + assert checks["p95DurationSeconds"] == 60 + assert checks["lastDurationSeconds"] == 60 + assert payload["window"]["coverage"] == "bounded" + assert "private-job-id" not in json.dumps(payload["history"]) + assert all(set(row) == {"at", "activeJobs", "queuedJobs", "cpuUsed", "memoryUsedBytes"} + for row in payload["history"]) + assert sanitize_statistics(payload)["status"] == "healthy" + + +def test_base_projection_does_not_claim_unattested_capacity(controller_module): + payload = controller_module.RunnerAdmissionController._base_snapshot( + "not_configured", "runner_admission_not_configured", time.time() + ) + assert payload["budget"] == { + "cpuLimit": None, + "memoryLimitBytes": None, + "activeJobLimit": None, + "heavyJobLimit": None, + "reservedCpu": None, + "reservedMemoryBytes": None, + "activeJobs": None, + "heavyJobs": None, + } + assert all(row["cpuLimit"] is None and row["memoryLimitBytes"] is None + for row in payload["profiles"]) + assert payload["window"]["coverage"] == "unavailable" + + +def test_capacity_projection_uses_independent_host_sampler_without_burst_state( + controller_module, monkeypatch +): + module = controller_module + calls = [] + sample = { + "cpuCount": 4, + "loadOne": 1.0, + "loadRatio": 0.25, + "availableMemoryBytes": 16 * 1024**3, + "cacheFreeBytes": 40 * 1024**3, + "swapInKbps": 0.0, + "swapOutKbps": 0.0, + "swapPressureSamples": 0, + "swapCritical": False, + "swapSampleAt": time.time(), + "swapInPages": 100, + "swapOutPages": 200, + } + + class HostSampler: + def host_snapshot(self, previous=None): + calls.append(previous) + return {"status": "observed", "generatedAt": "2026-10-01T00:00:00Z", "host": sample} + + monkeypatch.setitem( + sys.modules, + "runner_capacity_controller", + SimpleNamespace(RunnerCapacityController=HostSampler), + ) + monkeypatch.setitem( + sys.modules, + "runner_capacity_policy", + SimpleNamespace( + START_MAX_LOAD_RATIO=0.70, + START_MIN_AVAILABLE_MEMORY_BYTES=12 * 1024**3, + START_MIN_CACHE_FREE_BYTES=30 * 1024**3, + STOP_LOAD_RATIO=0.90, + STOP_MIN_AVAILABLE_MEMORY_BYTES=8 * 1024**3, + STOP_MIN_CACHE_FREE_BYTES=15 * 1024**3, + ), + ) + controller = module.RunnerAdmissionController() + + first = controller._capacity_snapshot() + second = controller._capacity_snapshot() + + assert calls[0] is None + assert calls[1] == sample + assert first["status"] == second["status"] == "ok" + assert first["decision"] == {"pressure": "headroom"} + assert second["host"]["swapOutPages"] == 200 + + +def test_capacity_projection_fails_closed_on_unavailable_host_sample( + controller_module, monkeypatch +): + module = controller_module + + class HostSampler: + def host_snapshot(self, previous=None): + return {"status": "unavailable", "host": None} + + monkeypatch.setitem( + sys.modules, + "runner_capacity_controller", + SimpleNamespace(RunnerCapacityController=HostSampler), + ) + controller = module.RunnerAdmissionController() + + with pytest.raises(module.RunnerAdmissionError, match="host_telemetry_unavailable"): + controller._capacity_snapshot() + + +def test_runner_inventory_maps_exact_gitea_display_names_to_fixed_containers( + controller_module, +): + module = controller_module + rows = [ + { + "name": profile.display_name, + "id": index, + "status": "offline", + "busy": False, + "disabled": True, + "labels": [profile.runner_label, *profile.legacy_labels], + } + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=10) + ] + + by_name, by_id = module.RunnerAdmissionController._normalized_runner_rows(rows) + + for profile in module.RUNNER_PROFILES.values(): + observed = by_name[profile.container_name] + assert observed["displayName"] == profile.display_name + assert observed["id"] in by_id + + canonical_alias = [dict(rows[0], name=module.RUNNER_PROFILES["checks"].container_name)] + with pytest.raises( + module.RunnerAdmissionError, + match="runner_admission_registration_name_invalid", + ): + module.RunnerAdmissionController._normalized_runner_rows(canonical_alias) + + duplicate_alias = [ + rows[0], + dict(rows[1], name=module.RUNNER_PROFILES["checks"].display_name), + ] + with pytest.raises( + module.RunnerAdmissionError, + match="runner_admission_runner_inventory_duplicate", + ): + module.RunnerAdmissionController._normalized_runner_rows(duplicate_alias) + + +def test_snapshot_reads_only_atomic_projection_and_downshifts_stale_health( + controller_module, monkeypatch, tmp_path +): + module = controller_module + now = time.time() + controller = module.RunnerAdmissionController(clock=lambda: now) + projection = module.RunnerAdmissionController._base_snapshot( + "healthy", "unused", now - 120 + ) + monkeypatch.setattr(module, "STATE_ROOT", tmp_path) + monkeypatch.setattr(module, "PUBLIC_STATISTICS_FILE", tmp_path / "statistics.json") + monkeypatch.setattr(module, "_safe_directory", lambda *_args, **_kwargs: True) + monkeypatch.setattr( + module.RunnerAdmissionController, + "_safe_private_file", + staticmethod(lambda *_args, **_kwargs: True), + ) + module.PUBLIC_STATISTICS_FILE.write_text(json.dumps(projection), encoding="utf-8") + monkeypatch.setattr(controller, "_observe", lambda **_kwargs: pytest.fail("snapshot polled")) + monkeypatch.setattr(controller, "_request", lambda *_args, **_kwargs: pytest.fail("snapshot called Gitea")) + monkeypatch.setattr(controller, "_docker_run", lambda *_args, **_kwargs: pytest.fail("snapshot called Docker")) + + result = controller.snapshot() + + assert result["status"] == "degraded" + assert result["generatedAt"] == projection["generatedAt"] + assert result["budget"]["cpuLimit"] is None + + +def test_projection_rejects_naive_future_and_out_of_order_history(controller_module): + module = controller_module + now = time.time() + assert module._parse_time("2026-10-01T12:00:00") is None + projection = module.RunnerAdmissionController._base_snapshot( + "healthy", "unused", now + ) + projection["history"] = [ + { + "at": module._utc_iso(now), + "activeJobs": 0, + "queuedJobs": 0, + "cpuUsed": None, + "memoryUsedBytes": None, + }, + { + "at": module._utc_iso(now - 30), + "activeJobs": 0, + "queuedJobs": 0, + "cpuUsed": None, + "memoryUsedBytes": None, + }, + ] + assert not module.RunnerAdmissionController._valid_projection(projection) + + +def test_resource_attestation_uses_fixed_outer_names_and_mapping_contract( + controller_module, +): + module = controller_module + requests = [] + proof = { + "ready": True, + "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, + "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, + "activeClaims": 1, + "heavyClaims": 0, + "runningRunnerContainers": 1, + "cgroups": {"familyPath": "/itworx-ci", "boundedGroups": 7}, + "digest": "b" * 64, + } + controller = module.RunnerAdmissionController( + resource_attestor=lambda **kwargs: requests.append(kwargs) or proof + ) + empty_legacy_inventory = { + name: None for name in module.ALL_LEGACY_RUNNER_NAMES + } + + result = controller._resource_attestation(["checks"], empty_legacy_inventory) + + assert result == proof + assert requests[0]["active_profiles"] == ("checks",) + assert requests[0]["expected_host_budget"] == {"cpuMilli": 8000, "memoryMiB": 16384} + assert requests[0]["expected_outer_containers"] == ( + *module.PROFILE_RUNNER_NAMES, + module.DIND_NAME, + ) + + +@pytest.mark.parametrize("verified", [True, False]) +def test_observation_requires_source_bound_profile_task_images( + controller_module, monkeypatch, verified +): + module = controller_module + calls = [] + fake_specializer = SimpleNamespace( + verify_profile_task_images=lambda profile_ids: calls.append(tuple(profile_ids)) + or verified + ) + monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer) + attestation_requests = [] + proof = { + "ready": True, + "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, + "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, + "activeClaims": 0, + "heavyClaims": 0, + "runningRunnerContainers": 0, + "cgroups": {}, + "digest": "f" * 64, + } + controller = module.RunnerAdmissionController( + resource_attestor=lambda **kwargs: attestation_requests.append(kwargs) or proof + ) + rows = {} + by_id = {} + profile_runner_ids = {} + runner_profile_ids = {} + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + row = { + "id": index, + "name": profile.container_name, + "disabled": True, + "busy": False, + "labels": {profile.runner_label, *profile.legacy_labels}, + } + rows[profile.container_name] = row + by_id[index] = row + profile_runner_ids[profile.id] = index + runner_profile_ids[index] = profile.id + + monkeypatch.setattr(controller, "_specialization_state", lambda: "active") + monkeypatch.setattr(controller, "_runner_inventory", lambda: []) + monkeypatch.setattr(controller, "_normalized_runner_rows", lambda _rows: (rows, by_id)) + monkeypatch.setattr(controller, "_job_rows", lambda _status: ([], True)) + monkeypatch.setattr(controller, "_normalized_jobs", lambda *_args: ([], {})) + monkeypatch.setattr( + controller, + "_active_claims", + lambda *_args: ([], profile_runner_ids, runner_profile_ids), + ) + monkeypatch.setattr(controller, "_container", lambda name: {"name": name, "running": False, "health": "none"}) + monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, {})) + + observation = controller._observe() + + assert calls == [tuple(module.PROFILE_IDS)] + assert observation["taskImagesVerified"] is verified + if verified: + assert observation["resourceProof"] == proof + assert observation["resourceReason"] is None + assert len(attestation_requests) == 1 + else: + assert observation["resourceProof"] is None + assert observation["resourceReason"] == "runner_admission_task_image_unverified" + assert attestation_requests == [] + + +def test_dependency_pending_job_does_not_count_or_admit_as_runnable( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + controller = module.RunnerAdmissionController(clock=lambda: now) + pending_build = { + "id": 101, + "labels": [module.RUNNER_PROFILES["build"].runner_label], + "attempt": 1, + "status": "requested", + "created_at": module._utc_iso(now - 600), + } + queued_checks = { + "id": 102, + "labels": [module.RUNNER_PROFILES["checks"].runner_label], + "attempt": 1, + "created_at": module._utc_iso(now - 30), + } + _install_gitea_observation( + monkeypatch, + controller, + module, + queued_rows=[queued_checks], + pending_rows=[pending_build], + ) + + observation = controller._observe() + state = module._initial_state() + assert controller._record_aggregate_sample(state, observation) + projection = controller._public_snapshot(observation, state) + choice = module.select_admission( + observation["queue"], + available_profiles=controller._enabled_candidate_profiles(observation, state), + active_claims=[], + cursor=0, + ) + + assert [job["id"] for job in observation["queue"]] == ["102"] + assert next(row for row in projection["profiles"] if row["id"] == "checks")["queueDepth"] == 1 + build = next(row for row in projection["profiles"] if row["id"] == "build") + assert build["queueDepth"] == 0 + assert build["oldestQueuedSeconds"] is None + assert projection["history"][-1]["queuedJobs"] == 1 + assert (choice.profile_id, choice.job_id) == ("checks", "102") + + +def test_observation_still_rejects_malformed_pending_inventory( + controller_module, monkeypatch +): + module = controller_module + controller = module.RunnerAdmissionController() + _install_gitea_observation(monkeypatch, controller, module) + original_job_rows = controller._job_rows + + def malformed_pending_rows(status): + if status == "pending": + return ([{"id": 103, "labels": [], "created_at": module._utc_iso()}], True) + return original_job_rows(status) + + monkeypatch.setattr(controller, "_job_rows", malformed_pending_rows) + + with pytest.raises( + module.RunnerAdmissionError, + match="runner_admission_job_inventory_invalid", + ): + controller._observe() + + +def test_pending_only_dependency_job_does_not_start_a_runner( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + controller = module.RunnerAdmissionController(clock=lambda: now) + pending_build = { + "id": 101, + "labels": [module.RUNNER_PROFILES["build"].runner_label], + "attempt": 1, + "status": "requested", + "created_at": module._utc_iso(now - 600), + } + _install_gitea_observation( + monkeypatch, + controller, + module, + pending_rows=[pending_build], + ) + state = module._initial_state() + state["terminalHistoryUpdatedAt"] = now + starts = [] + enables = [] + projections = [] + monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) + monkeypatch.setattr(controller, "_read_state", lambda: state) + monkeypatch.setattr(controller, "_save_state", lambda _state: None) + monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False) + monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False) + monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False) + monkeypatch.setattr(controller, "_write_projection", projections.append) + monkeypatch.setattr(controller, "_docker_run", lambda args, **_kwargs: starts.append(args)) + monkeypatch.setattr(controller, "_set_disabled", lambda *args: enables.append(args)) + monkeypatch.setattr( + controller, + "_continue_pending", + lambda *_args, **_kwargs: pytest.fail("no runner startup intent should be created"), + ) + + result = controller.cycle() + + assert result["status"] == "runner_admission_no_compatible_queue" + assert starts == [] + assert enables == [] + assert state["pending"] is None + assert next(row for row in projections[0]["profiles"] if row["id"] == "build")["queueDepth"] == 0 + + +def test_aggregate_history_uses_counter_deltas_and_real_memory(controller_module): + module = controller_module + now = time.time() + controller = module.RunnerAdmissionController(clock=lambda: now) + state = module._initial_state() + state["lastFamilyCpuUsageUsec"] = 100_000_000 + state["lastFamilyCpuSampleAt"] = now - 20 + observation = { + "activeJobs": {21: [{"id": "active-1"}]}, + "queue": [{"id": "queued-1"}, {"id": "queued-2"}], + "resourceProof": { + "familyCpuUsageUsec": 110_000_000, + "familyMemoryCurrentBytes": 512 * 1024**2, + }, + } + + assert controller._record_aggregate_sample(state, observation) + assert state["historySamples"][-1] == { + "at": module._utc_iso(now), + "activeJobs": 1, + "queuedJobs": 2, + "cpuUsed": 0.5, + "memoryUsedBytes": 512 * 1024**2, + } + + state["lastFamilyCpuUsageUsec"] = 200_000_000 + state["lastFamilyCpuSampleAt"] = now - 20 + state["historySamples"][-1]["at"] = module._utc_iso(now - 30) + observation["resourceProof"]["familyCpuUsageUsec"] = 100_000_000 + now += 31 + controller._clock = lambda: now + assert controller._record_aggregate_sample(state, observation) + assert state["historySamples"][-1]["cpuUsed"] is None + + +def test_controller_owned_drain_rechecks_operator_and_allows_guard_pause( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + controller, state, observation, inventory, containers = _lease_fixture( + module, enabled=("checks",) + ) + entry = state["profiles"]["checks"] + entry.update( + idleSince=now - module.IDLE_DRAIN_SECONDS - 1, + ) + profile = module.RUNNER_PROFILES["checks"] + runner_id = observation["profileRunnerIds"]["checks"] + disabled = [] + container_running = {"value": True} + row = next(item for item in inventory if item["id"] == runner_id) + monkeypatch.setattr(controller, "_save_state", lambda _state: None) + monkeypatch.setattr( + controller, + "_job_rows", + lambda _status: ([], True), + ) + monkeypatch.setattr( + controller, + "_runner_inventory", + lambda: [dict(item) for item in inventory], + ) + monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("paused", "guard")) + monkeypatch.setattr( + controller, + "_set_disabled", + lambda actual_id, value: ( + disabled.append((actual_id, value)), + row.update(disabled=value), + ), + ) + monkeypatch.setattr( + controller, + "_container", + lambda name: { + "id": containers[name]["id"], + "running": container_running["value"], + "health": "healthy" if container_running["value"] else "none", + }, + ) + def stop(args, **_kwargs): + assert args == ["docker", "stop", profile.container_name] + container_running["value"] = False + return "" + + monkeypatch.setattr(controller, "_docker_run", stop) + + action = controller._drain_one_idle_profile(state, observation) + + assert action == "runner_admission_drained_idle_profile" + assert disabled == [(runner_id, True)] + assert entry["enabledByController"] is False + assert entry["drainOwned"] is False + assert container_running["value"] is False + + +def test_drained_runner_requires_a_full_exact_container_id(controller_module): + module = controller_module + controller = module.RunnerAdmissionController() + state = module._initial_state() + state["profiles"]["checks"]["drainOwned"] = True + + assert controller._stop_drained_profile( + state, + profile_id="checks", + runner_id=101, + expected_container_id="", + ) == "runner_admission_container_identity_invalid" + + +def test_drain_does_not_stop_when_operator_reopens_registration_after_disable( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + controller, state, observation, inventory, containers = _lease_fixture( + module, enabled=("checks",) + ) + entry = state["profiles"]["checks"] + entry.update( + idleSince=now - module.IDLE_DRAIN_SECONDS - 1, + ) + runner_id = observation["profileRunnerIds"]["checks"] + row = next(item for item in inventory if item["id"] == runner_id) + calls = 0 + patches = [] + + monkeypatch.setattr(controller, "_save_state", lambda _state: None) + monkeypatch.setattr(controller, "_job_rows", lambda _status: ([], True)) + monkeypatch.setattr( + controller, + "_runner_inventory", + lambda: [dict(item) for item in inventory], + ) + monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("clear", "")) + + def operator_reopens(actual_id, _value): + patches.append((actual_id, True)) + row.update(disabled=False) + + monkeypatch.setattr( + controller, + "_set_disabled", + operator_reopens, + ) + monkeypatch.setattr(controller, "_container", lambda name: dict(containers[name])) + + def docker(args, **_kwargs): + nonlocal calls + calls += 1 + return "" + + monkeypatch.setattr(controller, "_docker_run", docker) + + action = controller._drain_one_idle_profile(state, observation) + + assert action == "runner_admission_drain_stop_deferred" + assert calls == 0 + assert entry["drainOwned"] is True + assert entry["enabledByController"] is False + + fresh_rows, _ = controller._normalized_runner_rows(inventory) + observation["runnerRows"] = fresh_rows + assert controller._reconcile_ownership(state, observation) + assert entry["drainOwned"] is False + assert entry["operatorEnabled"] is True + assert controller._drain_one_idle_profile(state, observation) is None + assert patches == [(runner_id, True)] + + +def test_busy_controller_owned_runner_gets_a_durable_disable_but_job_is_not_stopped( + controller_module, monkeypatch +): + module = controller_module + controller, state, observation, inventory, containers = _lease_fixture( + module, enabled=("backend",), busy=("backend",) + ) + profile = module.RUNNER_PROFILES["backend"] + runner_id = observation["profileRunnerIds"]["backend"] + active = _raw_job( + module, + {"id": "active-backend", "labels": [profile.runner_label]}, + runner_id=runner_id, + active=True, + ) + saved = [] + patches = [] + stops = [] + _install_lease_io( + monkeypatch, + controller, + module, + inventory, + containers, + active_rows=[active], + saved=saved, + ) + + def disable(actual_id, disabled): + patches.append((actual_id, disabled)) + next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled + + monkeypatch.setattr(controller, "_set_disabled", disable) + monkeypatch.setattr( + controller, + "_docker_run", + lambda args, **_kwargs: stops.append(args), + ) + + action = controller._drain_one_idle_profile(state, observation) + + assert action == "runner_admission_drain_stop_deferred" + assert patches == [(runner_id, True)] + assert stops == [] + assert saved[-1]["profiles"]["backend"]["drainOwned"] is True + assert state["profiles"]["backend"]["drainOwned"] is True + assert state["profiles"]["backend"]["enabledByController"] is False + fresh_rows, fresh_by_id = controller._normalized_runner_rows(inventory) + claims, _, _ = controller._active_claims( + fresh_rows, fresh_by_id, {runner_id: [{"id": "active-backend"}]} + ) + assert claims == ["backend"] + + +def test_cycle_fences_a_busy_owned_lease_even_without_a_resource_proof( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + controller, state, observation, inventory, containers = _lease_fixture( + module, enabled=("backend",), busy=("backend",) + ) + state["terminalHistoryUpdatedAt"] = module._utc_iso(now) + observation.update(specialization="active", at=now) + saved = [] + patches = [] + active = _raw_job( + module, + { + "id": "active-backend", + "labels": [module.RUNNER_PROFILES["backend"].runner_label], + }, + runner_id=observation["profileRunnerIds"]["backend"], + active=True, + ) + _install_lease_io( + monkeypatch, + controller, + module, + inventory, + containers, + active_rows=[active], + saved=saved, + ) + monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) + monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation) + monkeypatch.setattr(controller, "_read_state", lambda: state) + monkeypatch.setattr(controller, "_terminal_history", lambda: ([], True, None)) + monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False) + monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False) + monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": "healthy"}) + monkeypatch.setattr(controller, "_write_projection", lambda _projection: None) + + def disable(actual_id, disabled): + patches.append((actual_id, disabled)) + next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled + + monkeypatch.setattr(controller, "_set_disabled", disable) + + result = controller.cycle() + + assert result["status"] == "runner_admission_drain_stop_deferred" + assert patches == [(observation["profileRunnerIds"]["backend"], True)] + assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True + assert saved[-1]["profiles"]["backend"]["drainOwned"] is True + + +def test_oldest_other_profile_eventually_replaces_sustained_backend_frontend_claims( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + older_build = { + "id": "older-build", + "labels": [module.RUNNER_PROFILES["build"].runner_label], + "queued_at": now - 20, + } + newer_backend = { + "id": "newer-backend", + "labels": ["linux-validation"], + "queued_at": now - 10, + } + controller, state, observation, inventory, containers = _lease_fixture( + module, + enabled=("backend", "frontend"), + queue=[older_build, newer_backend], + ) + for profile_id in ("backend", "frontend"): + state["profiles"][profile_id]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1 + raw_queue = [_raw_job(module, older_build), _raw_job(module, newer_backend)] + patches = [] + _install_lease_io( + monkeypatch, + controller, + module, + inventory, + containers, + queue_rows=raw_queue, + ) + + def disable(actual_id, disabled): + patches.append((actual_id, disabled)) + next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled + + monkeypatch.setattr(controller, "_set_disabled", disable) + + first = controller._drain_one_idle_profile(state, observation) + assert first == "runner_admission_drained_idle_profile" + assert patches == [(observation["profileRunnerIds"]["backend"], True)] + assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is False + + # The next poll can yield the other idle broad-label lease as the older + # heavy job remains ahead of a newer backend job. + rows, by_id = controller._normalized_runner_rows(inventory) + observation["runnerRows"] = rows + observation["claims"] = controller._active_claims(rows, by_id, {})[0] + second = controller._drain_one_idle_profile(state, observation) + assert second == "runner_admission_drained_idle_profile" + assert patches == [ + (observation["profileRunnerIds"]["backend"], True), + (observation["profileRunnerIds"]["frontend"], True), + ] + assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is False + + rows, by_id = controller._normalized_runner_rows(inventory) + claims, _, _ = controller._active_claims(rows, by_id, {}) + choice = module.select_admission( + [older_build, newer_backend], + available_profiles=controller._enabled_candidate_profiles( + { + "runnerRows": rows, + "containers": containers, + "activeJobs": {}, + "profileRunnerIds": observation["profileRunnerIds"], + }, + state, + ), + active_claims=claims, + cursor=state["cursor"], + ) + assert (choice.profile_id, choice.job_id) == ("build", "older-build") + + +def test_prepatch_drain_intent_survives_restart_and_retries_with_complete_inventory( + controller_module, monkeypatch +): + module = controller_module + controller, state, observation, inventory, containers = _lease_fixture( + module, enabled=("backend",), busy=("backend",) + ) + runner_id = observation["profileRunnerIds"]["backend"] + active = _raw_job( + module, + {"id": "active-backend", "labels": ["itworx-ci-backend"]}, + runner_id=runner_id, + active=True, + ) + saved = [] + patches = [] + _install_lease_io( + monkeypatch, + controller, + module, + inventory, + containers, + active_rows=[active], + saved=saved, + ) + + def incomplete(status): + if status == "pending": + raise module.RunnerAdmissionError("runner_admission_pending_inventory_incomplete") + return ([active] if status == "in_progress" else [], True) + + monkeypatch.setattr(controller, "_job_rows", incomplete) + assert controller._drain_one_idle_profile(state, observation) == ( + "runner_admission_pending_inventory_incomplete" + ) + assert saved[-1]["profiles"]["backend"]["drainOwned"] is True + assert patches == [] + + recovered = copy.deepcopy(saved[-1]) + assert controller._reconcile_ownership(recovered, observation) is False + assert recovered["profiles"]["backend"]["drainOwned"] is True + assert recovered["profiles"]["backend"]["operatorEnabled"] is False + + def complete(status): + if status in {"queued", "pending"}: + return [], True + return [active], True + + monkeypatch.setattr(controller, "_job_rows", complete) + + def disable(actual_id, disabled): + patches.append((actual_id, disabled)) + next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled + + monkeypatch.setattr(controller, "_set_disabled", disable) + assert controller._drain_one_idle_profile(recovered, observation) == ( + "runner_admission_drain_stop_deferred" + ) + assert patches == [(runner_id, True)] + assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True + + +def test_stale_idle_drain_intent_is_cleared_when_fresh_queue_no_longer_yields( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + backend_job = { + "id": "newer-backend", + "labels": ["linux-validation"], + "queued_at": now - 10, + } + controller, state, observation, inventory, containers = _lease_fixture( + module, + enabled=("backend",), + busy=(), + queue=[backend_job], + ) + entry = state["profiles"]["backend"] + entry.update(drainOwned=True, enabledByController=True, idleSince=None) + saved = [] + patches = [] + _install_lease_io( + monkeypatch, + controller, + module, + inventory, + containers, + queue_rows=[_raw_job(module, backend_job)], + saved=saved, + ) + monkeypatch.setattr( + controller, + "_set_disabled", + lambda actual_id, disabled: patches.append((actual_id, disabled)), + ) + recovered = copy.deepcopy(state) + + assert controller._reconcile_ownership(recovered, observation) is False + result = controller._drain_one_idle_profile(recovered, observation) + + assert result == "runner_admission_drain_deferred_queue" + assert patches == [] + assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True + assert recovered["profiles"]["backend"]["drainOwned"] is False + assert saved[-1]["profiles"]["backend"]["drainOwned"] is False + + +def test_queue_identity_is_rechecked_before_yielding_and_busy_peer_is_never_stopped( + controller_module, monkeypatch +): + module = controller_module + now = time.time() + build = { + "id": "build", + "labels": [module.RUNNER_PROFILES["build"].runner_label], + "queued_at": now - 20, + } + backend = { + "id": "backend", + "labels": ["linux-validation"], + "queued_at": now - 10, + } + controller, state, observation, inventory, containers = _lease_fixture( + module, + enabled=("backend", "frontend"), + busy=(), + queue=[build, backend], + ) + state["profiles"]["backend"]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1 + raw_build = _raw_job(module, build) + fresh_backend = {**backend, "queued_at": now - 30} + fresh_build = {**build, "queued_at": now - 20} + raw_backend = _raw_job(module, fresh_backend) + raw_build = _raw_job(module, fresh_build) + patches = [] + stops = [] + fresh_queue = [raw_backend, raw_build] + fresh_active = [] + _install_lease_io( + monkeypatch, + controller, + module, + inventory, + containers, + # Fresh inventory reverses the queue identity order; the controller + # must not act on the stale observation's earlier build. + queue_rows=fresh_queue, + active_rows=fresh_active, + ) + def disable(actual_id, disabled): + patches.append((actual_id, disabled)) + next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled + + monkeypatch.setattr(controller, "_set_disabled", disable) + + def stop(args, **_kwargs): + stops.append(args) + containers[args[2]]["running"] = False + containers[args[2]]["health"] = "none" + return "" + + monkeypatch.setattr(controller, "_docker_run", stop) + + # Fresh backend demand is older, and Backend is still serving it. A drain + # must not proceed just because a stale snapshot preferred Build. + result = controller._drain_one_idle_profile(state, observation) + + assert result == "runner_admission_drain_deferred_queue" + assert patches == [] + assert stops == [] + assert state["profiles"]["backend"]["drainOwned"] is False + + # Once the fresh queue again places Build first, the busy Frontend lease + # gets its FetchTask fenced first, but its active process stays running. + raw_build["created_at"] = module._utc_iso(now - 40) + raw_backend["created_at"] = module._utc_iso(now - 30) + fresh_queue[:] = [raw_build, raw_backend] + frontend_id = observation["profileRunnerIds"]["frontend"] + frontend_row = next(row for row in inventory if row["id"] == frontend_id) + frontend_row["busy"] = True + frontend_active = _raw_job( + module, + {"id": "active-frontend", "labels": ["itworx-ci-frontend"]}, + runner_id=frontend_id, + active=True, + ) + fresh_active.append(frontend_active) + observation["runnerRows"][module.RUNNER_PROFILES["frontend"].container_name][ + "busy" + ] = True + observation["activeJobs"][frontend_id] = [{"id": "active-frontend"}] + observation["claims"] = ["backend", "frontend"] + first = controller._drain_one_idle_profile(state, observation) + assert first == "runner_admission_drain_stop_deferred" + assert patches == [(frontend_id, True)] + assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is True + + rows, by_id = controller._normalized_runner_rows(inventory) + observation["runnerRows"] = rows + observation["claims"] = controller._active_claims( + rows, by_id, observation["activeJobs"] + )[0] + second = controller._drain_one_idle_profile(state, observation) + assert second == "runner_admission_drained_idle_profile" + assert patches == [ + (frontend_id, True), + (observation["profileRunnerIds"]["backend"], True), + ] + assert stops == [["docker", "stop", module.RUNNER_PROFILES["backend"].container_name]] + assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is True + + +def test_bootstrap_receipt_seeds_only_stopped_disabled_fixed_baseline( + controller_module, monkeypatch, tmp_path +): + module = controller_module + controller = module.RunnerAdmissionController() + state = module._initial_state() + monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json") + saved = [] + consumed = [] + + rows = {} + containers = {} + registrations = [] + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + labels = [profile.runner_label, *profile.legacy_labels] + rows[profile.container_name] = { + "id": 100 + index, + "displayName": profile.display_name, + "disabled": True, + "busy": False, + "labels": set(labels), + } + containers[profile.container_name] = { + "id": f"{index:064x}", + "running": False, + "health": "none", + } + registrations.append( + { + "profileId": profile.id, + "displayName": profile.display_name, + "containerName": profile.container_name, + "id": 100 + index, + "labels": labels, + } + ) + receipt = { + "status": "pending", + "verifiedRunners": [ + profile.container_name for profile in module.RUNNER_PROFILES.values() + ], + "registrations": registrations, + } + receipt_sha = "d" * 64 + fake_specializer = SimpleNamespace( + read_admission_bootstrap_receipt=lambda: (receipt, receipt_sha), + consume_admission_bootstrap_receipt=lambda digest: ( + consumed.append((digest, state["revision"])) or True + ), + ) + monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer) + monkeypatch.setattr(controller, "_save_state", lambda value: ( + saved.append(value["revision"]), + value.update(revision=value["revision"] + 1), + )) + + seeded = controller._seed_or_consume_bootstrap_receipt( + state, + { + "runnerRows": rows, + "containers": containers, + "resourceProof": {"ready": True, "runningRunnerContainers": 0}, + "claims": [], + }, + ) + + assert seeded is True + assert saved == [0] + assert consumed == [(receipt_sha, 1)] + assert state["bootstrapReceiptSha256"] == receipt_sha + assert all(entry["observed"] for entry in state["profiles"].values()) + assert all(not entry["operatorDisabled"] for entry in state["profiles"].values()) + + +def test_bootstrap_receipt_rejects_any_running_initial_runner( + controller_module, monkeypatch, tmp_path +): + module = controller_module + controller = module.RunnerAdmissionController() + state = module._initial_state() + monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json") + rows = {} + containers = {} + registrations = [] + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + labels = [profile.runner_label, *profile.legacy_labels] + rows[profile.container_name] = { + "id": 100 + index, + "displayName": profile.display_name, + "disabled": True, + "busy": False, + "labels": set(labels), + } + containers[profile.container_name] = { + "id": f"{index:064x}", + "running": index == 1, + "health": "healthy" if index == 1 else "none", + } + registrations.append( + { + "profileId": profile.id, + "displayName": profile.display_name, + "containerName": profile.container_name, + "id": 100 + index, + "labels": labels, + } + ) + fake_specializer = SimpleNamespace( + read_admission_bootstrap_receipt=lambda: ( + { + "status": "pending", + "verifiedRunners": [ + profile.container_name for profile in module.RUNNER_PROFILES.values() + ], + "registrations": registrations, + }, + "e" * 64, + ), + consume_admission_bootstrap_receipt=lambda _digest: pytest.fail( + "receipt consumed before live baseline verification" + ), + ) + monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer) + saved = [] + monkeypatch.setattr(controller, "_save_state", lambda value: saved.append(value)) + + with pytest.raises(module.RunnerAdmissionError): + controller._seed_or_consume_bootstrap_receipt( + state, + { + "runnerRows": rows, + "containers": containers, + "resourceProof": {"ready": True, "runningRunnerContainers": 1}, + "claims": [], + }, + ) + assert saved == [] + + +def test_unattested_more_than_two_running_runner_outers_blocks_claims( + controller_module, +): + module = controller_module + controller = module.RunnerAdmissionController( + resource_attestor=lambda **_kwargs: { + "ready": True, + "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, + "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, + "activeClaims": 0, + "heavyClaims": 0, + "runningRunnerContainers": 3, + "cgroups": {}, + "digest": "c" * 64, + } + ) + + with pytest.raises( + module.RunnerAdmissionError, + match="runner_admission_running_outer_budget_exceeded", + ): + controller._resource_attestation( + [], + {name: None for name in module.ALL_LEGACY_RUNNER_NAMES}, + ) + + +def test_operator_open_then_disable_is_latched_from_bootstrap_baseline( + controller_module, +): + module = controller_module + controller = module.RunnerAdmissionController() + state = module._initial_state() + entry = state["profiles"]["checks"] + entry.update(observed=True, operatorDisabled=False, operatorEnabled=False) + profile = module.RUNNER_PROFILES["checks"] + rows = {} + for index, item in enumerate(module.RUNNER_PROFILES.values(), start=1): + rows[item.container_name] = { + "id": 200 + index, + "disabled": True, + "busy": False, + } + state["registrationIds"][item.id] = 200 + index + rows[profile.container_name]["disabled"] = False + observation = { + "runnerRows": rows + } + + assert controller._reconcile_ownership(state, observation) + assert entry["operatorEnabled"] is True + assert entry["enabledByController"] is False + rows[profile.container_name]["disabled"] = True + + assert controller._reconcile_ownership(state, observation) + assert entry["operatorDisabled"] is True + assert entry["operatorEnabled"] is False + + +def test_registration_identity_change_fails_closed(controller_module): + module = controller_module + controller = module.RunnerAdmissionController() + state = module._initial_state() + rows = {} + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): + expected_id = 300 + index + state["registrationIds"][profile.id] = expected_id + rows[profile.container_name] = { + "id": expected_id, + "disabled": True, + "busy": False, + } + rows[module.RUNNER_PROFILES["checks"].container_name]["id"] += 1 + + with pytest.raises( + module.RunnerAdmissionError, + match="runner_admission_registration_identity_changed", + ): + controller._reconcile_ownership(state, {"runnerRows": rows}) + + +def test_pending_runner_start_does_not_enable_after_guard_pauses( + controller_module, monkeypatch +): + module = controller_module + controller = module.RunnerAdmissionController(clock=lambda: time.time()) + state = module._initial_state() + profile = module.RUNNER_PROFILES["checks"] + runner_id = 101 + job_id = "job-42" + state["pending"] = {"profileId": "checks", "runnerId": runner_id, "jobId": job_id} + initial = { + "runnerRows": {profile.container_name: {"id": runner_id, "disabled": True, "busy": False}}, + "queue": [{"id": job_id, "labels": [profile.runner_label], "queued_at": time.time()}], + "claims": ["checks"], + "containers": {profile.container_name: {"running": False, "health": "none"}}, + "resourceProof": {"ready": True}, + } + after_start = copy.deepcopy(initial) + after_start["containers"][profile.container_name] = {"running": True, "health": "healthy"} + observations = iter([after_start, after_start]) + gates = iter([(None, {"host": {"swapSampleAt": time.time()}}), + ("runner_admission_host_guard_paused", None)]) + starts = [] + enables = [] + monkeypatch.setattr(controller, "_save_state", lambda _state: None) + monkeypatch.setattr(controller, "_observe", lambda **_kwargs: next(observations)) + monkeypatch.setattr(controller, "_fresh_mutation_gate", lambda *_args, **_kwargs: next(gates)) + monkeypatch.setattr(controller, "_docker_run", lambda args, **_kwargs: starts.append(args)) + monkeypatch.setattr(controller, "_set_disabled", lambda *args: enables.append(args)) + + result = controller._continue_pending(state, initial, reserved_profile_id="checks") + + assert result == "runner_admission_host_guard_paused" + assert starts == [["docker", "start", profile.container_name]] + assert enables == [] diff --git a/backend/tests/test_runner_admission_policy.py b/backend/tests/test_runner_admission_policy.py new file mode 100644 index 0000000..0f22dd9 --- /dev/null +++ b/backend/tests/test_runner_admission_policy.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +POLICY_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_admission_policy.py" +PROFILES_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py" + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def policy(): + profiles = _load("appops_runner_profiles_for_admission_test", PROFILES_PATH) + sys.modules["runner_profiles"] = profiles + return _load("appops_runner_admission_policy_test", POLICY_PATH) + + +def _job(job_id: str, labels: list[str], queued_at: float): + return {"id": job_id, "labels": labels, "queued_at": queued_at} + + +def test_oldest_compatible_job_wins_and_profile_choice_rotates(policy): + jobs = [ + _job("newer-security", ["itworx-ci-security"], 200.0), + _job("older-linux", ["self-hosted", "linux-validation"], 100.0), + ] + + first = policy.select_admission( + jobs, + available_profiles=["backend", "frontend", "security"], + active_claims=[], + cursor=0, + ) + second = policy.select_admission( + jobs, + available_profiles=["backend", "frontend", "security"], + active_claims=[], + cursor=first.next_cursor, + ) + + assert (first.job_id, first.profile_id) == ("older-linux", "backend") + assert (second.job_id, second.profile_id) == ("older-linux", "frontend") + + +def test_enabled_legacy_slots_count_as_claims_and_can_consume_all_capacity(policy): + decision = policy.select_admission( + [_job("checks", ["itworx-ci-checks"], 1.0)], + available_profiles=["checks"], + # Two old broad Tower registrations are charged as two backend-envelope slots. + active_claims=["backend", "backend"], + ) + + assert decision.profile_id is None + assert decision.reason == "runner_admission_claim_limit" + + +def test_duplicate_legacy_claims_are_charged_against_dind_and_family_budgets(policy): + dind_limited = policy.validate_claims(["frontend", "backend"]) + family_excess = policy.validate_claims(["build", "backend"]) + + assert dind_limited.ok + assert family_excess.reason == "runner_admission_dind_budget_exceeded" + + +def test_two_heavy_profiles_cannot_be_admitted_together(policy): + result = policy.validate_claims(["browser", "build"]) + + assert not result.ok + assert result.reason == "runner_admission_heavy_claim_limit" + + +def test_unmapped_enabled_registration_fails_closed(policy): + result = policy.validate_claims(["unknown-legacy-runner"]) + + assert not result.ok + assert result.reason == "runner_admission_profile_unmapped" + + +def test_candidate_is_reserved_before_it_can_be_selected(policy): + result = policy.select_admission( + [_job("build", ["itworx-ci-build"], 1.0)], + available_profiles=["build"], + active_claims=["browser"], + ) + + assert result.profile_id is None + assert result.reason == "runner_admission_heavy_claim_limit" + + +def test_unknown_required_job_label_does_not_fall_back_to_a_broad_profile(policy): + result = policy.select_admission( + [_job("mystery", ["linux-validation", "unraid-deploy"], 1.0)], + available_profiles=["backend", "frontend"], + active_claims=[], + ) + + assert result.profile_id is None + assert result.reason == "runner_admission_no_compatible_queue" + + +def test_legacy_docker_alias_keeps_heavy_reservation_and_budget_gates(policy): + selected = policy.select_admission( + [_job("docker", ["docker-validation"], 1.0)], + available_profiles=["build"], active_claims=[], + ) + assert selected.profile_id == "build" + blocked = policy.select_admission( + [_job("docker", ["docker-validation"], 1.0)], + available_profiles=["build"], active_claims=["browser"], + ) + assert blocked.profile_id is None + assert blocked.reason == "runner_admission_heavy_claim_limit" + + +def test_malformed_queue_row_fails_closed(policy): + with pytest.raises(policy.RunnerAdmissionPolicyError): + policy.select_admission( + [{"id": "bad", "labels": ["itworx-ci-checks"], "queued_at": "now"}], + available_profiles=["checks"], + active_claims=[], + ) + + +def test_lease_yields_to_the_oldest_eligible_other_profile_without_stealing_same_profile_work( + policy, +): + queue = [ + _job("older-build", ["itworx-ci-build"], 100.0), + _job("newer-backend", ["linux-validation"], 200.0), + ] + decision = policy.select_lease_yield( + queue, + leased_profile="backend", + available_profiles=["checks", "security", "browser", "build"], + active_claims=["backend", "frontend"], + cursor=0, + ) + + assert (decision.profile_id, decision.job_id) == ("build", "older-build") + + same_profile_is_oldest = policy.select_lease_yield( + [ + _job("older-backend", ["linux-validation"], 100.0), + _job("newer-build", ["itworx-ci-build"], 200.0), + ], + leased_profile="backend", + available_profiles=["checks", "security", "browser", "build"], + active_claims=["backend", "frontend"], + cursor=0, + ) + + assert same_profile_is_oldest.profile_id is None + assert same_profile_is_oldest.reason == "runner_admission_lease_retained" + + +def test_lease_does_not_yield_when_current_claims_can_admit_queued_work(policy): + decision = policy.select_lease_yield( + [_job("checks", ["itworx-ci-checks"], 100.0)], + leased_profile="backend", + available_profiles=["checks"], + active_claims=["backend"], + cursor=0, + ) + + assert decision.profile_id is None + assert decision.reason == "runner_admission_lease_not_needed" diff --git a/backend/tests/test_runner_capacity_controller.py b/backend/tests/test_runner_capacity_controller.py index b88eb34..fbc412b 100644 --- a/backend/tests/test_runner_capacity_controller.py +++ b/backend/tests/test_runner_capacity_controller.py @@ -1,7 +1,9 @@ from __future__ import annotations import importlib.util +import io import sys +import types from pathlib import Path import pytest @@ -14,6 +16,9 @@ CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py" @pytest.fixture def capacity_module(monkeypatch): + if sys.platform == "win32": + # The daemon lock is not exercised by these in-process cycle tests. + monkeypatch.setitem(sys.modules, "fcntl", types.ModuleType("fcntl")) monkeypatch.syspath_prepend(str(CONTROLLER_ROOT)) spec = importlib.util.spec_from_file_location( "appops_runner_capacity_controller_test", CONTROLLER_PATH @@ -25,6 +30,137 @@ def capacity_module(monkeypatch): return module +def _wire_read_only_host(service, module, monkeypatch, samples): + counters = iter(samples) + monkeypatch.setattr(service, "_strict_swap_pages", lambda: next(counters)) + monkeypatch.setattr(module.os, "cpu_count", lambda: 20) + monkeypatch.setattr(module.os, "getloadavg", lambda: (4.0, 3.0, 2.0), raising=False) + monkeypatch.setattr(module.os, "sysconf", lambda _name: 4096, raising=False) + monkeypatch.setattr(service, "_mem_available_bytes", lambda: 16 * 1024**3) + monkeypatch.setattr(service, "_cache_free_bytes", lambda: 32 * 1024**3) + + def forbidden(*_args, **_kwargs): + pytest.fail("read-only host sampling touched burst inventory or a mutation") + + for name in ("_burst_container", "_runner_registration", "_gitea_request", + "_run", "_read_state", "_write_state", "_record"): + monkeypatch.setattr(service, name, forbidden) + + +def test_host_snapshot_measures_swap_without_optional_burst(capacity_module, monkeypatch): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, [(50, 70)]) + prior = {"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0} + preserved = dict(prior) + monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("baseline already available")) + + result = service.host_snapshot(prior, now=100.0) + + assert result["status"] == "observed" + assert result["host"]["loadRatio"] == 0.2 + assert result["host"]["swapInKbps"] == 200.0 + assert result["host"]["swapOutKbps"] == 280.0 + assert result["host"]["swapCritical"] is False + assert prior == preserved + + +@pytest.mark.parametrize("prior", [None, {"swapSampleAt": 0}, { + "swapSampleAt": 1.0, "swapInPages": 5, "swapOutPages": 5, +}, {"swapSampleAt": float("nan"), "swapInPages": 5, "swapOutPages": 5}]) +def test_host_snapshot_takes_two_real_samples_for_missing_or_stale_baseline( + capacity_module, monkeypatch, prior +): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (266, 532)]) + elapsed = iter([4.0, 5.0]) + sleeps = [] + monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed)) + monkeypatch.setattr(capacity_module.time, "sleep", sleeps.append) + + result = service.host_snapshot(prior, now=100.0) + + assert result["status"] == "observed" + assert result["host"]["swapInKbps"] == 1024.0 + assert result["host"]["swapOutKbps"] == 2048.0 + assert result["host"]["swapSampleAt"] == 101.0 + assert sleeps == [1.0] + + +def test_host_snapshot_reports_unavailable_if_swap_counter_resets(capacity_module, monkeypatch): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (9, 20)]) + elapsed = iter([4.0, 5.0]) + monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed)) + monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: None) + + result = service.host_snapshot(now=100.0) + + assert result["status"] == "unavailable" + assert result["reason"] == "runner_capacity_swap_baseline_unavailable" + assert result["host"] is None + + +def test_host_snapshot_rejects_reset_against_recent_protected_baseline(capacity_module, monkeypatch): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, [(9, 20), (10, 21)]) + monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("recent counter reset must fail closed")) + + result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 10, "swapOutPages": 20}, now=100.0) + + assert result["status"] == "unavailable" + assert result["reason"] == "runner_capacity_swap_baseline_unavailable" + assert result["host"] is None + + +def test_host_snapshot_handles_absent_load_tuple(capacity_module, monkeypatch): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)]) + monkeypatch.setattr(capacity_module.os, "getloadavg", lambda: ()) + + result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0) + + assert result["status"] == "unavailable" + assert result["host"] is None + + +@pytest.mark.parametrize("raw", ["", "pswpin 1\n", "pswpin -1\npswpout 2\n", + "pswpin 1\npswpin 2\npswpout 3\n", "pswpin invalid\npswpout 2\n", "x" * 65537]) +def test_strict_swap_reader_rejects_missing_or_invalid_counters(capacity_module, monkeypatch, raw): + monkeypatch.setattr(capacity_module, "Path", lambda _path: types.SimpleNamespace( + open=lambda **_kwargs: io.StringIO(raw) + )) + with pytest.raises(capacity_module.RunnerCapacityError, match="swap_telemetry_unavailable"): + capacity_module.RunnerCapacityController._strict_swap_pages() + + +def test_host_snapshot_does_not_turn_missing_swap_into_zero(capacity_module, monkeypatch): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, []) + + def unavailable(): + raise capacity_module.RunnerCapacityError("runner_capacity_swap_telemetry_unavailable") + + monkeypatch.setattr(service, "_strict_swap_pages", unavailable) + result = service.host_snapshot(now=100.0) + assert result["status"] == "unavailable" + assert result["host"] is None + + +@pytest.mark.parametrize("metric", ["cpu_count", "getloadavg", "sysconf", "memory", "cache"]) +def test_host_snapshot_rejects_invalid_global_metrics(capacity_module, monkeypatch, metric): + service = capacity_module.RunnerCapacityController() + _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)]) + if metric == "getloadavg": + monkeypatch.setattr(capacity_module.os, metric, lambda: (float("nan"), 0, 0)) + elif metric in {"cpu_count", "sysconf"}: + monkeypatch.setattr(capacity_module.os, metric, lambda *_args: 0) + else: + monkeypatch.setattr(service, "_mem_available_bytes" if metric == "memory" else "_cache_free_bytes", lambda: True) + result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0) + assert result["status"] == "unavailable" + assert result["host"] is None + + def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True): monkeypatch.setattr( service, @@ -50,6 +186,7 @@ def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=Tr monkeypatch.setattr(service, "_read_state", lambda: {}) monkeypatch.setattr(service, "_write_state", lambda _state: None) monkeypatch.setattr(service, "_record", lambda _event: None) + monkeypatch.setattr(service, "_host_guard_state", lambda: ("clear", "")) def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch): @@ -218,3 +355,141 @@ def test_cycle_never_stops_when_gitea_still_has_active_assignment( assert result["decision"]["result"] == "stop_deferred_not_proven_idle" assert commands == [] + + +def test_paused_host_guard_blocks_burst_start_before_docker_or_gitea_write( + capacity_module, monkeypatch +): + service = capacity_module.RunnerCapacityController() + _wire_common(service, monkeypatch, running=False, disabled=True) + monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60}) + monkeypatch.setattr( + service, + "_host", + lambda _previous, now: { + "cpuCount": 20, + "loadOne": 2.0, + "loadRatio": 0.1, + "availableMemoryBytes": 20 * 1024**3, + "cacheFreeBytes": 100 * 1024**3, + "swapInKbps": 0.0, + "swapOutKbps": 0.0, + "swapPressureSamples": 0, + "swapCritical": False, + "swapSampleAt": now, + "swapInPages": 0, + "swapOutPages": 0, + }, + ) + monkeypatch.setattr(service, "_host_guard_state", lambda: ("paused", "guard")) + commands = [] + disabled = [] + monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "") + monkeypatch.setattr( + service, + "_set_disabled", + lambda runner_id, value: disabled.append((runner_id, value)), + ) + + result = service.cycle() + + assert result["decision"]["action"] == "start" + assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused" + assert commands == [] + assert disabled == [] + + +def test_guard_is_rechecked_after_docker_start_before_gitea_enable( + capacity_module, monkeypatch +): + service = capacity_module.RunnerCapacityController() + _wire_common(service, monkeypatch, running=False, disabled=True) + monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60}) + monkeypatch.setattr( + service, + "_host", + lambda _previous, now: { + "cpuCount": 20, + "loadOne": 2.0, + "loadRatio": 0.1, + "availableMemoryBytes": 20 * 1024**3, + "cacheFreeBytes": 100 * 1024**3, + "swapInKbps": 0.0, + "swapOutKbps": 0.0, + "swapPressureSamples": 0, + "swapCritical": False, + "swapSampleAt": now, + "swapInPages": 0, + "swapOutPages": 0, + }, + ) + guard_states = iter([("clear", ""), ("paused", "guard")]) + monkeypatch.setattr(service, "_host_guard_state", lambda: next(guard_states)) + commands = [] + disabled = [] + monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "") + monkeypatch.setattr( + service, + "_set_disabled", + lambda runner_id, value: disabled.append((runner_id, value)), + ) + + result = service.cycle() + + assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused" + assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]] + assert disabled == [] + + +def test_fresh_pressure_is_rechecked_before_burst_start( + capacity_module, monkeypatch +): + service = capacity_module.RunnerCapacityController() + _wire_common(service, monkeypatch, running=False, disabled=True) + monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60}) + samples = iter( + [ + { + "loadRatio": 0.1, + "availableMemoryBytes": 20 * 1024**3, + "cacheFreeBytes": 100 * 1024**3, + "swapCritical": False, + }, + { + "loadRatio": 0.95, + "availableMemoryBytes": 20 * 1024**3, + "cacheFreeBytes": 100 * 1024**3, + "swapCritical": False, + }, + ] + ) + + def sample(_previous, now): + values = next(samples) + return { + "cpuCount": 20, + "loadOne": values["loadRatio"] * 20, + **values, + "swapInKbps": 0.0, + "swapOutKbps": 0.0, + "swapPressureSamples": 0, + "swapSampleAt": now, + "swapInPages": 0, + "swapOutPages": 0, + } + + monkeypatch.setattr(service, "_host", sample) + commands = [] + monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "") + disabled = [] + monkeypatch.setattr( + service, + "_set_disabled", + lambda runner_id, value: disabled.append((runner_id, value)), + ) + + result = service.cycle() + + assert result["decision"]["result"] == "failed:runner_capacity_pressure_critical" + assert commands == [] + assert disabled == [] diff --git a/backend/tests/test_runner_controller_artifact.py b/backend/tests/test_runner_controller_artifact.py new file mode 100644 index 0000000..bcc43cb --- /dev/null +++ b/backend/tests/test_runner_controller_artifact.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import importlib.util +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def artifact(tmp_path, monkeypatch): + renderer = load('runner_artifact_renderer', ROOT / 'scripts/prepare_controller_artifact.py') + controller_dir = ROOT / 'ops/gitea-deploy-broker' + monkeypatch.syspath_prepend(str(controller_dir)) + controller = load('runner_artifact_controller', controller_dir / 'status_server.py') + stage = tmp_path / 'controller' + stage.mkdir() + for name in renderer._ARTIFACT_FILES: + source = ROOT / 'scripts/run_repopulse_adoption.py' if name == 'run_repopulse_adoption.py' else controller_dir / name + shutil.copyfile(source, stage / name) + result = subprocess.run([ + sys.executable, str(ROOT / 'scripts/prepare_controller_artifact.py'), + '--artifact-dir', str(stage), '--source-repository', 'jens/unraid-appops-gateway', + '--source-revision', 'a' * 40, '--source-tree', 'b' * 40, + '--controller-version', '2.1.13', '--built-at', '2026-10-01T12:00:00Z', + ], capture_output=True, text=True, check=True) + assert 'artifactSha256' in result.stdout + monkeypatch.setattr(controller, 'ROOT', stage) + monkeypatch.setattr(controller, 'CONTROLLER_RELEASE_PATH', stage / 'controller-release.json') + return controller, stage + + +def test_renderer_and_runtime_accept_the_same_complete_specialized_artifact(artifact): + controller, _stage = artifact + release = controller._controller_release() + assert release['controllerVersion'] == '2.1.13' + assert release['sourceRevision'] == 'a' * 40 + + +@pytest.mark.parametrize('damage', ['missing_binding', 'missing_hub_store', 'changed_admission', 'symlink_envelope']) +def test_incomplete_or_changed_specialized_artifact_cannot_be_promoted(artifact, damage): + controller, stage = artifact + if damage == 'missing_binding': + (stage / 'runner_statistics_binding.py').unlink() + elif damage == 'missing_hub_store': + (stage / 'mcp_hub_release_store.py').unlink() + elif damage == 'changed_admission': + with (stage / 'runner_admission_policy.py').open('ab') as stream: + stream.write(b'\n# changed after manifest rendering\n') + else: + target = stage / 'runner_resource_envelope.py' + original = stage / 'moved_envelope.py' + target.rename(original) + try: + target.symlink_to(original) + except OSError: + pytest.skip('Host cannot create symlink fixture') + assert controller._controller_release() is None diff --git a/backend/tests/test_runner_fleet.py b/backend/tests/test_runner_fleet.py index 5bca2e9..1e1251c 100644 --- a/backend/tests/test_runner_fleet.py +++ b/backend/tests/test_runner_fleet.py @@ -1,6 +1,9 @@ from __future__ import annotations import importlib.util +import sys +import types +from datetime import UTC, datetime from pathlib import Path import pytest @@ -11,14 +14,71 @@ FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py" @pytest.fixture -def fleet_module(): +def fleet_module(monkeypatch): + if sys.platform == "win32" and "fcntl" not in sys.modules: + fcntl = types.ModuleType("fcntl") + fcntl.LOCK_EX = 2 + fcntl.LOCK_UN = 8 + fcntl.LOCK_NB = 4 + fcntl.flock = lambda *_args, **_kwargs: None + monkeypatch.setitem(sys.modules, "fcntl", fcntl) spec = importlib.util.spec_from_file_location("appops_runner_fleet_test", FLEET_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + monkeypatch.setitem( + sys.modules, + "runner_fleet_specialize", + types.SimpleNamespace(specialization_state=lambda: "legacy"), + ) + monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", "")) + monkeypatch.setattr( + module.RunnerCapacityController, + "snapshot", + lambda _self: _healthy_capacity_snapshot(), + ) return module +def _healthy_capacity_snapshot(): + return { + "status": "ok", + "decision": {"pressure": "headroom"}, + "host": { + "swapCritical": False, + "loadRatio": 0.1, + "availableMemoryBytes": 12 * 1024**3, + "cacheFreeBytes": 100 * 1024**3, + "swapSampleAt": datetime.now(UTC).timestamp(), + }, + } + + +def test_dind_uses_private_unix_socket_only_for_trusted_active_specialization( + fleet_module, monkeypatch +): + service = fleet_module.RunnerFleetService() + commands = [] + monkeypatch.setattr( + service, + "_run", + lambda args, **_kwargs: commands.append(args) or "", + ) + specialization = sys.modules["runner_fleet_specialize"] + + service._dind(["info"]) + assert "DOCKER_HOST=tcp://127.0.0.1:2375" in commands[-1] + + monkeypatch.setattr(specialization, "specialization_state", lambda: "active") + service._dind(["info"]) + assert f"DOCKER_HOST={fleet_module.DIND_DOCKER_HOST}" in commands[-1] + + for state in ("legacy", "blocked", "in_progress"): + monkeypatch.setattr(specialization, "specialization_state", lambda value=state: value) + service._dind(["info"]) + assert "DOCKER_HOST=tcp://127.0.0.1:2375" in commands[-1] + + def test_cleanup_candidates_exclude_unknown_attached_and_labeled_networks( fleet_module, monkeypatch ): @@ -80,6 +140,7 @@ def test_automatic_cycle_only_deletes_exact_preview_candidates(fleet_module, mon } snapshot = { "status": "healthy", + "capacity": _healthy_capacity_snapshot(), "networks": {"cleanupCandidates": [candidate]}, "storage": {"status": "healthy"}, "topology": { @@ -143,6 +204,7 @@ def test_automatic_cycle_starts_only_an_exited_known_runner(fleet_module, monkey service = fleet_module.RunnerFleetService() snapshot = { "status": "healthy", + "capacity": _healthy_capacity_snapshot(), "networks": {"cleanupCandidates": []}, "storage": {"status": "healthy"}, "topology": { @@ -178,3 +240,21 @@ def test_automatic_cycle_starts_only_an_exited_known_runner(fleet_module, monkey assert commands == [["docker", "start", "itworx-validation-runner-2"]] assert {event["action"] for event in events} == {"start", "restart.deferred"} + + +@pytest.mark.parametrize("specialization", ["active", "blocked"]) +def test_legacy_automatic_cycle_never_observes_or_mutates_after_specialization( + fleet_module, monkeypatch, specialization +): + service = fleet_module.RunnerFleetService() + monkeypatch.setitem( + sys.modules, + "runner_fleet_specialize", + types.SimpleNamespace(specialization_state=lambda: specialization), + ) + monkeypatch.setattr(service, "snapshot", lambda: pytest.fail("legacy snapshot ran")) + monkeypatch.setattr(service, "_run", lambda *_args, **_kwargs: pytest.fail("legacy mutation ran")) + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == "runner_fleet_specialization_owns_recovery" diff --git a/backend/tests/test_runner_fleet_scheduler_recovery.py b/backend/tests/test_runner_fleet_scheduler_recovery.py index aff8bf5..36dd021 100644 --- a/backend/tests/test_runner_fleet_scheduler_recovery.py +++ b/backend/tests/test_runner_fleet_scheduler_recovery.py @@ -1,6 +1,8 @@ from __future__ import annotations import importlib.util +import sys +import types from datetime import UTC, datetime from pathlib import Path @@ -12,41 +14,93 @@ FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py" @pytest.fixture -def fleet_module(): +def fleet_module(monkeypatch): + if sys.platform == "win32" and "fcntl" not in sys.modules: + fcntl = types.ModuleType("fcntl") + fcntl.LOCK_EX = 2 + fcntl.LOCK_UN = 8 + fcntl.LOCK_NB = 4 + fcntl.flock = lambda *_args, **_kwargs: None + monkeypatch.setitem(sys.modules, "fcntl", fcntl) spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + module._host_guard_state_unpatched = module._host_guard_state + monkeypatch.setitem( + sys.modules, + "runner_fleet_specialize", + types.SimpleNamespace(specialization_state=lambda: "legacy"), + ) + monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", "")) + monkeypatch.setattr( + module.RunnerCapacityController, + "snapshot", + lambda _self: _healthy_capacity_snapshot(), + ) return module +def _healthy_capacity_snapshot(): + return { + "status": "ok", + "decision": {"pressure": "headroom"}, + "host": { + "swapCritical": False, + "loadRatio": 0.1, + "availableMemoryBytes": 12 * 1024**3, + "cacheFreeBytes": 100 * 1024**3, + "swapSampleAt": datetime.now(UTC).timestamp(), + }, + } + + def _healthy_snapshot(): + runners = [ + { + "name": name, + "running": True, + "health": "healthy", + "restarts": 0, + "startedAt": "2026-09-16T18:00:00Z", + "oomKilled": False, + "restartPolicy": "unless-stopped", + } + for name in ( + "itworx-security-runner", + "itworx-validation-runner-2", + ) + ] return { "status": "healthy", - "networks": {"cleanupCandidates": []}, - "storage": {"status": "healthy"}, + "generatedAt": "2026-10-01T10:00:00Z", + "capacity": _healthy_capacity_snapshot(), + "networks": {"count": 0, "capacity": None, "cleanupCandidates": []}, + "storage": { + "status": "healthy", + "freeBytes": 30 * 1024**3, + "totalBytes": 40 * 1024**3, + }, + "scheduler": { + "status": "healthy", + "candidate": None, + "stallThresholdSeconds": 600, + }, "topology": { "dind": { "name": "gitea-runner-dind", "running": True, "health": "healthy", + "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", + "oomKilled": False, "restartPolicy": "unless-stopped", }, - "runners": [ - { - "name": "itworx-validation-runner-2", - "running": True, - "health": "healthy", - "startedAt": "2026-09-16T18:00:00Z", - }, - { - "name": "itworx-security-runner", - "running": True, - "health": "healthy", - "startedAt": "2026-09-16T18:00:00Z", - }, - ] + "runners": runners, + "generalSlots": 2, + "runnerCount": len(runners), + "specializedRunners": [], + "burstRunner": None, }, } @@ -220,19 +274,20 @@ def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting( assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"} -def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind( +def test_automatic_cycle_recovers_critical_exited_dind_with_safe_host_capacity( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" - snapshot["topology"]["dind"] = { - "name": "gitea-runner-dind", - "running": False, - "health": "none", - "startedAt": "", - "restartPolicy": "no", - } + snapshot["topology"]["dind"].update( + { + "running": False, + "health": "none", + "startedAt": "", + "restartPolicy": "no", + } + ) commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) @@ -250,14 +305,45 @@ def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind( (["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30), (["docker", "start", "gitea-runner-dind"], 30), ] - assert result["dindRecovery"]["status"] == "requested" - assert result["schedulerRecovery"]["status"] == "deferred_for_dind_recovery" - assert any( - event.get("action") == "start" - and event.get("target") == "gitea-runner-dind" - and event.get("reason") == "known_ci_dind_exited" - for event in events + assert [event["action"] for event in events] == [ + "restart-policy.update", + "start", + ] + assert result["status"] == "ok" + assert result["dindRecovery"] == { + "status": "requested", + "restartPolicy": "requested", + "target": "gitea-runner-dind", + } + + +def test_automatic_cycle_repairs_fixed_dind_policy_when_healthy_and_guard_clear( + fleet_module, monkeypatch +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["topology"]["dind"]["restartPolicy"] = "no" + commands: list[tuple[list[str], int]] = [] + events: list[dict[str, object]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr( + service, + "_run", + lambda args, timeout=20: commands.append((args, timeout)) or "", ) + monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) + + result = service.automatic_cycle() + + assert commands == [ + (["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30) + ] + assert result["dindRecovery"] == { + "status": "not_needed", + "restartPolicy": "requested", + } + assert len(events) == 1 + assert events[0]["action"] == "restart-policy.update" def test_automatic_cycle_never_restarts_running_unhealthy_dind( @@ -265,7 +351,7 @@ def test_automatic_cycle_never_restarts_running_unhealthy_dind( ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() - snapshot["status"] = "critical" + snapshot["status"] = "healthy" snapshot["topology"]["dind"] = { "name": "gitea-runner-dind", "running": True, @@ -293,6 +379,385 @@ def test_automatic_cycle_never_restarts_running_unhealthy_dind( ) +@pytest.mark.parametrize( + ("guard_state", "guard_reason"), + [ + (("paused", "host_resource_guard_paused"), "host_resource_guard_paused"), + (("unavailable", "host_guard_state_untrusted"), "host_guard_state_untrusted"), + ], +) +def test_automatic_cycle_never_mutates_when_host_guard_is_paused_or_untrusted( + fleet_module, monkeypatch, guard_state, guard_reason +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["topology"]["dind"].update( + {"running": False, "health": "none", "restartPolicy": "no"} + ) + commands: list[list[str]] = [] + events: list[dict[str, object]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") + monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "") + monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: guard_state) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == guard_reason + assert commands == [] + assert events == [] + + +@pytest.mark.parametrize( + "action", + [ + "restart-policy-update", + "dind-start", + "scheduler-restart", + "network-remove", + "cache-prune", + "runner-start", + ], +) +def test_guard_pause_appearing_after_cycle_entry_blocks_each_mutation( + fleet_module, monkeypatch, action +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + dind = snapshot["topology"]["dind"] + monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) + if action == "restart-policy-update": + dind["restartPolicy"] = "no" + elif action == "dind-start": + dind.update({"running": False, "health": "none"}) + elif action == "scheduler-restart": + snapshot["topology"]["runners"] = [ + { + "name": "itworx-validation-runner-2", + "running": True, + "health": "healthy", + } + ] + monkeypatch.setattr( + service, + "_scheduler_recovery_candidate", + lambda: { + "target": "itworx-validation-runner-2", + "jobId": 22, + "ageSeconds": 900, + "reason": "stale_scheduler_assignment", + }, + ) + monkeypatch.setattr( + service, + "_container", + lambda name: { + "name": name, + "running": True, + "health": "healthy", + }, + ) + elif action == "network-remove": + snapshot["networks"]["cleanupCandidates"] = [ + { + "id": "a" * 64, + "name": "GITEA-ACTIONS-TASK-22-WORKFLOW-ci-JOB-test-network", + "ageSeconds": 3600, + "reason": "empty_gitea_actions_task_network_in_dedicated_dind", + } + ] + elif action == "cache-prune": + snapshot["storage"]["status"] = "critical" + elif action == "runner-start": + snapshot["topology"]["runners"] = [ + { + "name": "itworx-validation-runner-2", + "running": False, + "health": "unhealthy", + } + ] + + guard_states = iter( + [ + ("clear", ""), + ("paused", "host_resource_guard_paused"), + ] + ) + commands: list[list[str]] = [] + events: list[dict[str, object]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: next(guard_states)) + monkeypatch.setattr(service, "_recovery_available", lambda _target: True) + monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") + monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "") + monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == "host_resource_guard_paused" + assert commands == [] + assert events == [] + + +def test_critical_current_pressure_after_cycle_entry_blocks_mutation( + fleet_module, monkeypatch +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["topology"]["dind"]["restartPolicy"] = "no" + current_samples = iter( + [ + { + "status": "ok", + "decision": {"pressure": "critical"}, + "host": {"swapCritical": True}, + } + ] + ) + commands: list[list[str]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) + monkeypatch.setattr( + fleet_module.RunnerCapacityController, + "snapshot", + lambda _self: next(current_samples), + ) + monkeypatch.setattr( + service, + "_run", + lambda args, timeout=20: commands.append(args) or "", + ) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == "runner_capacity_pressure_critical" + assert commands == [] + + +def test_stale_capacity_sample_blocks_automatic_mutation(fleet_module, monkeypatch): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["topology"]["dind"]["restartPolicy"] = "no" + stale = _healthy_capacity_snapshot() + stale["host"]["swapSampleAt"] = ( + datetime.now(UTC).timestamp() + - fleet_module.CAPACITY_SAMPLE_MAX_AGE_SECONDS + - 1 + ) + commands: list[list[str]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) + monkeypatch.setattr( + fleet_module.RunnerCapacityController, + "snapshot", + lambda _self: stale, + ) + monkeypatch.setattr( + service, + "_run", + lambda args, timeout=20: commands.append(args) or "", + ) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == "runner_capacity_telemetry_unavailable" + assert commands == [] + + +@pytest.mark.parametrize("capacity", [{"status": "unavailable"}, {"status": "not_configured"}]) +def test_unavailable_or_missing_current_pressure_blocks_mutation( + fleet_module, monkeypatch, capacity +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["topology"]["dind"]["restartPolicy"] = "no" + commands: list[list[str]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) + monkeypatch.setattr( + fleet_module.RunnerCapacityController, + "snapshot", + lambda _self: capacity, + ) + monkeypatch.setattr( + service, + "_run", + lambda args, timeout=20: commands.append(args) or "", + ) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == "runner_capacity_telemetry_unavailable" + assert commands == [] + + +def test_automatic_cycle_rejects_incomplete_critical_snapshot( + fleet_module, monkeypatch +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["status"] = "critical" + snapshot["topology"]["dind"].update( + {"running": False, "health": "none", "restartPolicy": "no"} + ) + snapshot["topology"].pop("runnerCount") + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr( + service, "_run", lambda *_args, **_kwargs: pytest.fail("critical fleet must not mutate") + ) + monkeypatch.setattr( + service, "_record", lambda **_event: pytest.fail("critical fleet must not audit a mutation") + ) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == "runner_fleet_snapshot_untrusted" + assert result["dindRecovery"]["status"] == "deferred" + + +def test_automatic_cycle_recovers_stalled_scheduler_in_critical_snapshot( + fleet_module, monkeypatch +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["status"] = "critical" + snapshot["scheduler"] = { + "status": "stalled", + "candidate": { + "target": "itworx-validation-runner-2", + "jobId": 22931, + "label": "linux-validation", + "ageSeconds": 900, + "reason": "stale_scheduler_assignment", + }, + "stallThresholdSeconds": 600, + } + commands: list[tuple[list[str], int]] = [] + events: list[dict[str, object]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr( + service, + "_scheduler_recovery_candidate", + lambda: snapshot["scheduler"]["candidate"], + ) + monkeypatch.setattr( + service, + "_container", + lambda name: { + "name": name, + "running": True, + "health": "healthy", + "startedAt": "2026-09-16T18:00:00Z", + "restarts": 0, + "oomKilled": False, + "restartPolicy": "unless-stopped", + }, + ) + monkeypatch.setattr(service, "_recovery_available", lambda _target: True) + monkeypatch.setattr( + service, + "_run", + lambda args, timeout=20: commands.append((args, timeout)) or "", + ) + monkeypatch.setattr(service, "_record", lambda **event: events.append(event) or True) + + result = service.automatic_cycle() + + assert commands == [ + (["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45) + ] + assert result["status"] == "ok" + assert result["schedulerRecovery"] == { + "status": "requested", + "target": "itworx-validation-runner-2", + "jobId": 22931, + } + assert [event["action"] for event in events] == ["restart", "restart.result"] + + +def test_automatic_cycle_allows_critical_storage_prune_with_safe_host_capacity( + fleet_module, monkeypatch +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["status"] = "critical" + snapshot["storage"] = { + "status": "critical", + "freeBytes": 8 * 1024**3, + "totalBytes": 40 * 1024**3, + } + commands: list[list[str]] = [] + events: list[dict[str, object]] = [] + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) + monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "") + monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) + + result = service.automatic_cycle() + + assert commands == [ + ["image", "prune", "--force"], + ["builder", "prune", "--force", "--filter", "until=168h"], + ] + assert result["status"] == "ok" + assert events == [ + { + "action": "cache.prune", + "target": "gitea-runner-dind", + "reason": "ci_storage_critical", + "result": "completed", + } + ] + + +@pytest.mark.parametrize( + ("capacity", "reason"), + [ + ({"status": "ok", "decision": {"pressure": "critical"}, "host": {"swapCritical": False}}, + "runner_capacity_pressure_critical"), + ({"status": "unavailable"}, "runner_capacity_telemetry_unavailable"), + ({"status": "ok", "decision": {}, "host": {}}, "runner_capacity_telemetry_untrusted"), + ], +) +def test_automatic_cycle_blocks_critical_or_untrusted_capacity_snapshot( + fleet_module, monkeypatch, capacity, reason +): + service = fleet_module.RunnerFleetService() + snapshot = _healthy_snapshot() + snapshot["status"] = "critical" + snapshot["capacity"] = capacity + snapshot["topology"]["dind"].update( + {"running": False, "health": "none", "restartPolicy": "no"} + ) + monkeypatch.setattr(service, "snapshot", lambda: snapshot) + monkeypatch.setattr( + service, "_run", lambda *_args, **_kwargs: pytest.fail("unsafe capacity must not mutate") + ) + monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) + + result = service.automatic_cycle() + + assert result["status"] == "deferred" + assert result["reason"] == reason + + +def test_host_guard_probe_fails_closed_when_path_is_not_posix_root(monkeypatch, fleet_module): + monkeypatch.setattr(fleet_module.os, "name", "nt") + + assert fleet_module._host_guard_state_unpatched() == ( + "unavailable", + "host_guard_state_untrusted", + ) + + def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 @@ -952,7 +1417,11 @@ def test_scheduler_recovery_rejects_runner_without_healthy_docker_state( ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() - snapshot["topology"]["runners"][0]["health"] = "unhealthy" + next( + runner + for runner in snapshot["topology"]["runners"] + if runner["name"] == "itworx-validation-runner-2" + )["health"] = "unhealthy" commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( diff --git a/backend/tests/test_runner_fleet_specialize.py b/backend/tests/test_runner_fleet_specialize.py new file mode 100644 index 0000000..c703c8d --- /dev/null +++ b/backend/tests/test_runner_fleet_specialize.py @@ -0,0 +1,2719 @@ +from __future__ import annotations + +import hashlib +import copy +import importlib.util +import io +import json +import os +import sys +import tarfile +from contextlib import nullcontext +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +BROKER = ROOT / "ops" / "gitea-deploy-broker" +SPECIALIZE_PATH = BROKER / "runner_fleet_specialize.py" + + +@pytest.fixture +def specialize(monkeypatch): + monkeypatch.syspath_prepend(str(BROKER)) + spec = importlib.util.spec_from_file_location("runner_fleet_specialize_test", SPECIALIZE_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _valid_task_image_manifest(specialize, revision="a" * 40, tree="b" * 40): + source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" + return { + "apiVersion": specialize.IMAGE_MANIFEST_SCHEMA, + "sourceRepository": specialize.SOURCE_REPOSITORY, + "sourceRevision": revision, + "sourceTree": tree, + "workflowPath": specialize.IMAGE_WORKFLOW, + "workflowRunId": "12", + "fullRunId": "13", + "artifactName": specialize.IMAGE_ARTIFACT_PREFIX + revision, + "dockerTarSha256": "c" * 64, + "dockerTarBytes": 1, + "images": [ + { + "profile": profile_id, + "reference": f"localhost/itworx-ci-{profile_id}:sha-{revision}", + "imageId": f"sha256:{str(index + 1) * 64}", + "architecture": "amd64", + "dockerfileSha256": "d" * 64, + "labels": { + "org.opencontainers.image.source": source, + "org.opencontainers.image.revision": revision, + "tech.itworx.source-tree": tree, + "tech.itworx.ci.profile": profile_id, + }, + } + for index, profile_id in enumerate(specialize.PROFILE_IDS) + ], + } + + +def _build_probe_record(specialize, *, container_id=None, status="intent"): + revision = "a" * 40 + tree = "b" * 40 + image_labels = { + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.revision": revision, + "tech.itworx.source-tree": tree, + "tech.itworx.ci.profile": "build", + } + record = specialize._new_build_probe_record( + {"sourceRevision": revision, "sourceTree": tree}, + { + "taskImageId": "sha256:" + "1" * 64, + "dockerfileSha256": "d" * 64, + "imageLabels": image_labels, + }, + ) + record["status"] = status + record["containerId"] = container_id + return record + + +def _build_probe_inspect(specialize, record, *, container_id=None, changes=None): + image_environment = ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] + item = { + "Id": container_id or "e" * 64, + "Name": f"/{record['name']}", + "Image": record["taskImageId"], + "Config": { + "Image": record["taskImageId"], + "Entrypoint": [specialize._BUILD_PROBE_EXECUTABLE], + "Cmd": [ + record["sourceRevision"], + record["sourceTree"], + "--hold-seconds", + str(specialize._BUILD_PROBE_HOLD_SECONDS), + ], + "Env": image_environment, + "Labels": specialize._build_probe_container_labels(record), + "Volumes": {"/home/itworx-builder/.local/share/docker": {}}, + }, + "HostConfig": { + "NetworkMode": "bridge", + "PidMode": "", + "IpcMode": "private", + "CgroupnsMode": "private", + "CgroupParent": "docker", + "NanoCpus": 4_000_000_000, + "Memory": 6 * 1024**3, + "PidsLimit": 1024, + "BlkioWeight": 100, + "Privileged": False, + "CapAdd": [], + "CapDrop": [], + "SecurityOpt": ["seccomp=unconfined", "systempaths=unconfined"], + "Devices": [{ + "PathOnHost": "/dev/net/tun", + "PathInContainer": "/dev/net/tun", + "CgroupPermissions": "rwm", + }], + "DeviceRequests": [], + "DeviceCgroupRules": [], + "ExtraHosts": ["gitea.example.com:198.19.104.1"], + "PortBindings": {}, + "Binds": [], + "Mounts": [], + "VolumesFrom": [], + "Tmpfs": {}, + "RestartPolicy": {"Name": "no"}, + }, + "Mounts": [{ + "Type": "volume", "Name": "9" * 64, + "Source": f"/var/lib/docker/volumes/{'9' * 64}/_data", + "Destination": "/home/itworx-builder/.local/share/docker", + "Driver": "local", "Mode": "", "RW": True, "Propagation": "", + }], + "State": {"Running": True, "Pid": 77}, + } + if changes: + for section, values in changes.items(): + item[section].update(values) + return item, image_environment + + +def _write_task_image_archive(specialize, path: Path, *, extra_label: bool = False): + images = _valid_task_image_manifest(specialize)["images"] + docker_manifest = [] + members: dict[str, bytes] = {} + for index, image in enumerate(images): + labels = specialize._runtime_task_image_labels(image) + assert labels is not None + if extra_label and index == 0: + labels["untrusted.extra"] = "value" + config = { + "architecture": "amd64", + "os": "linux", + "config": {"Labels": labels}, + } + config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() + config_name = f"{image['profile']}.json" + layer_name = f"{image['profile']}/layer.tar" + image["imageId"] = "sha256:" + hashlib.sha256(config_bytes).hexdigest() + members[config_name] = config_bytes + members[layer_name] = b"layer" + docker_manifest.append( + { + "Config": config_name, + "RepoTags": [image["reference"]], + "Layers": [layer_name], + } + ) + members["manifest.json"] = json.dumps(docker_manifest).encode() + with tarfile.open(path, mode="w:") as archive: + for name, payload in members.items(): + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + return images + + +def _active_recovery_fixture(specialize): + rows = [] + registration_ids = {} + originals = [] + containers = {} + for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=100): + registration_ids[profile.id] = index + rows.append( + { + "name": profile.container_name, + "displayName": profile.display_name, + "id": index, + "labels": sorted({profile.runner_label, *profile.legacy_labels}), + "disabled": True, + "busy": False, + "status": "offline", + } + ) + containers[profile.container_name] = {"State": {"Running": False}} + for index, name in enumerate(specialize._LEGACY_V1_NAMES, start=9): + labels = sorted(specialize._legacy_allowed_labels("v1", name)) + legacy_row = { + "name": name, + "id": index, + "labels": labels, + "disabled": True, + "busy": False, + "status": "offline", + } + rows.append(legacy_row) + originals.append({"name": name, "id": index, "labels": labels, "disabled": True}) + dind = {"State": {"Running": False, "Health": {"Status": "none"}}} + containers[specialize.DIND_NAME] = dind + journal = { + "legacyTopology": "v1", + "newRegistrationIds": registration_ids, + "originalRegistrations": originals, + } + model = {"services": {"runner-dind": {"privileged": True}}} + return rows, containers, dind, journal, model + + +def _legacy_dind_mounts(specialize): + return [ + { + "type": "bind", + "source": specialize.LEGACY_DIND_STORAGE_ROOT.as_posix(), + "target": "/home/rootless/.local/share/docker", + "read_only": False, + }, + { + "type": "bind", + "source": "/mnt/cache/appdata/itworx-security-runner-workspace", + "target": "/workspace", + "read_only": False, + }, + { + "type": "bind", + "source": "/mnt/cache/appdata/itworx-validation-runner-2-workspace", + "target": "/workspace-secondary", + "read_only": False, + }, + ] + + +def _specialized_dind_mounts(specialize): + return [ + { + "type": "bind", + "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), + "target": "/var/lib/docker", + "read_only": False, + }, + { + "type": "bind", + "source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), + "target": specialize._DIND_CONTROL_CONTAINER_ROOT, + "read_only": False, + }, + { + "type": "bind", + "source": str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), + "target": "/home/rootless/.local/share/docker", + "read_only": True, + }, + *[ + { + "type": "bind", + "source": str( + specialize._host_volume_path( + specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name + ) + ), + "target": f"/{profile.container_name}-workspace", + "read_only": False, + } + for profile in specialize.RUNNER_PROFILES.values() + ], + ] + + +def _base_v1_compose_model(specialize): + services = { + "deploy-control": { + "container_name": specialize.CONTROLLER_CONTAINER, + "image": "itworx/appops-deploy-control:fixed", + }, + "runner-dind": { + "container_name": specialize.DIND_NAME, + "image": "docker@sha256:7c3e797187e43738220462658f4586572cbd3bf009f728b21e34d9c5c06ce431", + "user": "rootless", + "privileged": True, + "cpus": 6, + "mem_limit": "12g", + "restart": "unless-stopped", + "security_opt": ["label=disable"], + "command": ["--default-address-pool=base=198.19.0.0/16,size=27"], + "volumes": _legacy_dind_mounts(specialize), + }, + } + for name, service_name in specialize._OLD_SERVICES.items(): + if name not in specialize._LEGACY_V1_NAMES: + continue + services[service_name] = { + "container_name": name, + "image": specialize.RUNNER_IMAGE, + "profiles": ["runner"], + "cpus": 1, + "mem_limit": "1g", + "volumes": [ + { + "type": "bind", + "source": source, + "target": target, + "read_only": read_only, + } + for source, target, read_only in specialize._expected_legacy_mounts("v1", name) + ], + "labels": {"tech.itworx.role": "gitea-actions-runner"}, + } + return { + "services": services, + "networks": { + "runner-sandbox": {"name": specialize._RUNNER_NETWORK_NAME} + }, + } + + +def _gitea_28_run( + specialize, revision, *, run_id=21533, attempt=1, status="completed", conclusion="success" +): + filename = specialize.FULL_WORKFLOW.rsplit("/", 1)[-1] + return { + "id": run_id, + "run_attempt": attempt, + "path": f"{filename}@refs/heads/main", + "status": status, + "conclusion": conclusion, + "event": "push", + "head_branch": "main", + "head_sha": revision, + "repository": {"id": 9, "full_name": "Jens/Unraid-AppOps-Gateway"}, + "head_repository": None, + } + + +def _gitea_workflow_api(specialize, rows, *, run_pages=None, calls=None): + run_pages = run_pages or {1: rows} + calls = calls if calls is not None else [] + + def api(path): + calls.append(path) + if path.endswith("/actions/workflows?limit=100&page=1"): + return { + "workflows": [ + { + "id": workflow.rsplit("/", 1)[-1], + "path": workflow, + "state": "active", + } + for workflow in (specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW) + ] + } + if "/actions/runs?" in path: + page = int(path.rsplit("page=", 1)[-1]) + return {"workflow_runs": run_pages.get(page, [])} + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + return api, calls + + +def test_gitea_workflow_inventory_uses_exact_active_filename_and_path( + specialize, monkeypatch +): + workflow = specialize.FULL_WORKFLOW + filename = workflow.rsplit("/", 1)[-1] + inventory = { + "workflows": [ + {"id": filename, "path": workflow, "state": "active"}, + { + "id": specialize.IMAGE_WORKFLOW.rsplit("/", 1)[-1], + "path": specialize.IMAGE_WORKFLOW, + "state": "active", + }, + ] + } + monkeypatch.setattr(specialize, "_api_json", lambda _path: inventory) + assert specialize._workflow_id(workflow) == filename + + for bad in ( + {"id": 34, "path": workflow, "state": "active"}, + {"id": filename, "path": ".gitea/workflows/other.yml", "state": "active"}, + {"id": filename, "path": workflow, "state": "enabled"}, + ): + monkeypatch.setattr( + specialize, "_api_json", lambda _path, row=bad: {"workflows": [row]} + ) + with pytest.raises( + specialize.RunnerFleetSpecializeError, match="workflow_unavailable" + ): + specialize._workflow_id(workflow) + + +def test_successful_main_run_accepts_actual_gitea_run_without_workflow_id( + specialize, monkeypatch +): + revision = "a" * 40 + run = _gitea_28_run(specialize, revision) + api, calls = _gitea_workflow_api(specialize, [run]) + monkeypatch.setattr(specialize, "_api_json", api) + + selected = specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) + + assert selected is run + assert "workflow_id" not in selected + assert selected["head_repository"] is None + assert calls[0].endswith("/actions/workflows?limit=100&page=1") + assert calls[1].endswith("/actions/runs?branch=main&limit=100&page=1") + + +@pytest.mark.parametrize( + "changes", + [ + {"path": "managed-validation.yml@refs/heads/feature"}, + {"repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, + {"head_repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, + {"head_sha": "b" * 40}, + {"event": "workflow_dispatch"}, + {"head_branch": "feature"}, + {"pull_requests": [{"number": 123}]}, + ], +) +def test_successful_main_run_rejects_untrusted_gitea_identity( + specialize, monkeypatch, changes +): + revision = "a" * 40 + run = {**_gitea_28_run(specialize, revision), **changes} + api, _calls = _gitea_workflow_api(specialize, [run]) + monkeypatch.setattr(specialize, "_api_json", api) + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="protected_validation_missing", + ): + specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) + + +@pytest.mark.parametrize( + ("latest", "error"), + [ + ({"status": "in_progress", "conclusion": None}, "protected_validation_missing"), + ({"status": "completed", "conclusion": "failure"}, "protected_validation_missing"), + ( + {"run_attempt": 2, "status": "completed", "conclusion": "failure"}, + "protected_validation_missing", + ), + ({"run_attempt": None}, "workflow_runs_invalid"), + ], +) +def test_newest_matching_gitea_run_or_attempt_supersedes_prior_success( + specialize, monkeypatch, latest, error +): + revision = "a" * 40 + successful = _gitea_28_run(specialize, revision, run_id=21532) + newer = { + **_gitea_28_run(specialize, revision, run_id=21533), + **latest, + } + if latest.get("run_attempt") == 2: + newer["id"] = successful["id"] + api, _calls = _gitea_workflow_api(specialize, [successful, newer]) + monkeypatch.setattr(specialize, "_api_json", api) + with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): + specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) + + +def test_workflow_run_pagination_fails_closed_at_fixed_page_limit(specialize, monkeypatch): + revision = "a" * 40 + wrong = {"path": "unrelated.yml@refs/heads/main"} + pages = {page: [wrong] * 100 for page in range(1, 11)} + api, calls = _gitea_workflow_api(specialize, [], run_pages=pages) + monkeypatch.setattr(specialize, "_api_json", api) + with pytest.raises( + specialize.RunnerFleetSpecializeError, match="workflow_runs_invalid" + ): + specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) + assert len([path for path in calls if "/actions/runs?" in path]) == 10 + + +def test_archive_verification_hashes_large_tar_without_read_bytes(specialize, tmp_path, monkeypatch): + archive_path = tmp_path / "images.tar" + payload = b"bounded-stream" * 4096 + with tarfile.open(archive_path, "w") as archive: + member = tarfile.TarInfo("layer.tar") + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + digest = hashlib.sha256(archive_path.open("rb").read()).hexdigest() + + def forbidden_read_bytes(_path): + raise AssertionError("archive verifier must stream the tar file") + + monkeypatch.setattr(Path, "read_bytes", forbidden_read_bytes) + specialize._verify_docker_archive(archive_path, archive_path.stat().st_size, digest) + + +def test_archive_verifier_requires_exact_pinned_inherited_labels( + specialize, tmp_path +): + archive_path = tmp_path / "task-images.tar" + images = _write_task_image_archive(specialize, archive_path) + specialize._verify_docker_archive( + archive_path, + archive_path.stat().st_size, + hashlib.sha256(archive_path.read_bytes()).hexdigest(), + images, + ) + + images = _write_task_image_archive(specialize, archive_path, extra_label=True) + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="image_archive_identity_invalid", + ): + specialize._verify_docker_archive( + archive_path, + archive_path.stat().st_size, + hashlib.sha256(archive_path.read_bytes()).hexdigest(), + images, + ) + + +def test_installed_image_manifest_requires_complete_source_and_profile_pins(specialize): + manifest = _valid_task_image_manifest(specialize) + + assert specialize._valid_installed_manifest(manifest) is True + manifest["images"][0]["reference"] = "localhost/itworx-ci-checks:latest" + assert specialize._valid_installed_manifest(manifest) is False + + +def test_task_image_runtime_verifier_accepts_only_exact_pinned_inherited_labels( + specialize, monkeypatch +): + expected = _valid_task_image_manifest(specialize)["images"][0] + runtime_labels = specialize._runtime_task_image_labels(expected) + assert runtime_labels is not None + assert len(runtime_labels) == 14 + actual = { + "Id": expected["imageId"], + "Os": "linux", + "Architecture": "amd64", + "Config": {"Labels": runtime_labels}, + } + monkeypatch.setattr( + specialize, + "_run", + lambda *_args, **_kwargs: json.dumps([actual]), + ) + assert specialize._verify_manifest_images_in_dind([expected]) is True + + actual["Config"]["Labels"]["untrusted.extra"] = "value" + assert specialize._verify_manifest_images_in_dind([expected]) is False + del actual["Config"]["Labels"]["untrusted.extra"] + del expected["labels"]["tech.itworx.source-tree"] + assert specialize._verify_manifest_images_in_dind([expected]) is False + + +def test_bootstrap_receipt_is_source_bound_private_and_cas_consumed( + specialize, tmp_path, monkeypatch +): + state_root = tmp_path / "runner-fleet-state" + state_root.mkdir(mode=0o700) + os.chmod(state_root, 0o700) + manifest_path = state_root / "task-image-manifest.json" + marker_path = state_root / "specialized.json" + receipt_path = state_root / "admission-bootstrap.json" + manifest_bytes = json.dumps( + _valid_task_image_manifest(specialize), sort_keys=True, separators=(",", ":") + ).encode() + b"\n" + manifest_path.write_bytes(manifest_bytes) + os.chmod(manifest_path, 0o600) + manifest_sha = hashlib.sha256(manifest_bytes).hexdigest() + marker = { + "schemaVersion": 1, + "status": "active", + "sourceScriptSha256": "e" * 64, + "imageManifestSha256": manifest_sha, + "imageSourceRevision": "a" * 40, + "imageSourceTree": "b" * 40, + "verifiedRunners": list(specialize._SPECIALIZED_NAMES), + "resourceEnvelopeDigest": "f" * 64, + "updatedAt": "2026-10-01T12:00:00Z", + } + marker_path.write_text(json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n") + os.chmod(marker_path, 0o600) + registrations = [] + for profile in specialize.RUNNER_PROFILES.values(): + registrations.append( + { + "profileId": profile.id, + "displayName": profile.display_name, + "containerName": profile.container_name, + "id": len(registrations) + 10, + "labels": sorted({profile.runner_label, *profile.legacy_labels}), + } + ) + receipt = { + "schemaVersion": 1, + "status": "pending", + "sourceScriptSha256": marker["sourceScriptSha256"], + "imageManifestSha256": manifest_sha, + "imageSourceRevision": "a" * 40, + "imageSourceTree": "b" * 40, + "verifiedRunners": list(specialize._SPECIALIZED_NAMES), + "registrations": registrations, + "resourceEnvelopeDigest": marker["resourceEnvelopeDigest"], + "createdAt": "2026-10-01T12:00:00Z", + } + receipt_path.write_text(json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n") + os.chmod(receipt_path, 0o600) + + assert specialize._valid_admission_bootstrap_receipt(receipt) + validated = specialize.read_admission_bootstrap_receipt( + receipt_path, marker_path=marker_path, image_manifest_path=manifest_path + ) + assert validated is not None + pending_sha = validated[1] + assert validated[0]["status"] == "pending" + + assert specialize.consume_admission_bootstrap_receipt( + pending_sha, + receipt_path, + marker_path=marker_path, + image_manifest_path=manifest_path, + ) + consumed = specialize.read_admission_bootstrap_receipt( + receipt_path, marker_path=marker_path, image_manifest_path=manifest_path + ) + assert consumed is not None + assert consumed[0]["status"] == "consumed" + assert consumed[1] == pending_sha + assert specialize.consume_admission_bootstrap_receipt( + pending_sha, + receipt_path, + marker_path=marker_path, + image_manifest_path=manifest_path, + ) + assert not specialize.consume_admission_bootstrap_receipt( + "0" * 64, + receipt_path, + marker_path=marker_path, + image_manifest_path=manifest_path, + ) + + +def test_runner_config_is_profile_locked_and_only_backend_frontend_keep_legacy_labels(specialize): + revision = "a" * 40 + checks = specialize._render_runner_config("checks", revision).decode() + backend = specialize._render_runner_config("backend", revision).decode() + frontend = specialize._render_runner_config("frontend", revision).decode() + security = specialize._render_runner_config("security", revision).decode() + + assert "itworx-ci-checks:docker://localhost/itworx-ci-checks:sha-" in checks + assert "linux-validation" not in checks and "ubuntu-latest" not in checks + assert "linux-validation" in backend and "ubuntu-latest" in backend + assert "linux-validation" in frontend and "ubuntu-latest" in frontend + assert "linux-validation" not in security and "ubuntu-latest" not in security + for profile_id, profile in specialize.RUNNER_PROFILES.items(): + config = specialize._render_runner_config(profile_id, revision).decode() + task_image = specialize._task_image_reference(profile_id, revision) + job_cpu_milli, job_memory_mib = specialize._runner_job_limits(profile_id) + cpu = f"{job_cpu_milli / 1000:.3f}".rstrip("0").rstrip(".") + assert f"--cpus={cpu} --memory={job_memory_mib}m" in config + assert f"source=itworx-ci-{profile_id}-cache,target=/opt/itworx-ci-cache" in config + assert f'valid_volumes:\n - "itworx-ci-{profile_id}-cache"' in config + assert "[0-9]*/**" not in config + assert "DOCKER_HOST:" not in config + assert 'docker_host: "-"' in config + assert "host.docker.internal" not in config + assert all( + f"docker://{task_image}" in line + for line in config.splitlines() + if ":docker://" in line + ) + if profile_id == "build": + assert "--security-opt=seccomp=unconfined" in config + assert "--security-opt=systempaths=unconfined" in config + assert "--device=/dev/net/tun:/dev/net/tun:rwm" in config + else: + assert "--security-opt=" not in config + assert "--device=" not in config + + +def test_fixed_workload_reservations_are_subtracted_from_aggregate_profiles(specialize): + for profile_id in ("backend", "frontend", "browser"): + profile = specialize.RUNNER_PROFILES[profile_id] + service_cpu, service_memory = specialize.FIXED_WORKLOAD_RESERVATION[profile_id] + job_cpu, job_memory = specialize._runner_job_limits(profile_id) + assert job_cpu + service_cpu == profile.cpu_milli + assert job_memory + service_memory == profile.memory_mib + config = specialize._render_runner_config(profile_id, "a" * 40).decode() + cpu = f"{job_cpu / 1000:.3f}".rstrip("0").rstrip(".") + assert f"--cpus={cpu} --memory={job_memory}m" in config + + for profile_id in ("checks", "security"): + profile = specialize.RUNNER_PROFILES[profile_id] + assert specialize._runner_job_limits(profile_id) == ( + profile.cpu_milli, + profile.memory_mib, + ) + assert specialize._runner_job_limits("build") == (4000, 6144) + + +def test_base_compose_model_explicitly_resolves_the_runner_profile(specialize, monkeypatch): + compose = b"captured v1 source" + base = _base_v1_compose_model(specialize) + monkeypatch.setattr( + specialize, + "_KNOWN_V1_COMPOSE_SHA256", + hashlib.sha256(compose).hexdigest(), + ) + calls = [] + + def run_compose(_compose_bytes, *arguments, **_kwargs): + calls.append(arguments) + return json.dumps(base) + + monkeypatch.setattr(specialize, "_run_compose", run_compose) + + assert specialize._validate_base_composition(compose) == "v1" + assert calls == [("--profile", "runner", "config", "--format", "json")] + + +def test_rendered_compose_uses_fixed_rootful_dind_and_shared_workspaces(specialize): + base = _base_v1_compose_model(specialize) + rendered = json.loads(specialize._render_specialized_compose(base)) + services = rendered["services"] + dind = services["runner-dind"] + + assert services["deploy-control"] == base["services"]["deploy-control"] + assert dind["user"] == "0" + assert dind["command"][0] == "dockerd" + assert dind["command"][1] == f"--host={specialize._DIND_DOCKER_HOST}" + assert "--data-root=/var/lib/docker" in dind["command"] + assert "--exec-opt=native.cgroupdriver=cgroupfs" in dind["command"] + assert dind["environment"] == {"DOCKER_HOST": specialize._DIND_DOCKER_HOST} + assert _mounts(specialize, dind) == { + (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), + ( + str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), + specialize._DIND_CONTROL_CONTAINER_ROOT, + False, + ), + ( + str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), + "/home/rootless/.local/share/docker", + True, + ), + *{ + ( + str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), + f"/{profile.container_name}-workspace", + False, + ) + for profile in specialize.RUNNER_PROFILES.values() + }, + } + assert len([volume for volume in dind["volumes"] if volume["type"] == "bind"]) == 9 + expected_services = {"deploy-control", "runner-dind"} + expected_services.update( + specialize._OLD_SERVICES[name] for name in specialize._LEGACY_V1_NAMES + ) + expected_services.update(f"runner-{profile_id}" for profile_id in specialize.PROFILE_IDS) + assert set(services) == expected_services + for profile_id in specialize.PROFILE_IDS: + service = services[f"runner-{profile_id}"] + assert service["environment"]["DOCKER_HOST"] == specialize._DIND_DOCKER_HOST + assert service["extra_hosts"] == ["gitea.example.com:198.19.104.1"] + assert ( + str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), + specialize._DIND_CONTROL_CONTAINER_ROOT, + True, + ) in specialize._compose_mounts(service) + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") +def test_profile_state_creation_uses_effective_runner_image_owners_and_resumes( + specialize, tmp_path, monkeypatch +): + state = tmp_path / "state" + state.mkdir(mode=0o700) + data_root = state / "runner-data" + workspace_root = state / "workspaces" + config_root = state / "runner-configs" + image_root = state / "task-images" + dind_root = state / "dind" / "docker" + shadow_root = state / "dind" / "rootless-unused" + control_root = state / "dind-control" + for name, value in ( + ("_SPECIALIZED_DATA_ROOT", data_root), + ("_SPECIALIZED_WORKSPACE_ROOT", workspace_root), + ("RUNNER_CONFIG_ROOT", config_root), + ("TASK_IMAGE_ROOT", image_root), + ("DIND_STORAGE_ROOT", dind_root), + ("_DIND_ROOTLESS_SHADOW_ROOT", shadow_root), + ("_DIND_CONTROL_ROOT", control_root), + ): + monkeypatch.setattr(specialize, name, value) + + control_root.mkdir(mode=0o700) + specialize._prepare_profile_state("a" * 40) + for profile_id, profile in specialize.RUNNER_PROFILES.items(): + data_path = data_root / profile_id + workspace_path = workspace_root / profile.container_name + assert data_path.stat().st_uid == 0 + assert workspace_path.stat().st_uid == 1000 + assert control_root.stat().st_uid == 0 + assert os.stat(control_root).st_mode & 0o777 == 0o700 + + config_path = config_root / "checks.yaml" + expected = specialize._render_runner_config("checks", "a" * 40) + invalid_configs = ( + expected.replace( + b'valid_volumes:\n - "itworx-ci-checks-cache"', + b'valid_volumes:\n - "/itworx-ci-checks-workspace/[0-9]*/**"', + 1, + ), + expected.replace( + b'valid_volumes:\n - "itworx-ci-checks-cache"', + b'valid_volumes:\n - "/run/itworx-ci-control/**"', + 1, + ), + expected.replace( + b' docker_host: "-"', + b' docker_host: "tcp://runner-dind:2375"', + 1, + ), + expected.replace( + b" envs:\n", + b" envs:\n DOCKER_HOST: tcp://runner-dind:2375\n", + 1, + ), + ) + for invalid_config in invalid_configs: + config_path.write_bytes(invalid_config) + os.chmod(config_path, 0o600) + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="profile_config_adoption_rejected", + ): + specialize._prepare_profile_state("a" * 40) + config_path.write_bytes(expected) + os.chmod(config_path, 0o600) + specialize._prepare_profile_state("a" * 40) + + +def _mounts(specialize, service): + return specialize._compose_mounts(service) + + +def test_rendered_compose_uses_only_canonical_protected_volume_sources(specialize): + base = _base_v1_compose_model(specialize) + rendered = json.loads(specialize._render_specialized_compose(base)) + services = rendered["services"] + assert specialize._compose_mounts(services["runner-dind"]) == { + (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), + ( + str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), + specialize._DIND_CONTROL_CONTAINER_ROOT, + False, + ), + ( + str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), + "/home/rootless/.local/share/docker", + True, + ), + *{ + ( + str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), + f"/{profile.container_name}-workspace", + False, + ) + for profile in specialize.RUNNER_PROFILES.values() + }, + } + assert services["runner-dind"]["restart"] == "no" + for profile_id, profile in specialize.RUNNER_PROFILES.items(): + service = services[f"runner-{profile_id}"] + assert specialize._compose_mounts(service) == { + (str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "/data", False), + (str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "/runner-config.yaml", True), + (str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True), + (str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False), + } + assert "-f" in specialize._compose_command("up", "-d") + assert specialize._compose_command("up", "-d")[specialize._compose_command("up", "-d").index("-f") + 1] == "-" + + +def test_specialized_compose_validation_rejects_socket_and_gateway_broadening( + specialize, monkeypatch +): + rendered = json.loads( + specialize._render_specialized_compose(_base_v1_compose_model(specialize)) + ) + mutations = ( + lambda model: model["services"]["runner-dind"]["environment"].update( + DOCKER_HOST="tcp://0.0.0.0:2375" + ), + lambda model: next( + volume + for volume in model["services"]["runner-dind"]["volumes"] + if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT + ).update(read_only=True), + lambda model: next( + volume + for volume in model["services"]["runner-frontend"]["volumes"] + if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT + ).update(read_only=False), + lambda model: model["services"]["runner-frontend"]["extra_hosts"].append( + "host.docker.internal:host-gateway" + ), + lambda model: model["services"]["runner-frontend"].update( + network_mode="host" + ), + lambda model: model["services"]["runner-frontend"].update( + privileged=True + ), + lambda model: model["networks"]["runner-sandbox"].update( + name="runner-sandbox" + ), + ) + for mutate in mutations: + candidate = copy.deepcopy(rendered) + mutate(candidate) + monkeypatch.setattr(specialize, "_compose_model", lambda _raw, value=candidate: value) + with pytest.raises(specialize.RunnerFleetSpecializeError): + specialize._validate_composition(b"candidate") + + +def test_specialized_compose_validation_accepts_real_compose_normalization( + specialize, monkeypatch +): + normalized = json.loads( + specialize._render_specialized_compose(_base_v1_compose_model(specialize)) + ) + service = normalized["services"]["runner-frontend"] + service.pop("privileged") + service["extra_hosts"] = ["gitea.example.com=198.19.104.1"] + service["networks"] = {"runner-sandbox": {"aliases": ["itworx-ci-frontend"]}} + monkeypatch.setattr(specialize, "_compose_model", lambda _raw: normalized) + + assert specialize._validate_composition(b"normalized candidate") + + +def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize): + base = _base_v1_compose_model(specialize) + item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, + running=True, restart_policy="unless-stopped") + assert specialize._legacy_dind_observation(item, base)["running"] is True + for key, value in (("User", "0"), ("Cmd", ["--data-root=/var/lib/docker"]), + ("Entrypoint", ["sh"])): + changed = copy.deepcopy(item) + changed["Config"][key] = value + with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): + specialize._legacy_dind_observation(changed, base) + changed = copy.deepcopy(item) + changed["Mounts"][-1]["Source"] = "/untrusted/docker" + with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): + specialize._legacy_dind_observation(changed, base) + + +@pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false", "2|cgroupfs|true"]) +def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces( + specialize, monkeypatch, support +): + model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize))) + service = model["services"]["runner-dind"] + item = _legacy_dind_fixture(specialize, _base_v1_compose_model(specialize), + image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no") + item["Config"].update(User="0", Cmd=service["command"]) + item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"] + item["HostConfig"].update( + CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={} + ) + item["Mounts"] = [{"Type": "bind", "Source": m["source"], + "Destination": m["target"], "RW": not m["read_only"]} + for m in service["volumes"]] + assert specialize._validate_dind_outer(item, model) is True + monkeypatch.setattr(specialize, "_inspect", lambda _name: item) + commands = [] + + def run(args, **_kwargs): + commands.append(args) + return support + + monkeypatch.setattr(specialize, "_run", run) + if support == "2|cgroupfs|true": + specialize._require_dind_resource_support(model) + assert f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}" in commands[-1] + else: + with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_limits_unsupported"): + specialize._require_dind_resource_support(model) + missing_workspace = copy.deepcopy(item) + missing_workspace["Mounts"].pop() + assert specialize._validate_dind_outer(missing_workspace, model) is False + changed_cases = [] + tcp_host = copy.deepcopy(item) + tcp_host["Config"]["Env"] = ["DOCKER_HOST=tcp://0.0.0.0:2375"] + changed_cases.append(tcp_host) + tcp_command = copy.deepcopy(item) + tcp_command["Config"]["Cmd"][1] = "--host=tcp://0.0.0.0:2375" + changed_cases.append(tcp_command) + published_port = copy.deepcopy(item) + published_port["HostConfig"]["PortBindings"] = { + "2375/tcp": [{"HostIp": "0.0.0.0", "HostPort": "2375"}] + } + changed_cases.append(published_port) + for changed in changed_cases: + assert specialize._validate_dind_outer(changed, model) is False + + +def test_outer_runner_must_match_fixed_compose_mount_and_image_identity(specialize): + profile_id = "frontend" + profile = specialize.RUNNER_PROFILES[profile_id] + name = profile.container_name + image = { + "Name": f"/{name}", + "Image": specialize.RUNNER_IMAGE_ID, + "Config": { + "Image": specialize.RUNNER_IMAGE, + "Labels": { + "com.docker.compose.project": specialize.PROJECT, + "com.docker.compose.service": f"runner-{profile_id}", + "tech.itworx.role": "gitea-actions-runner", + "tech.itworx.ci.profile": profile_id, + }, + "Env": [ + "CONFIG_FILE=/runner-config.yaml", + f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}", + ], + }, + "State": {"Running": False}, + "HostConfig": { + "CgroupParent": "itworx-ci", + "NanoCpus": 500_000_000, + "Memory": 512 * specialize._MIB, + "PidsLimit": 192, + "PidMode": "", + "NetworkMode": specialize._RUNNER_NETWORK_NAME, + "Privileged": False, + "ExtraHosts": ["gitea.example.com:198.19.104.1"], + "RestartPolicy": {"Name": "no"}, + }, + "Mounts": [ + {"Source": str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "Destination": "/data", "RW": True}, + {"Source": str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "Destination": "/runner-config.yaml", "RW": False}, + {"Source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "Destination": specialize._DIND_CONTROL_CONTAINER_ROOT, "RW": False}, + { + "Source": str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / name)), + "Destination": f"/{name}-workspace", + "RW": True, + }, + ], + } + + assert specialize._validate_outer_item(name, image, profile_id) + unqualified_network = copy.deepcopy(image) + unqualified_network["HostConfig"]["NetworkMode"] = "runner-sandbox" + assert not specialize._validate_outer_item(name, unqualified_network, profile_id) + tcp_env = copy.deepcopy(image) + tcp_env["Config"]["Env"][-1] = "DOCKER_HOST=tcp://runner-dind:2375" + assert not specialize._validate_outer_item(name, tcp_env, profile_id) + writable_socket = copy.deepcopy(image) + next( + mount for mount in writable_socket["Mounts"] + if mount["Destination"] == specialize._DIND_CONTROL_CONTAINER_ROOT + )["RW"] = True + assert not specialize._validate_outer_item(name, writable_socket, profile_id) + host_gateway = copy.deepcopy(image) + host_gateway["HostConfig"]["ExtraHosts"].append("host.docker.internal:host-gateway") + assert not specialize._validate_outer_item(name, host_gateway, profile_id) + image["Image"] = "sha256:" + "0" * 64 + assert not specialize._validate_outer_item(name, image, profile_id) + + +def test_atomic_guard_replace_uses_expected_hash_as_compare_and_swap(specialize, tmp_path): + path = tmp_path / "guard.sh" + path.write_bytes(b"original guard\n") + original_sha = hashlib.sha256(path.read_bytes()).hexdigest() + + specialize._atomic_write_owned( + path, + b"patched guard\n", + mode=0o600, + expected_sha256=original_sha, + ) + assert path.read_bytes() == b"patched guard\n" + + with pytest.raises(specialize.RunnerFleetSpecializeError, match="compare_and_swap_failed"): + specialize._atomic_write_owned( + path, + b"stale write\n", + mode=0o600, + expected_sha256=original_sha, + ) + assert path.read_bytes() == b"patched guard\n" + + +def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(specialize): + source = ( + "stack_recovery=0\n" + "set_managed_runners_disabled false\n" + "unexpected=0\n" + ' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' + ) + patched = specialize._patch_runner_guard( + source, + "fixed-module --state", + 'fixed-module --guard-allows-container "$name"', + ) + + assert "--fence-legacy" in patched + assert "--guard-allows-legacy-container" in patched + assert 'if [ "$specialization_state" = "legacy" ]; then' in patched + assert 'elif [ "$specialization_state" = "active" ]' in patched + assert '--guard-allows-container "$name"' in patched + assert " set_managed_runners_disabled false\n" in patched + with pytest.raises(specialize.RunnerFleetSpecializeError, match="runner_guard_shape_invalid"): + specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow") + + +def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch): + script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() + monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest()) + real_os = os + + class RootPosixOS: + name = "posix" + + @staticmethod + def geteuid(): + return 0 + + def __getattr__(self, name): + return getattr(real_os, name) + + monkeypatch.setattr(specialize, "os", RootPosixOS()) + monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False}) + shell_calls = [] + monkeypatch.setattr(specialize, "_run_approved_shell_script", shell_calls.append) + + with pytest.raises(specialize.RunnerFleetSpecializeError, match="script_not_approved"): + specialize.execute(script + b"# altered\n") + + assert shell_calls == [] + + +def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_edits( + specialize, tmp_path, monkeypatch +): + runner = tmp_path / "runner-guard" + host = tmp_path / "host-guard" + runner.write_bytes( + b"stack_recovery=0\n" + b" set_managed_runners_disabled false\n" + b"unexpected=0\n" + b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' + ) + host.write_bytes( + b"# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" + b"old capacity writer\n" + b"# ITWORX_RUNNER_CAPACITY_V2_END\n" + b"runner_healthy=1\n" + b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' + b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' + b" recover legacy stack\n" + b" fi\n" + ) + runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() + host_sha = hashlib.sha256(host.read_bytes()).hexdigest() + monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) + monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) + monkeypatch.setattr(specialize, "RUNNER_GUARD_SOURCE_SHA256", runner_sha) + monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", host_sha) + monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") + monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) + journal = { + "guardFiles": [ + {"path": str(runner), "originalSha256": runner_sha, "installedSha256": None}, + {"path": str(host), "originalSha256": host_sha, "installedSha256": None}, + ] + } + + specialize._service_guard_scripts(journal) + + installed_runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() + assert journal["guardFiles"][0]["installedSha256"] == installed_runner_sha + assert "--fence-legacy" in runner.read_text(encoding="utf-8") + assert "# ITWORX_RUNNER_CAPACITY_V2_BEGIN" not in host.read_text(encoding="utf-8") + + host.write_text("operator-owned guard revision\n", encoding="utf-8") + with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_source_changed"): + specialize._service_guard_scripts(journal) + assert host.read_text(encoding="utf-8") == "operator-owned guard revision\n" + + +def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(specialize, monkeypatch): + script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() + digest = hashlib.sha256(script).hexdigest() + monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", digest) + + real_os = os + + class RootPosixOS: + name = "posix" + + @staticmethod + def geteuid(): + return 0 + + def __getattr__(self, name): + return getattr(real_os, name) + + monkeypatch.setattr(specialize, "os", RootPosixOS()) + monkeypatch.setattr(specialize, "_migration_lock", nullcontext) + monkeypatch.setattr(specialize, "specialization_state", lambda: "legacy") + monkeypatch.setitem( + sys.modules, + "status_server", + type("StatusServerStub", (), {"_require_controller_release": staticmethod(lambda: { + "sourceRepository": specialize.SOURCE_REPOSITORY, + "sourceRevision": "a" * 40, + "sourceTree": "b" * 40, + })}), + ) + shell_calls: list[bytes] = [] + state = {"active": False} + + def run_approved(received): + shell_calls.append(received) + state["active"] = True + + monkeypatch.setattr(specialize, "_run_approved_shell_script", run_approved) + monkeypatch.setattr( + specialize, + "verified_topology", + lambda: { + "ready": state["active"], + "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if state["active"] else [], + "resourceEnvelope": {"ready": state["active"], "digest": "c" * 64}, + }, + ) + + result = specialize.execute(script) + + assert result["status"] == "specialized" + assert result["verifiedRunners"] == list(specialize._SPECIALIZED_NAMES) + assert shell_calls == [script] + + +def test_run_with_input_keeps_registration_secret_out_of_command_arguments(specialize, monkeypatch): + observed = {} + + def fake_run(args, **kwargs): + observed["args"] = args + observed["input"] = kwargs["input"] + observed["capture_output"] = kwargs["capture_output"] + return type("Result", (), {"returncode": 0, "stdout": b"registered"})() + + monkeypatch.setattr(specialize.subprocess, "run", fake_run) + result = specialize._run_with_input(["docker", "run", "fixed"], b"sensitive-runner-token\n") + + assert result == "registered" + assert observed["args"] == ["docker", "run", "fixed"] + assert observed["input"] == b"sensitive-runner-token\n" + assert "sensitive-runner-token" not in " ".join(observed["args"]) + + +def test_build_probe_create_command_is_fixed_and_has_no_mount_or_shared_socket(specialize): + record = _build_probe_record(specialize) + args = specialize._build_probe_create_args(record) + + assert args[:8] == [ + "create", + "--name", + record["name"], + "--restart=no", + "--cgroup-parent=docker", + "--network=bridge", + "--cgroupns=private", + "--label", + ] + assert record["taskImageId"] in args + assert "--cpus=4" in args + assert "--memory=6144m" in args + assert "--pids-limit=1024" in args + assert "--blkio-weight=100" in args + assert "--security-opt=seccomp=unconfined" in args + assert "--security-opt=systempaths=unconfined" in args + assert "--device=/dev/net/tun:/dev/net/tun:rwm" in args + assert "--entrypoint" in args + assert specialize._BUILD_PROBE_EXECUTABLE in args + assert record["sourceRevision"] in args + assert record["sourceTree"] in args + assert not any( + value == "--privileged" + or value == "-e" + or value.startswith(("--cap-add", "--volume", "--mount", "--env")) + for value in args + ) + assert not any("docker.sock" in value or "host.docker.internal" in value for value in args) + + +def test_build_probe_creation_inspect_requires_stopped_zero_pid_and_exact_inherited_labels( + specialize, +): + record = _build_probe_record(specialize, container_id="e" * 64, status="created") + item, environment = _build_probe_inspect(specialize, record) + item["State"] = {"Running": False, "Pid": 0} + + assert specialize._validate_build_probe_container( + item, + record, + image_environment=environment, + require_running=False, + require_created_stopped=True, + ) + labels = specialize._build_probe_container_labels(record) + assert set(specialize.PINNED_TASK_BASE_INHERITED_LABELS).issubset(labels) + assert record["imageLabels"].items() <= labels.items() + assert item["Config"]["Labels"] == labels + + running, _ = _build_probe_inspect(specialize, record) + assert not specialize._validate_build_probe_container( + running, + record, + image_environment=environment, + require_running=False, + require_created_stopped=True, + ) + bad_pid = copy.deepcopy(item) + bad_pid["State"]["Pid"] = 9 + assert not specialize._validate_build_probe_container( + bad_pid, + record, + image_environment=environment, + require_running=False, + require_created_stopped=True, + ) + foreign_label = copy.deepcopy(item) + foreign_label["Config"]["Labels"]["untrusted.extra"] = "true" + assert not specialize._validate_build_probe_container( + foreign_label, + record, + image_environment=environment, + require_running=False, + require_created_stopped=True, + ) + + +def test_build_probe_container_inspect_accepts_only_fixed_private_build_job(specialize): + record = _build_probe_record(specialize, container_id="e" * 64, status="created") + item, environment = _build_probe_inspect(specialize, record) + + assert specialize._validate_build_probe_container(item, record, image_environment=environment) + for section, changes in ( + ("HostConfig", {"Privileged": True}), + ("HostConfig", {"PidMode": "host"}), + ("HostConfig", {"NetworkMode": "host"}), + ("HostConfig", {"CapAdd": ["SYS_ADMIN"]}), + ("HostConfig", {"Devices": [ + *item["HostConfig"]["Devices"], + {"PathOnHost": "/dev/sda", "PathInContainer": "/dev/sda", "CgroupPermissions": "rwm"}, + ]}), + ("HostConfig", {"Mounts": [{"Type": "bind", "Source": "/host", "Target": "/host"}]}), + ("HostConfig", {"SecurityOpt": ["seccomp=unconfined"]}), + ("Config", {"Env": [*environment, "DOCKER_HOST=tcp://shared-dind:2375"]}), + ("Config", {"Image": "localhost/itworx-ci-build:latest"}), + ): + changed, expected_environment = _build_probe_inspect(specialize, record) + changed[section].update(changes) + assert not specialize._validate_build_probe_container( + changed, + record, + image_environment=expected_environment, + ) + + +@pytest.mark.parametrize("field,value", [ + ("Type", "bind"), ("Name", "persistent-builder-cache"), + ("Source", "/var/run/docker.sock"), ("Destination", "/host"), + ("Driver", "nfs"), ("RW", False), ("Propagation", "rshared"), +]) +def test_build_probe_rejects_foreign_or_shared_data_mount(specialize, field, value): + record = _build_probe_record(specialize, container_id="e" * 64, status="created") + item, environment = _build_probe_inspect(specialize, record) + item["Mounts"][0][field] = value + assert not specialize._validate_build_probe_container(item, record, image_environment=environment) + + +def test_build_probe_volume_identity_detects_same_name_recreation(specialize, monkeypatch): + name = "9" * 64 + volume = { + "Name": name, "Driver": "local", "Scope": "local", + "Mountpoint": f"/var/lib/docker/volumes/{name}/_data", + "Labels": {"com.docker.volume.anonymous": ""}, "Options": None, "CreatedAt": "2026-10-03T00:00:00Z", + } + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: json.dumps([volume])) + identity = specialize._attest_build_probe_data_volume(name) + volume["Labels"]["foreign.owner"] = "another-job" + with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): + specialize._attest_build_probe_data_volume(name, identity) + volume["Labels"] = {"com.docker.volume.anonymous": ""} + volume["CreatedAt"] = "2026-10-03T00:00:01Z" + with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): + specialize._attest_build_probe_data_volume(name, identity) + + +def test_build_probe_volume_recovery_never_removes_volume_used_by_another_container(specialize, monkeypatch): + identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} + mutations = [] + + def run(*args, **kwargs): + if args[:2] == ("volume", "ls"): + return identity["name"] + if args[:2] == ("ps", "-a"): + return "f" * 64 + mutations.append(args) + raise AssertionError("unexpected volume mutation") + + monkeypatch.setattr(specialize, "_run_dind_docker", run) + monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_in_use"): + specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) + assert mutations == [] + + +def test_build_probe_volume_recovery_removes_only_exact_journal_owned_volume(specialize, monkeypatch): + identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} + mutations = [] + present = True + + def run(*args, **kwargs): + nonlocal present + if args[:2] == ("volume", "ls"): + assert args == ("volume", "ls", "--quiet", "--filter", f"name={identity['name']}") + return identity["name"] if present else "" + if args[:2] == ("ps", "-a"): + return "" + assert args == ("volume", "rm", identity["name"]) + mutations.append(args) + present = False + return identity["name"] + + monkeypatch.setattr(specialize, "_run_dind_docker", run) + monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) + specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) + assert mutations == [("volume", "rm", identity["name"])] + + +def test_build_probe_result_requires_exact_marker_and_every_rootless_tool_check(specialize): + record = _build_probe_record(specialize) + payload = { + "schema": specialize._BUILD_PROBE_SCHEMA, + "status": "passed", + "sourceRevision": record["sourceRevision"], + "sourceTree": record["sourceTree"], + "engineVersion": "29.8.2", + "engineRootless": True, + "engineStorageDriver": "overlay2", + "engineCgroupDriver": "none", + "engineSocket": "unix:///run/itworx-job-docker/docker.sock", + "engineSocketUid": 1000, + "composeVersion": "5.5.1", + "buildRun": True, + "composeHealthy": True, + "composeDnsResolved": True, + "archiveNormalized": True, + "archiveLoaded": True, + "imageId": "sha256:" + "2" * 64, + "imageLabels": record["imageLabels"], + "archiveSha256": "sha256:" + "3" * 64, + "holdSeconds": 30, + } + line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ) + + assert specialize._validate_build_probe_result(line, record=record) == payload + with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_missing"): + specialize._validate_build_probe_result(line + "\n" + line, record=record) + for key, value in ( + ("engineRootless", False), + ("engineSocket", "tcp://0.0.0.0:2375"), + ("composeHealthy", False), + ("holdSeconds", 0), + ): + changed = {**payload, key: value} + changed_line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( + changed, + sort_keys=True, + separators=(",", ":"), + ) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_untrusted"): + specialize._validate_build_probe_result(changed_line, record=record) + + +def test_build_probe_live_process_inventory_requires_daemon_and_compose_children(specialize): + output = "\n".join( + ( + "PID PPID COMMAND", + "1 0 python3 /usr/local/bin/itworx-ci-job-docker-probe", + "10 1 rootlesskit --net=slirp4netns", + "11 10 dockerd --host=unix:///run/itworx-job-docker/docker.sock --storage-driver=overlay2", + "12 11 containerd --config /run/containerd/config.toml", + "21 20 sleep 300", + ) + ) + + result = specialize._validate_build_probe_processes(output) + + assert result == { + "processCount": 5, + "probe": True, + "rootlesskit": True, + "dockerd": True, + "containerd": True, + "composeService": True, + } + with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): + specialize._validate_build_probe_processes( + output.replace("21 20 sleep 300", "21 20 cat /etc/os-release") + ) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): + specialize._validate_build_probe_processes(output.replace("containerd --config", "unknown --config")) + + +def test_build_probe_intent_binds_revision_tree_image_nonce_and_exact_container_id(specialize): + record = _build_probe_record(specialize) + + assert specialize._validated_build_probe_record( + record, + source_revision="a" * 40, + source_tree="b" * 40, + ) == record + invalid = dict(record, name=record["name"] + "-foreign") + with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): + specialize._validated_build_probe_record( + invalid, + source_revision="a" * 40, + source_tree="b" * 40, + ) + + +def _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, ids): + image_environment = item["Config"]["Env"] + present = list(ids) + removed = [] + monkeypatch.setattr( + specialize, + "_inspect", + lambda name: {"State": {"Running": True}} if name == specialize.DIND_NAME else None, + ) + monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) + monkeypatch.setattr( + specialize, + "_build_probe_ids", + lambda *, label=True, name=None: list(present) + if label or name == record["name"] and present + else [], + ) + monkeypatch.setattr(specialize, "_inspect_dind_container", lambda _cid: item) + monkeypatch.setattr( + specialize, + "_build_probe_image_environment", + lambda _record: image_environment, + ) + + def run_dind(*args, **_kwargs): + if args[:3] == ("rm", "--force", "--volumes"): + removed.append(args[3]) + present.clear() + return args[3] + if args[:2] == ("volume", "ls"): + return "" + raise AssertionError(f"unexpected private-DinD command: {args}") + + monkeypatch.setattr(specialize, "_run_dind_docker", run_dind) + monkeypatch.setattr( + specialize, "_attest_build_probe_data_volume", + lambda name, expected=None: {"name": name, "createdAt": "2026-10-03T00:00:00Z"}, + ) + monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) + return removed + + +def test_build_probe_recovery_cleans_only_intent_owned_container_after_create_crash( + specialize, + monkeypatch, +): + record = _build_probe_record(specialize, status="intent") + cid = "e" * 64 + item, _environment = _build_probe_inspect(specialize, record, container_id=cid) + removed = _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, [cid]) + journal = { + "sourceRevision": record["sourceRevision"], + "sourceTree": record["sourceTree"], + "buildProbe": record, + } + + specialize._cleanup_build_probe(journal) + + assert removed == [cid] + assert journal["buildProbe"]["containerId"] == cid + assert journal["buildProbe"]["status"] == "cleaned" + + +def test_build_probe_recovery_refuses_foreign_cid_even_with_matching_name_and_nonce( + specialize, + monkeypatch, +): + expected_cid = "d" * 64 + foreign_cid = "e" * 64 + record = _build_probe_record( + specialize, + container_id=expected_cid, + status="started", + ) + item, _environment = _build_probe_inspect(specialize, record, container_id=foreign_cid) + removed = _install_build_probe_cleanup_fakes( + specialize, + monkeypatch, + record, + item, + [foreign_cid], + ) + journal = { + "sourceRevision": record["sourceRevision"], + "sourceTree": record["sourceTree"], + "buildProbe": record, + } + + with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_container_untrusted"): + specialize._cleanup_build_probe(journal) + + assert removed == [] + + +def test_build_probe_recovery_refuses_ambiguous_owned_label_inventory( + specialize, + monkeypatch, +): + record = _build_probe_record(specialize, status="intent") + cid = "e" * 64 + item, _environment = _build_probe_inspect(specialize, record, container_id=cid) + removed = _install_build_probe_cleanup_fakes( + specialize, + monkeypatch, + record, + item, + [cid, "f" * 64], + ) + journal = { + "sourceRevision": record["sourceRevision"], + "sourceTree": record["sourceTree"], + "buildProbe": record, + } + + with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_ambiguous_containers"): + specialize._cleanup_build_probe(journal) + + assert removed == [] + invalid = dict(record, status="started") + with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): + specialize._validated_build_probe_record( + invalid, + source_revision="a" * 40, + source_tree="b" * 40, + ) + + +@pytest.mark.parametrize("recovering_cgroup_intent", [False, True]) +@pytest.mark.parametrize("probe_fails", [False, True]) +def test_specialization_stage_fixture_quiesces_old_before_registering_and_never_runs_over_two( + specialize, tmp_path, monkeypatch, recovering_cgroup_intent, probe_fails +): + revision = "a" * 40 + tree = "b" * 40 + manifest_sha = "d" * 64 + manifest = { + "images": [ + {"profile": profile_id, "reference": specialize._task_image_reference(profile_id, revision)} + for profile_id in specialize.PROFILE_IDS + ] + } + archive = tmp_path / "task-images.tar" + archive.write_bytes(b"fixture") + journal = { + "status": "in_progress", + "stage": "prepared", + "legacyTopology": "v1", + "sourceRevision": revision, + "sourceTree": tree, + "cgroupCreated": False, + "cgroupEnsurePending": recovering_cgroup_intent, + "baseComposeSha256": hashlib.sha256(b"captured-base").hexdigest(), + "baseComposeModelSha256": hashlib.sha256(b"base-model").hexdigest(), + "specializedComposeSha256": hashlib.sha256(b"specialized-model").hexdigest(), + "backupDirectory": str(tmp_path / "specialized-test"), + "newRegistrationIds": {}, + } + events: list[str] = [] + running: set[str] = set() + maximum_running = 0 + marker_active = {"value": False} + + def rows(): + return [ + { + "name": specialize.RUNNER_PROFILES[profile_id].container_name, + "id": index + 1, + "labels": sorted( + { + specialize.RUNNER_PROFILES[profile_id].runner_label, + *specialize.RUNNER_PROFILES[profile_id].legacy_labels, + } + ), + "disabled": True, + "busy": False, + } + for index, profile_id in enumerate(specialize.PROFILE_IDS) + ] + + def inventory(): + return ( + { + name: {"name": name, "State": {"Running": name in running}} + for name in (*specialize._SPECIALIZED_NAMES, specialize.DIND_NAME) + }, + [], + ) + + def resource_proof(_claims, _containers, _legacy): + return { + "ready": True, + "digest": "e" * 64, + "runningRunnerContainers": len(running), + } + + monkeypatch.setattr(specialize, "_captured_base_source", lambda _journal: b"captured-base") + monkeypatch.setattr( + specialize, + "_captured_compose_model", + lambda _journal, *, specialized: b"specialized-model" if specialized else b"base-model", + ) + monkeypatch.setattr(specialize, "_read_source_file", lambda *_args, **_kwargs: b"captured-base") + monkeypatch.setattr(specialize, "_validate_base_composition", lambda _bytes=None: "v1") + monkeypatch.setattr(specialize, "_validate_composition", lambda raw: raw) + monkeypatch.setattr(specialize, "_run_specialized_compose", lambda _journal, *args, **_kwargs: events.append(f"compose:{args[-1]}")) + monkeypatch.setattr(specialize, "_wait_guards_quiet", lambda: events.append("guards-quiet")) + monkeypatch.setattr(specialize, "_service_guard_scripts", lambda _journal: events.append("guards-fenced")) + monkeypatch.setattr(specialize, "_stop_legacy_capacity_daemon", lambda: events.append("capacity-stopped")) + monkeypatch.setattr(specialize, "_set_legacy_disabled", lambda _journal: events.append("legacy-disabled")) + monkeypatch.setattr(specialize, "_check_legacy_queued_jobs", lambda: None) + monkeypatch.setattr(specialize, "_stop_remove_legacy_containers", lambda _journal: events.append("legacy-removed")) + monkeypatch.setattr( + specialize, + "_cgroup_family_snapshot", + lambda: { + "exists": recovering_cgroup_intent, + "rootSubtreeControllers": ["cpu", "memory", "pids"], + }, + ) + + def ensure_cgroup_family(**kwargs): + assert kwargs["allow_existing"] is recovering_cgroup_intent + events.append("cgroup-ensured") + return not recovering_cgroup_intent + + monkeypatch.setattr( + specialize, + "_ensure_cgroup_family", + ensure_cgroup_family, + ) + monkeypatch.setattr(specialize, "_prepare_profile_state", lambda _revision: None) + monkeypatch.setattr(specialize, "_write_private_atomic", lambda *_args, **_kwargs: None) + + def register(current_journal): + assert "legacy-disabled" in events + events.append("registrations-created") + current_journal["newRegistrationIds"] = { + profile_id: index + 1 for index, profile_id in enumerate(specialize.PROFILE_IDS) + } + specialize._set_stage(current_journal, "registrations_created") + + monkeypatch.setattr(specialize, "_register_profiles", register) + monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") + monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: None) + monkeypatch.setattr(specialize, "_inspect", lambda _name: {"State": {"Running": True}}) + monkeypatch.setattr(specialize, "_require_dind_resource_support", lambda _model: None) + monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {"services": {}}) + monkeypatch.setattr(specialize, "_load_images_into_dind", lambda *_args: None) + monkeypatch.setattr(specialize, "_verify_manifest_images_in_dind", lambda _items: True) + monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) + monkeypatch.setattr(specialize, "_resource_proof", resource_proof) + monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) + monkeypatch.setattr( + specialize, + "_trusted_private_file", + lambda path, **_kwargs: specialize._render_runner_config("build", revision) + if str(path).endswith("build.yaml") + else b"fixture", + ) + monkeypatch.setattr(specialize, "_cleanup_build_probe", lambda _journal: None) + + def run_build_probe(_journal, _manifest): + assert not running + events.append("build-probed") + if probe_fails: + raise specialize.RunnerFleetSpecializeError( + "runner_fleet_specialization_build_probe_failed" + ) + + monkeypatch.setattr(specialize, "_run_build_probe", run_build_probe) + monkeypatch.setattr(specialize, "_runner_rows", rows) + def write_journal(current_journal): + if current_journal.get("cgroupEnsurePending") is True: + events.append("cgroup-intent-durable") + if current_journal.get("cgroupCreated") is True: + events.append("cgroup-result-durable") + if current_journal.get("status") == "active": + events.append("journal-active-durable") + + monkeypatch.setattr(specialize, "_write_journal", write_journal) + + batches: list[tuple[str, ...]] = [] + + def set_containers(profiles, start): + nonlocal maximum_running + names = {specialize.RUNNER_PROFILES[item].container_name for item in profiles} + if start: + assert len(running | names) <= 2 + running.update(names) + maximum_running = max(maximum_running, len(running)) + batches.append(tuple(profiles)) + events.append("batch-started") + else: + running.difference_update(names) + events.append("batch-stopped") + + monkeypatch.setattr(specialize, "_set_runner_containers_running", set_containers) + monkeypatch.setattr(specialize, "_bootstrap_registrations", lambda: [{"name": name} for name in specialize._SPECIALIZED_NAMES]) + monkeypatch.setattr(specialize, "_read_installed_manifest_sha256", lambda: manifest_sha) + monkeypatch.setattr(specialize, "_write_pending_bootstrap_receipt", lambda *_args: events.append("receipt-written")) + + crash = {"next_active_write": True} + + class SimulatedPowerLoss(BaseException): + pass + + def write_marker(**kwargs): + assert kwargs["status"] == "active" + assert not running + assert "receipt-written" in events + events.append("marker-active-attempt") + if crash["next_active_write"]: + crash["next_active_write"] = False + raise SimulatedPowerLoss + marker_active["value"] = True + events.append("marker-active") + + monkeypatch.setattr(specialize, "_write_marker", write_marker) + monkeypatch.setattr(specialize, "specialization_state", lambda: "active" if marker_active["value"] else "blocked") + + if probe_fails: + monkeypatch.setattr(specialize, "_rollback_before_activation", lambda _journal: True) + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="failed_rolled_back", + ): + specialize._perform_migration_stages(journal, manifest, archive) + assert "build-probed" in events + assert "receipt-written" not in events + assert "marker-active-attempt" not in events + assert not marker_active["value"] + return + + with pytest.raises(SimulatedPowerLoss): + specialize._perform_migration_stages(journal, manifest, archive) + + assert journal["status"] == "active" + assert not marker_active["value"] + assert events.index("journal-active-durable") < events.index("marker-active-attempt") + + marker = { + "status": "in_progress", + "sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256, + "imageSourceRevision": revision, + "imageSourceTree": tree, + } + script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() + monkeypatch.setattr(specialize, "_read_json_file", lambda *_args, **_kwargs: marker) + monkeypatch.setattr(specialize, "_valid_marker", lambda _marker: True) + monkeypatch.setattr(specialize, "_migration_lock", lambda: nullcontext()) + monkeypatch.setattr( + specialize, + "verified_topology", + lambda: { + "ready": marker_active["value"], + "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if marker_active["value"] else [], + "resourceEnvelope": {"ready": marker_active["value"], "digest": "e" * 64}, + }, + ) + monkeypatch.setitem( + sys.modules, + "status_server", + type( + "StatusServerStub", + (), + { + "_require_controller_release": staticmethod( + lambda: { + "sourceRepository": specialize.SOURCE_REPOSITORY, + "sourceRevision": revision, + "sourceTree": tree, + } + ) + }, + ), + ) + monkeypatch.setattr( + specialize, + "_run_approved_shell_script", + lambda _script: specialize._perform_migration_stages(journal, manifest, archive), + ) + result = specialize.execute(script) + assert result["status"] == "specialized" + + assert maximum_running == 2 + assert events.index("cgroup-intent-durable") < events.index("cgroup-ensured") + assert events.index("cgroup-ensured") < events.index("cgroup-result-durable") + assert batches == [ + tuple(specialize.PROFILE_IDS[0:2]), + tuple(specialize.PROFILE_IDS[2:4]), + tuple(specialize.PROFILE_IDS[4:6]), + ] + assert events.index("legacy-disabled") < events.index("registrations-created") + assert events.index("batch-stopped") < events.index("build-probed") + assert events.index("build-probed") < events.index("receipt-written") + assert events.index("receipt-written") < events.index("marker-active") + assert journal["cgroupCreated"] is True + assert journal["status"] == "active" + assert not running + + +@pytest.mark.parametrize("original_running", [False, True]) +def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( + specialize, tmp_path, monkeypatch, original_running +): + image_id = specialize._DIND_IMAGE_ID + base_model = _base_v1_compose_model(specialize) + specialized_model = { + "services": { + "runner-dind": { + **base_model["services"]["runner-dind"], + "cgroup_parent": "itworx-ci", + "restart": "no", + "volumes": [ + { + "type": "bind", + "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), + "target": "/var/lib/docker", + "read_only": False, + } + ], + } + } + } + journal = { + "originalDind": { + "present": True, + "running": original_running, + "restartPolicy": "unless-stopped", + "imageId": image_id, + }, + "originalContainers": [], + "originalRegistrations": [], + "newRegistrationIds": {}, + "guardFiles": [], + "cgroupCreated": True, + "cgroupEnsurePending": False, + "cgroupRootSubtree": ["cpu", "memory", "pids"], + "backupDirectory": str(tmp_path / "specialized-test"), + } + events = [] + current = {"kind": "specialized", "State": {"Running": True}} + current_marker = {"value": current} + + def inspect(name): + assert name == specialize.DIND_NAME or name in specialize._SPECIALIZED_NAMES + return current_marker["value"] if name == specialize.DIND_NAME else None + + def run(args, **_kwargs): + events.append(tuple(args)) + if args[:2] == ["docker", "update"] and args[-1] == specialize.DIND_NAME: + if args[2] == "--restart=no": + return "" + current_marker["value"]["HostConfig"]["RestartPolicy"]["Name"] = args[2].partition("=")[2] + return "" + if args[:2] == ["docker", "stop"]: + current_marker["value"]["State"]["Running"] = False + return "" + if args[:2] == ["docker", "rm"]: + current_marker["value"] = None + return "" + return "" + + def run_compose(_compose, *args, **_kwargs): + events.append(("compose", *args)) + assert args[-1] == "runner-dind" + restored_running = original_running if args[0] == "up" else False + current_marker["value"] = _legacy_dind_fixture( + specialize, + base_model, + image_id=image_id, + running=restored_running, + restart_policy="no", + ) + + base_bytes = json.dumps(base_model, sort_keys=True, separators=(",", ":")).encode() + specialized_bytes = json.dumps(specialized_model, sort_keys=True, separators=(",", ":")).encode() + monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) + monkeypatch.setattr( + specialize, + "_captured_compose_model", + lambda _journal, *, specialized: specialized_bytes if specialized else base_bytes, + ) + monkeypatch.setattr( + specialize, + "_validate_dind_outer", + lambda item, _model: item.get("kind") == "specialized", + ) + monkeypatch.setattr(specialize, "_inspect", inspect) + monkeypatch.setattr(specialize, "_run", run) + monkeypatch.setattr(specialize, "_run_compose", run_compose) + monkeypatch.setattr(specialize, "_runner_rows", lambda: []) + monkeypatch.setattr( + specialize, + "rollback_host_cgroup_family", + lambda **_kwargs: events.append("cgroup-cleanup") or {"familyExists": False}, + ) + monkeypatch.setattr(specialize, "SPECIALIZED_COMPOSE_PATH", tmp_path / "specialized-compose.yml") + monkeypatch.setattr(specialize, "TASK_IMAGE_MANIFEST_PATH", tmp_path / "task-image-manifest.json") + monkeypatch.setattr(specialize, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "receipt.json") + monkeypatch.setattr(specialize, "RUNNER_CONFIG_ROOT", tmp_path / "runner-configs") + monkeypatch.setattr(specialize, "_SPECIALIZED_DATA_ROOT", tmp_path / "runner-data") + monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", tmp_path / "workspaces") + monkeypatch.setattr(specialize, "TASK_IMAGE_ROOT", tmp_path / "task-images") + monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", tmp_path / "specialized.json") + monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", tmp_path / "journal.json") + + assert specialize._rollback_before_activation(journal) is True + + restored = current_marker["value"] + assert restored["Image"] == image_id + assert restored["State"]["Running"] is original_running + assert restored["HostConfig"]["RestartPolicy"]["Name"] == "unless-stopped" + compose = next(event for event in events if isinstance(event, tuple) and event[0] == "compose") + assert compose[1] == ("up" if original_running else "create") + remove = next(index for index, event in enumerate(events) if isinstance(event, tuple) and event[:2] == ("docker", "rm")) + assert remove < events.index(compose) < events.index("cgroup-cleanup") + + +def _legacy_dind_fixture( + specialize, base_model, *, image_id, running, restart_policy +): + service = base_model["services"]["runner-dind"] + return { + "Name": f"/{specialize.DIND_NAME}", + "Image": image_id, + "Config": { + "Image": service["image"], + "User": "rootless", + "Entrypoint": ["dockerd-entrypoint.sh"], + "Cmd": ["--default-address-pool=base=198.19.0.0/16,size=27"], + "Volumes": {"/home/rootless/.local/share/docker": {}, "/var/lib/docker": {}}, + "Labels": { + "com.docker.compose.project": specialize.PROJECT, + "com.docker.compose.service": "runner-dind", + }, + }, + "State": {"Running": running}, + "HostConfig": { + "NanoCpus": 6_000_000_000, + "Memory": 12 * 1024**3, + "PidMode": "", + "Privileged": True, + "RestartPolicy": {"Name": restart_policy}, + }, + "Mounts": [ + { + "Type": "bind", + "Source": mount["source"], + "Destination": mount["target"], + "RW": not mount.get("read_only", False), + } + for mount in service["volumes"] + ] + [{ + "Type": "volume", "Driver": "local", "Name": "f" * 64, + "Source": "/var/lib/docker/volumes/" + "f" * 64 + "/_data", + "Destination": "/var/lib/docker", "RW": True, + }], + } + + +def test_known_v1_and_v2_topologies_are_explicit_and_v1_does_not_require_absent_v2_services( + specialize, monkeypatch +): + compose = b"known Tower v1 compose fixture" + monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest()) + v1_services = { + service_name: {"container_name": name} + for name, service_name in specialize._OLD_SERVICES.items() + if name in specialize._LEGACY_V1_NAMES + } + assert specialize._classify_legacy_topology(compose, v1_services) == "v1" + + v2_services = {} + for name in specialize._LEGACY_V2_NAMES: + service_name = specialize._OLD_SERVICES[name] + burst = name.endswith("burst-1") + v2_services[service_name] = { + "container_name": name, + "image": specialize.RUNNER_IMAGE, + "profiles": ["runner"], + "labels": { + "tech.itworx.role": "gitea-actions-burst-runner" if burst else "gitea-actions-runner" + }, + **({"cpus": 0.5, "mem_limit": "512m"} if name in specialize._LEGACY_V2_NAMES[2:] else {}), + } + assert specialize._classify_legacy_topology(b"known v2 compose", v2_services) == "v2" + + partial_v2 = dict(v1_services) + partial_v2["runner-3"] = v2_services["runner-3"] + with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_compose_invalid"): + specialize._classify_legacy_topology(compose, partial_v2) + + +def test_exact_live_v1_registration_and_container_contract_accepts_only_two_known_runners(specialize): + rows = [ + { + "name": name, + "id": runner_id, + "labels": sorted(specialize._OLD_LABELS[name]), + "disabled": False, + "busy": False, + } + for runner_id, name in zip((9, 22), specialize._LEGACY_V1_NAMES, strict=True) + ] + rows.extend( + [ + {"name": "itworx-unraid-deploy", "id": 13, "labels": ["unraid-deploy"], "disabled": False, "busy": False}, + {"name": "itworx-windows11-native", "id": 14, "labels": ["windows-native"], "disabled": False, "busy": False}, + ] + ) + assert len(specialize._validate_legacy_registrations(rows, "v1")) == 2 + + name = "itworx-security-runner" + mount_entries = [ + {"Source": source, "Destination": target, "RW": True} + for source, target, _read_only in specialize._V1_RUNNER_VOLUMES[name] + ] + item = { + "Image": specialize.RUNNER_IMAGE_ID, + "Config": { + "Image": specialize.RUNNER_IMAGE, + "Labels": { + "com.docker.compose.project": specialize.PROJECT, + "com.docker.compose.service": "runner", + "tech.itworx.role": "gitea-actions-runner", + }, + }, + "State": {"Running": True}, + "HostConfig": { + "NanoCpus": 1_000_000_000, + "Memory": 1024 * specialize._MIB, + "PidMode": "", + "RestartPolicy": {"Name": "unless-stopped"}, + }, + "Mounts": mount_entries, + } + assert specialize._validate_legacy_container_item("v1", name, item)["running"] is True + + item["HostConfig"]["PidMode"] = "host" + with pytest.raises(specialize.RunnerFleetSpecializeError, match="v1_container_budget_invalid"): + specialize._validate_legacy_container_item("v1", name, item) + + extra_known_runner = [*rows, {"name": "itworx-validation-runner-3", "id": 30}] + with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_registration_invalid"): + specialize._validate_legacy_registrations(extra_known_runner, "v1") + + +def test_queue_gate_allows_only_legacy_labels_covered_by_new_profiles(specialize): + assert specialize._queued_job_is_covered(["self-hosted", "linux", "linux-validation", "ubuntu-latest"]) + assert specialize._queued_job_is_covered(["itworx-ci-backend"]) + assert specialize._queued_job_is_covered(["docker-validation"]) + assert not specialize._queued_job_is_covered(["mcp-validation", "unraid"]) + + +def _legacy_queue_job(job_id, status, labels, *, run_id=21500): + return {"id": job_id, "run_id": run_id, "status": status, "labels": labels} + + +def test_legacy_queue_gate_reads_queued_and_requested_admin_inventories(specialize, monkeypatch): + calls = [] + + def api(path): + calls.append(path) + if "status=pending" in path: + return { + "total_count": 1, + "jobs": [_legacy_queue_job(102, "requested", ["linux-validation", "ubuntu-latest"])], + } + if "status=queued" in path: + return { + "total_count": 1, + "jobs": [_legacy_queue_job(101, "queued", ["itworx-ci-backend"])], + } + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + specialize._check_legacy_queued_jobs() + + assert len(calls) == 2 + assert "status=pending&limit=100&page=1&sort=id&order=asc" in calls[0] + assert "status=queued&limit=100&page=1&sort=id&order=asc" in calls[1] + + +def test_legacy_queue_gate_blocks_unsupported_requested_dependency_job(specialize, monkeypatch): + def api(path): + if "status=queued" in path: + return {"total_count": 0, "jobs": []} + if "status=pending" in path: + return { + "total_count": 1, + "jobs": [_legacy_queue_job(201, "requested", ["mcp-validation", "unraid"])], + } + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_legacy_queue_present", + ): + specialize._check_legacy_queued_jobs() + + +def test_legacy_queue_gate_catches_requested_to_queued_transition_between_filters( + specialize, monkeypatch +): + state = "requested" + calls = [] + + def api(path): + nonlocal state + calls.append(path) + if "status=pending" in path: + # The job changes status immediately before this filtered read. + state = "queued" + return {"total_count": 0, "jobs": []} + if "status=queued" in path: + jobs = ( + [_legacy_queue_job(250, "queued", ["mcp-validation", "unraid"])] + if state == "queued" + else [] + ) + return {"total_count": len(jobs), "jobs": jobs} + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_legacy_queue_present", + ): + specialize._check_legacy_queued_jobs() + + assert "status=pending" in calls[0] + assert "status=queued" in calls[1] + + +def test_legacy_queue_gate_paginates_and_blocks_unsupported_page_two_job(specialize, monkeypatch): + calls = [] + + def api(path): + calls.append(path) + if "status=queued" in path: + page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) + if page == 1: + return { + "total_count": 101, + "jobs": [ + _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) + for index in range(100) + ], + } + return { + "total_count": 101, + "jobs": [_legacy_queue_job(101, "queued", ["mcp-validation", "unraid"])], + } + if "status=pending" in path: + return {"total_count": 0, "jobs": []} + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_legacy_queue_present", + ): + specialize._check_legacy_queued_jobs() + + assert any("status=queued&limit=100&page=2" in path for path in calls) + + +def test_legacy_queue_gate_rejects_total_count_changes_between_pages(specialize, monkeypatch): + def api(path): + if "status=queued" in path: + page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) + if page == 1: + return { + "total_count": 101, + "jobs": [ + _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) + for index in range(100) + ], + } + return { + "total_count": 102, + "jobs": [_legacy_queue_job(101, "queued", ["itworx-ci-backend"])], + } + if "status=pending" in path: + return {"total_count": 0, "jobs": []} + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_legacy_queue_untrusted", + ): + specialize._check_legacy_queued_jobs() + + +@pytest.mark.parametrize( + "payload", + [ + {"total_count": 1, "jobs": []}, + {"total_count": 1, "jobs": [_legacy_queue_job(1, "requested", ["itworx-ci-backend"])]}, + {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", [""])]}, + {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"], run_id=0)]}, + {"total_count": 2, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"])]}, + { + "total_count": 2, + "jobs": [ + _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), + _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), + ], + }, + { + "total_count": 2, + "jobs": [ + _legacy_queue_job(2, "queued", ["itworx-ci-backend"]), + _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), + ], + }, + ], +) +def test_legacy_queue_gate_rejects_malformed_or_truncated_inventory(specialize, monkeypatch, payload): + def api(path): + if "status=queued" in path: + return payload + if "status=pending" in path: + return {"total_count": 0, "jobs": []} + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_legacy_queue_untrusted", + ): + specialize._check_legacy_queued_jobs() + + +def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specialize, monkeypatch): + row = _legacy_queue_job(301, "queued", ["itworx-ci-backend"]) + + def api(path): + if "status=queued" in path: + return {"total_count": 1, "jobs": [row]} + if "status=pending" in path: + return { + "total_count": 1, + "jobs": [{**row, "status": "requested"}], + } + raise AssertionError(f"unexpected fixed Gitea API path: {path}") + + monkeypatch.setattr(specialize, "_api_json", api) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_legacy_queue_untrusted", + ): + specialize._check_legacy_queued_jobs() + + +def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url(): + workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8") + build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8") + frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8") + + assert "github-server-url: http://gitea-runner-internal:3000" in workflow + assert "bash scripts/build_ci_task_images.sh" in workflow + assert "jens/unraid-appops-gateway" in build + assert "Jens/Unraid-AppOps-Gateway" in build + assert "corepack prepare pnpm@11.15.0 --activate" in frontend + assert "test \"$(pnpm --version)\" = 11.15.0" in frontend + + +def test_specialization_shell_entrypoint_is_fixed_and_has_no_caller_selected_target(): + script = ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh" + content = script.read_text(encoding="utf-8") + + assert content.startswith("#!/usr/bin/env bash\n") + assert "SPECIALIZE_CI_RUNNER_FLEET" in content + assert "--apply-fixed" in content + assert "eval " not in content + assert "docker run" not in content + assert "docker compose" not in content + + +def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity( + specialize, monkeypatch +): + registrations = [ + { + "name": profile.display_name, + "id": index, + "labels": [profile.runner_label, *profile.legacy_labels], + "disabled": True, + "busy": False, + } + for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=10) + ] + monkeypatch.setattr(specialize, "_api_json", lambda _path: {"runners": registrations}) + + normalized = specialize._runner_rows() + assert [row["name"] for row in normalized] == list(specialize._SPECIALIZED_NAMES) + assert [row["displayName"] for row in normalized] == [ + profile.display_name for profile in specialize.RUNNER_PROFILES.values() + ] + + registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].container_name + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="registration_name_invalid", + ): + specialize._runner_rows() + + registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].display_name + registrations[1]["name"] = specialize.RUNNER_PROFILES["checks"].display_name + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="registration_duplicate", + ): + specialize._runner_rows() + + +def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): + rows, containers, dind, journal, model = _active_recovery_fixture(specialize) + events = [] + + class Volume: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def assert_current(self): + events.append("volume-current") + + monkeypatch.setattr(specialize, "_migration_lock", lambda **_kwargs: nullcontext()) + monkeypatch.setattr(specialize, "_open_fixed_release_volume", lambda: Volume()) + monkeypatch.setattr( + specialize, + "_active_release_bundle", + lambda: ({"status": "active"}, journal, model, b"captured-compose"), + ) + monkeypatch.setattr(specialize, "_runner_rows", lambda: rows) + monkeypatch.setattr(specialize, "_docker_outer_inventory", lambda: (containers, [])) + monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) + monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) + monkeypatch.setattr( + specialize, + "_cgroup_family_snapshot", + lambda: { + "exists": family_exists, + "safeToStartDind": True, + "rootSubtreeControllers": ["cpu", "memory", "pids"], + }, + ) + monkeypatch.setattr( + specialize, + "_fresh_recovery_host_gate", + lambda: events.append("fresh-host-gate"), + ) + monkeypatch.setattr( + specialize, + "_ensure_cgroup_family_details", + lambda **_kwargs: events.append("cgroup-create") + or {"created": True, "rootSubtreeBefore": ["cpu", "memory", "pids"]}, + ) + monkeypatch.setattr( + specialize, + "_run_compose", + lambda compose, *args, **kwargs: events.append(("compose", compose, args, kwargs)), + ) + monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy")) + monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True) + monkeypatch.setattr( + specialize, + "_resource_proof", + lambda *_args: {"ready": True, "runningRunnerContainers": 0}, + ) + + def inspect(name): + if name == specialize.DIND_NAME: + dind["State"] = {"Running": True, "Health": {"Status": "healthy"}} + return dind + return None + + def inventory(): + return containers, [] + + monkeypatch.setattr(specialize, "_inspect", inspect) + monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) + return events, rows, containers, dind + + +def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates( + specialize, monkeypatch +): + events, _rows, _containers, _dind = _stub_active_recovery( + specialize, monkeypatch, family_exists=True + ) + + result = specialize.ensure_active_fleet_ready() + + assert result == {"ready": True, "reason": None} + compose_events = [item for item in events if isinstance(item, tuple) and item[0] == "compose"] + assert len(compose_events) == 1 + _, compose_bytes, args, kwargs = compose_events[0] + assert compose_bytes == b"captured-compose" + assert args == ("up", "-d", "--no-deps", "runner-dind") + assert kwargs["timeout"] <= 30 + assert events.count("fresh-host-gate") == 2 + assert events.index("fresh-host-gate") < events.index("healthy") + + +def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind( + specialize, monkeypatch +): + events, _rows, _containers, _dind = _stub_active_recovery( + specialize, monkeypatch, family_exists=False + ) + gate_count = {"value": 0} + + def fresh_gate(): + gate_count["value"] += 1 + events.append(f"fresh-host-gate-{gate_count['value']}") + if gate_count["value"] == 2: + raise specialize.RunnerFleetSpecializeError( + "runner_fleet_specialization_host_guard_paused" + ) + + rollback = [] + monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fresh_gate) + monkeypatch.setattr( + specialize, + "rollback_host_cgroup_family", + lambda **kwargs: rollback.append(kwargs) or {"removed": True, "familyExists": False}, + ) + + result = specialize.ensure_active_fleet_ready() + + assert result == { + "ready": False, + "reason": "runner_fleet_specialization_host_guard_paused", + } + assert rollback == [ + {"created": True, "initial_controllers": ["cpu", "memory", "pids"]} + ] + assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) + + +def test_active_recovery_accepts_only_disabled_legacy_registrations_after_containers_removed( + specialize, monkeypatch +): + events, rows, _containers, dind = _stub_active_recovery( + specialize, monkeypatch, family_exists=True + ) + rows[-1]["disabled"] = False + result = specialize.ensure_active_fleet_ready() + assert result["ready"] is False + assert result["reason"] == "runner_fleet_specialization_legacy_registration_untrusted" + assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) + assert dind["State"]["Running"] is False + + +def test_active_recovery_cli_reports_only_sanitized_ready_or_reason( + specialize, monkeypatch, capsys +): + monkeypatch.setattr( + specialize, + "ensure_active_fleet_ready", + lambda: {"ready": False, "reason": "runner_fleet_specialization_host_guard_paused"}, + ) + assert specialize._main(("--ensure-active-fleet-ready",)) == 1 + assert capsys.readouterr().err.strip() == "runner_fleet_specialization_host_guard_paused" + + monkeypatch.setattr( + specialize, + "ensure_active_fleet_ready", + lambda: {"ready": True, "reason": None}, + ) + assert specialize._main(("--ensure-active-fleet-ready",)) == 0 + assert capsys.readouterr().out.strip() == "ready" + + +def test_active_recovery_host_gate_rechecks_guard_after_fresh_sample( + specialize, monkeypatch +): + now = specialize.time.time() + sample = { + "loadRatio": 0.2, + "availableMemoryBytes": 16 * 1024**3, + "cacheFreeBytes": 40 * 1024**3, + "swapSampleAt": now, + "swapCritical": False, + } + guard_states = iter([("clear", ""), ("paused", "guard marker appeared")]) + + class HostSampler: + def host_snapshot(self): + return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} + + monkeypatch.setitem( + sys.modules, + "runner_fleet", + type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: next(guard_states))}), + ) + monkeypatch.setitem( + sys.modules, + "runner_capacity_controller", + type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), + ) + + with pytest.raises( + specialize.RunnerFleetSpecializeError, + match="runner_fleet_specialization_host_guard_paused", + ): + specialize._fresh_recovery_host_gate() + + +@pytest.mark.parametrize( + "sample,expected_reason", + [ + (None, "runner_fleet_specialization_host_telemetry_unavailable"), + ( + { + "loadRatio": 0.2, + "availableMemoryBytes": 16 * 1024**3, + "cacheFreeBytes": 40 * 1024**3, + "swapSampleAt": 1, + "swapCritical": False, + }, + "runner_fleet_specialization_host_headroom_insufficient", + ), + ], +) +def test_active_recovery_host_gate_fails_closed_on_missing_or_stale_sample( + specialize, monkeypatch, sample, expected_reason +): + class HostSampler: + def host_snapshot(self): + if sample is None: + return {"status": "unavailable", "host": None} + return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} + + monkeypatch.setitem( + sys.modules, + "runner_fleet", + type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: ("clear", ""))}), + ) + monkeypatch.setitem( + sys.modules, + "runner_capacity_controller", + type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), + ) + + with pytest.raises(specialize.RunnerFleetSpecializeError, match=expected_reason): + specialize._fresh_recovery_host_gate() diff --git a/backend/tests/test_runner_profiles.py b/backend/tests/test_runner_profiles.py new file mode 100644 index 0000000..5b4d7ad --- /dev/null +++ b/backend/tests/test_runner_profiles.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +PROFILES_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py" + + +def _profiles_module(): + spec = importlib.util.spec_from_file_location("appops_runner_profiles_test", PROFILES_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_fixed_profiles_expose_stable_names_labels_and_resource_envelopes(): + profiles = _profiles_module() + + assert tuple(profiles.RUNNER_PROFILES) == ( + "checks", + "backend", + "frontend", + "security", + "browser", + "build", + ) + assert [profile.display_name for profile in profiles.RUNNER_PROFILES.values()] == [ + "ITWorx CI ยท Checks", + "ITWorx CI ยท Backend", + "ITWorx CI ยท Frontend", + "ITWorx CI ยท Security", + "ITWorx CI ยท Browser", + "ITWorx CI ยท Build", + ] + assert [profile.container_name for profile in profiles.RUNNER_PROFILES.values()] == [ + "itworx-ci-checks", + "itworx-ci-backend", + "itworx-ci-frontend", + "itworx-ci-security", + "itworx-ci-browser", + "itworx-ci-build", + ] + assert [ + (profile.cpu_milli, profile.memory_mib, profile.heavy) + for profile in profiles.RUNNER_PROFILES.values() + ] == [ + (1000, 1536, False), + (3000, 5120, False), + (3000, 5120, False), + (1000, 2048, False), + (3000, 6144, True), + (4000, 6144, True), + ] + assert (profiles.BASE_OVERHEAD_CPU_MILLI, profiles.BASE_OVERHEAD_MEMORY_MIB) == ( + 1000, + 2048, + ) + assert (profiles.HOST_CPU_BUDGET_MILLI, profiles.HOST_MEMORY_BUDGET_MIB) == ( + 8000, + 16384, + ) + assert (profiles.MAX_ACTIVE_CLAIMS, profiles.MAX_HEAVY_ACTIVE) == (2, 1) + + +def test_shared_legacy_labels_only_admit_backend_and_frontend_profiles(): + profiles = _profiles_module() + + assert [ + profile.id for profile in profiles.profiles_for_labels(["linux-validation"]) + ] == ["backend", "frontend"] + assert [ + profile.id for profile in profiles.profiles_for_labels(["ubuntu-latest"]) + ] == ["backend", "frontend"] + assert profiles.profiles_for_labels(["itworx-ci-security"])[0].id == "security" + assert profiles.profiles_for_labels(["linux-validation", "itworx-ci-security"]) == () + assert profiles.profile_for_container("itworx-ci-browser").id == "browser" + assert profiles.profile_for_container("itworx-ci-unknown") is None + + +def test_legacy_docker_jobs_use_only_the_fixed_heavy_build_profile(): + profiles = _profiles_module() + assert [profile.id for profile in profiles.profiles_for_labels(["docker-validation"])] == ["build"] + assert profiles.RUNNER_PROFILES['build'].heavy + assert profiles.profiles_for_labels(["docker-validation", "linux-validation"]) == () + assert profiles.profiles_for_labels(["docker-validation", "unraid-deploy"]) == () diff --git a/backend/tests/test_runner_resource_envelope.py b/backend/tests/test_runner_resource_envelope.py new file mode 100644 index 0000000..14c512a --- /dev/null +++ b/backend/tests/test_runner_resource_envelope.py @@ -0,0 +1,1106 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py" + + +def _module(): + broker_dir = str(ENVELOPE_PATH.parent) + if broker_dir not in sys.path: + sys.path.insert(0, broker_dir) + spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]: + return { + "path": path, + "cpuMax": cpu_max, + "memoryMax": str(memory_max), + "cpuUsageUsec": "1200", + "memoryCurrentBytes": "4096", + } + + +def _snapshot(): + names = ( + "itworx-ci-checks", + "itworx-ci-backend", + "itworx-ci-frontend", + "itworx-ci-security", + "itworx-ci-browser", + "itworx-ci-build", + "gitea-runner-dind", + ) + containers = [] + groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)] + dind_path = "" + for index, name in enumerate(names, start=1): + container_id = f"{index:064x}" + path = f"/itworx-ci/{container_id}" + if name == "gitea-runner-dind": + dind_path = path + cpu_max = "600000 100000" + memory_max = 12 * 1024**3 + running = True + else: + cpu_max = "50000 100000" + memory_max = 512 * 1024**2 + # Only two exact outer runner processes may be live. Stopped + # configured containers remain part of the identity inventory. + running = name in {"itworx-ci-checks", "itworx-ci-backend"} + containers.append( + { + "name": name, + "id": container_id, + "path": path if running else None, + "running": running, + **( + {} + if name == "gitea-runner-dind" + else { + "configuredCpuMilli": 500, + "configuredMemoryBytes": 512 * 1024**2, + } + ), + } + ) + if running: + groups.append(_group(path, cpu_max, memory_max)) + groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3)) + return { + "version": 2, + "family": _group("/itworx-ci", "800000 100000", 16 * 1024**3), + "dind": { + "path": dind_path, + "cpuMax": "600000 100000", + "memoryMax": str(12 * 1024**3), + "configuredCpuMilli": 6000, + "configuredMemoryBytes": 12 * 1024**3, + }, + "containers": containers, + "groups": groups, + } + + +def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config): + snapshot = _snapshot() + dind_path = str(snapshot["dind"]["path"]) + workload_specs = [ + ( + "8" * 64, + {"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0}, + 1_000, + 1024**3, + "100000 100000", + ), + ( + "9" * 64, + builder_cpu_config, + 3_000, + 5 * 1024**3, + "300000 100000", + ), + ] + workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs] + other_outer_ids = { + item["id"] + for item in snapshot["containers"] + if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"} + } + old_workload_path = f"{dind_path}/docker/job-1" + snapshot["groups"] = [ + group + for group in snapshot["groups"] + if group["path"] != old_workload_path + and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids} + ] + for row in snapshot["containers"]: + if row["name"] == envelope.DIND_NAME: + continue + running = row["name"] == "itworx-ci-build" + row["running"] = running + row["path"] = f"/itworx-ci/{row['id']}" if running else None + if running: + snapshot["groups"].append( + _group(row["path"], "50000 100000", 512 * 1024**2) + ) + nested_items = [] + for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs: + config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config} + nested_items.append( + { + "Id": container_id, + "State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71}, + "HostConfig": config, + } + ) + snapshot["groups"].append( + _group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes) + ) + snapshot["workloads"] = [ + { + "id": container_id, + "path": path, + "nestedPid": 70 if container_id.startswith("8") else 71, + "cpuMilli": cpu_milli, + "memoryBytes": memory_bytes, + } + for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path + in zip(workload_specs, workload_paths, strict=True) + ] + + profile_names = { + profile.container_name: profile.id + for profile in envelope.RUNNER_PROFILES.values() + } + ids = {row["name"]: row["id"] for row in snapshot["containers"]} + pids = {name: 2000 + index for index, name in enumerate(ids)} + outer_items = {} + for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: + if name == envelope.DIND_NAME: + labels = { + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.service": "runner-dind", + } + host = { + "CgroupParent": "itworx-ci", + "NanoCpus": 6_000_000_000, + "Memory": 12 * 1024**3, + } + state = {"Running": True, "Pid": pids[name]} + else: + profile_id = profile_names[name] + labels = { + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.service": f"runner-{profile_id}", + "tech.itworx.role": "gitea-actions-runner", + "tech.itworx.ci.profile": profile_id, + } + is_running = name == "itworx-ci-build" + host = { + "CgroupParent": "itworx-ci", + "NanoCpus": 500_000_000, + "Memory": 512 * 1024**2, + } + state = {"Running": is_running, "Pid": pids[name] if is_running else 0} + outer_items[name] = { + "Name": f"/{name}", + "Id": ids[name], + "Config": {"Labels": labels}, + "State": state, + "HostConfig": host, + } + + def command(args, **_kwargs): + if "info" in args: + return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" + if args[-3:] == ["ps", "--no-trunc", "-q"]: + return "\n".join(item["Id"] for item in nested_items) + if "inspect" in args: + return json.dumps(nested_items) + raise AssertionError(f"unexpected fixed Docker command: {args}") + + monkeypatch.setattr(envelope, "_run", command) + monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) + probe_calls = [] + + def fake_probe(operation, payload=None): + probe_calls.append((operation, payload)) + return snapshot + + monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) + return snapshot, nested_items, probe_calls + + +def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded(): + envelope = _module() + result = envelope._evaluate_cgroup_snapshot( + ["build", "backend"], + _snapshot(), + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is True + assert result["reason"] is None + assert result["effectiveCpuMilli"] == 8000 + assert result["effectiveMemoryMiB"] == 16384 + assert result["activeClaims"] == 2 + assert result["heavyClaims"] == 1 + assert result["runningRunnerContainers"] == 2 + assert result["cgroups"]["dindCpuMilli"] == 6000 + assert result["cgroups"]["dindMemoryMiB"] == 12288 + assert result["familyCpuUsageUsec"] == 1200 + assert result["familyMemoryCurrentBytes"] == 4096 + assert len(result["digest"]) == 64 + + +def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook(): + envelope = _module() + + result = envelope._evaluate_cgroup_snapshot( + ["backend", "backend"], + _snapshot(), + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is True + assert result["activeClaims"] == 2 + assert result["cgroups"]["reservedCpuMilli"] == 7000 + assert result["cgroups"]["reservedMemoryMiB"] == 12288 + + denied = envelope._evaluate_cgroup_snapshot( + ["build", "browser"], + _snapshot(), + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + assert denied["ready"] is False + assert denied["reason"] == "resource_claim_heavy_limit_exceeded" + + +def test_attestor_rejects_an_unbounded_workload_descendant(): + envelope = _module() + snapshot = _snapshot() + snapshot["groups"].append( + _group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3) + ) + + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_unbounded_descendant" + + +def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded(): + envelope = _module() + snapshot = _snapshot() + dind_path = str(snapshot["dind"]["path"]) + snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max")) + + result = envelope._evaluate_cgroup_snapshot( + ["backend"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is True + assert result["cgroups"]["dindCpuMilli"] == 6000 + + +def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation(): + envelope = _module() + snapshot = _snapshot() + dind_path = str(snapshot["dind"]["path"]) + original_workload = next( + item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1" + ) + original_workload["cpuMax"] = "50000 100000" + original_workload["memoryMax"] = str(1024**3) + snapshot["groups"].extend( + [ + _group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3), + _group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3), + ] + ) + + result = envelope._evaluate_cgroup_snapshot( + ["backend"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is True + assert result["cgroups"]["reservedCpuMilli"] == 4000 + assert result["cgroups"]["reservedMemoryMiB"] == 7168 + + +def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim(): + envelope = _module() + snapshot = _snapshot() + dind_path = str(snapshot["dind"]["path"]) + snapshot["groups"].extend( + [ + _group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3), + _group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3), + ] + ) + + result = envelope._evaluate_cgroup_snapshot( + ["backend"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved" + + +def test_attestor_rejects_workload_without_an_active_claim(): + envelope = _module() + + result = envelope._evaluate_cgroup_snapshot( + [], + _snapshot(), + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_workload_without_claim" + + +def test_attestor_rejects_unknown_outer_container_and_parent_escape(): + envelope = _module() + snapshot = _snapshot() + snapshot["containers"].append( + {"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"} + ) + snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2)) + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_unknown_outer_container" + + escaped = _snapshot() + escaped["containers"][0]["path"] = "/docker/outside" + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + escaped, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_container_outside_family" + + +def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract(): + envelope = _module() + snapshot = _snapshot() + del snapshot["groups"][-1]["memoryCurrentBytes"] + + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_usage_unavailable" + + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + _snapshot(), + expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384}, + ) + assert result["ready"] is False + assert result["reason"] == "resource_budget_contract_mismatch" + + +def test_claim_count_and_unknown_profiles_fail_closed(): + envelope = _module() + + too_many = envelope._evaluate_cgroup_snapshot( + ["checks", "security", "backend"], + _snapshot(), + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + assert too_many["ready"] is False + assert too_many["reason"] == "resource_claim_slot_limit_exceeded" + + unknown = envelope._evaluate_cgroup_snapshot( + ["general"], + _snapshot(), + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + assert unknown["ready"] is False + assert unknown["reason"] == "resource_claim_profile_unknown" + + +def test_attestor_reports_and_rejects_more_than_two_running_outer_runners(): + envelope = _module() + snapshot = _snapshot() + name = "itworx-ci-frontend" + item = next(container for container in snapshot["containers"] if container["name"] == name) + item["running"] = True + item["path"] = f"/itworx-ci/{item['id']}" + snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2)) + + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded" + assert result["runningRunnerContainers"] == 3 + + +def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket(): + envelope = _module() + image_id = "sha256:" + "a" * 64 + code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" + + args = envelope._host_probe_create_args("snapshot", image_id, code_source) + + assert args[0:3] == ["docker", "create", "--interactive"] + assert "--pid=host" not in args + assert "--cgroupns=private" in args + assert "--network=none" in args + assert "--read-only" in args + assert "--cap-drop=ALL" in args + assert "--cpus=0.1" in args + assert "--memory=64m" in args + assert "--pids-limit=64" in args + assert "--mount" in args + mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] + assert len(mounts) == 2 + mount = mounts[0] + assert "source=/sys/fs/cgroup" in mount + assert "target=/host-cgroup" in mount + assert "readonly" in mount + assert mounts[1] == ( + f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rprivate" + ) + assert not any("/proc" in item or "docker.sock" in item for item in args) + assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"] + + +def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): + envelope = _module() + image_id = "sha256:" + "a" * 64 + code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" + base = { + "Id": "b" * 64, + "Name": "/itworx-runner-cgroup-probe", + "Image": image_id, + "Config": { + "Image": image_id, + "Entrypoint": ["python3"], + "Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"], + "Labels": envelope._host_probe_labels(), + "User": "0:0", + }, + "State": {"Running": False, "ExitCode": 0}, + "HostConfig": { + "NetworkMode": "none", + "PidMode": "", + "CgroupnsMode": "private", + "ReadonlyRootfs": True, + "Privileged": False, + "NanoCpus": 100_000_000, + "Memory": 64 * 1024**2, + "PidsLimit": 64, + "CapAdd": [], + "CapDrop": ["ALL"], + "SecurityOpt": ["no-new-privileges:true"], + "RestartPolicy": {"Name": "no"}, + "Binds": None, + "Volumes": None, + "Mounts": [ + { + "Type": "bind", + "Source": "/sys/fs/cgroup/itworx-ci", + "Target": "/host-cgroup", + "ReadOnly": True, + "BindOptions": {"Propagation": "rprivate"}, + }, + { + "Type": "bind", + "Source": code_source, + "Target": "/opt/gitea-deploy", + "ReadOnly": True, + "BindOptions": {"Propagation": "rprivate"}, + }, + ], + }, + "Mounts": [ + { + "Type": "bind", + "Source": "/sys/fs/cgroup/itworx-ci", + "Destination": "/host-cgroup", + "Mode": "", + "RW": False, + "Propagation": "rprivate", + }, + { + "Type": "bind", + "Source": code_source, + "Destination": "/opt/gitea-deploy", + "Mode": "", + "RW": False, + "Propagation": "rprivate", + }, + ], + } + + assert envelope._validate_host_probe_inspect( + base, mode="snapshot", image_id=image_id, code_source=code_source + ) + for mode in ("family", "ensure", "rollback"): + candidate = copy.deepcopy(base) + expected_source = "/sys/fs/cgroup" + expected_rw = mode in {"ensure", "rollback"} + candidate["Mounts"][0].update( + Source=expected_source, + Mode="", + RW=expected_rw, + ) + candidate["HostConfig"]["Mounts"][0].update( + Source=expected_source, + ReadOnly=not expected_rw, + ) + candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else [] + assert envelope._validate_host_probe_inspect( + candidate, mode=mode, image_id=image_id, code_source=code_source + ) + + for mutate in ( + lambda item: item["HostConfig"].update(PidMode="host"), + lambda item: item["HostConfig"].update(CgroupnsMode="host"), + lambda item: item["HostConfig"].update(NetworkMode="bridge"), + lambda item: item["HostConfig"].update(ReadonlyRootfs=False), + lambda item: item["HostConfig"].update(Privileged=True), + lambda item: item["HostConfig"].update(Memory=0), + lambda item: item["Mounts"].append( + {"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False} + ), + lambda item: item["Mounts"][0].update(RW=True), + lambda item: item["Mounts"][1].update(RW=True), + lambda item: item["Mounts"][0].update(Mode="ro"), + lambda item: item["Mounts"][1].update(Mode="ro"), + lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"), + lambda item: item["HostConfig"]["Mounts"].append( + { + "Type": "bind", + "Source": "/proc", + "Target": "/host-proc", + "ReadOnly": True, + "BindOptions": {"Propagation": "rprivate"}, + } + ), + lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False), + lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"), + lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"), + ): + candidate = copy.deepcopy(base) + mutate(candidate) + assert not envelope._validate_host_probe_inspect( + candidate, mode="snapshot", image_id=image_id, code_source=code_source + ) + + +def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch): + envelope = _module() + revision = "d" * 40 + code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}" + image_id = "sha256:" + "e" * 64 + release = {"sourceRevision": revision} + controller = { + "Name": "/gitea-deploy-control", + "Image": image_id, + "State": {"Running": True}, + "Mounts": [ + { + "Type": "bind", + "Source": code_source, + "Destination": "/opt/gitea-deploy", + "RW": False, + } + ], + } + monkeypatch.setitem( + sys.modules, + "status_server", + SimpleNamespace(_require_controller_release=lambda: release), + ) + monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller])) + + assert envelope._controller_runtime_identity() == (image_id, code_source) + + controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}" + with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"): + envelope._controller_runtime_identity() + + +def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch): + envelope = _module() + calls = [] + + def probe(operation, payload=None): + calls.append((operation, payload)) + return {"created": True, "rootSubtreeBefore": ["pids"]} + + monkeypatch.setattr(envelope, "_run_host_probe", probe) + result = envelope.ensure_host_cgroup_family(allow_existing=False) + + assert result == {"created": True, "rootSubtreeBefore": ["pids"]} + assert calls == [("ensure", {"allowExisting": False})] + + +def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch): + envelope = _module() + calls = [] + monkeypatch.setattr( + envelope, + "_run_host_probe", + lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False}, + ) + + result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"]) + + assert result == {"removed": True, "familyExists": False} + assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})] + with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): + envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"]) + + +def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch): + envelope = _module() + required = {"cpu", "memory", "pids"} + monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required)) + monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123) + monkeypatch.setattr(envelope.os, "close", lambda _fd: None) + groups = [ + { + "path": envelope.FAMILY_CGROUP, + "cpuMax": "800000 100000", + "memoryMax": str(16 * 1024**3), + } + ] + monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups)) + reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": ""} + monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name]) + + ready = envelope._probe_family_info(1) + assert ready["safeToStartDind"] is True + assert ready["safeToStartDindReason"] is None + + groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"}) + blocked = envelope._probe_family_info(1) + assert blocked["safeToStartDind"] is False + assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty" + + groups.pop() + reads["cgroup.procs"] = "4321" + occupied = envelope._probe_family_info(1) + assert occupied["safeToStartDind"] is False + assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty" + + +def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback(): + envelope = _module() + image_id = "sha256:" + "a" * 64 + code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" + + snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source) + ensure = envelope._host_probe_create_args("ensure", image_id, code_source) + rollback = envelope._host_probe_create_args("rollback", image_id, code_source) + family = envelope._host_probe_create_args("family", image_id, code_source) + + def mounts(args): + return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] + + snapshot_mount, snapshot_code_mount = mounts(snapshot) + ensure_mount, ensure_code_mount = mounts(ensure) + rollback_mount, rollback_code_mount = mounts(rollback) + family_mount, family_code_mount = mounts(family) + assert snapshot_mount == ( + "type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate" + ) + assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup") + assert "readonly" not in ensure_mount + assert "readonly" not in rollback_mount + assert family_mount == ( + "type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate" + ) + assert all( + code_mount + == f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rprivate" + for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount) + ) + for args in (snapshot, ensure, rollback, family): + assert "--pid=host" not in args + assert "--network=none" in args + assert "--cap-drop=ALL" in args + assert "--cpus=0.1" in args + assert "--memory=64m" in args + assert "--pids-limit=64" in args + assert not any("/proc" in item or "docker.sock" in item for item in args) + assert "--cap-add=DAC_OVERRIDE" not in snapshot + assert "--cap-add=DAC_OVERRIDE" in ensure + assert "--cap-add=DAC_OVERRIDE" in rollback + with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"): + envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code") + + +def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path): + envelope = _module() + family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + dind_id = "7" * 64 + workload_id = "8" * 64 + dind_path = f"/itworx-ci/{dind_id}" + workload_path = f"{dind_path}/docker/{workload_id}" + monkeypatch.setattr( + envelope, + "_probe_walk", + lambda _fd: [ + {"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)}, + {"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)}, + {"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)}, + ], + ) + monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd)) + # The host cgroup contains global PID 4321; nested Docker reports PID 7 + # inside DinD's private namespace. Membership is proven by ID path and a + # populated cgroup, never by comparing those unrelated PID numbers. + monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw: "4321" if name == "cgroup.procs" else "") + outer = [] + for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1): + if name == envelope.DIND_NAME: + outer.append({"name": name, "id": dind_id, "running": True, "pid": 4321}) + else: + outer.append( + { + "name": name, + "id": f"{index:064x}", + "running": False, + "pid": None, + "configuredCpuMilli": 500, + "configuredMemoryBytes": 512 * 1024**2, + } + ) + try: + snapshot = envelope._probe_snapshot( + { + "op": "snapshot", + "outer": outer, + "workloads": [ + { + "id": workload_id, + "cgroupParent": "docker", + "nestedPid": 7, + "cpuMilli": 100, + "memoryBytes": 1024**3, + } + ], + }, + family_fd, + ) + finally: + os.close(family_fd) + + assert snapshot["workloads"][0]["path"] == workload_path + assert snapshot["workloads"][0]["nestedPid"] == 7 + assert snapshot["containers"][-1]["path"] == dind_path + assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path + for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"): + try: + envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent) + except RuntimeError as exc: + assert str(exc) == "resource_cgroup_workload_identity_invalid" + else: + raise AssertionError("unmapped nested Docker cgroup parent was accepted") + + +def test_attestor_rejects_unowned_descendants_below_dind(): + envelope = _module() + snapshot = _snapshot() + dind_path = str(snapshot["dind"]["path"]) + snapshot["groups"].append( + _group(f"{dind_path}/unknown-child", "100000 100000", 1024**3) + ) + snapshot["workloads"] = [ + {"path": f"{dind_path}/docker/job-1"} + ] + + result = envelope._evaluate_cgroup_snapshot( + ["checks"], + snapshot, + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_unmapped_descendant" + + +def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch): + envelope = _module() + snapshot = _snapshot() + family_path = str(snapshot["family"]["path"]) + dind_path = str(snapshot["dind"]["path"]) + probe_id = "f" * 64 + snapshot["containers"] = [ + { + **item, + "running": item["name"] == envelope.DIND_NAME, + "path": item["path"] if item["name"] == envelope.DIND_NAME else None, + } + for item in snapshot["containers"] + ] + snapshot["groups"] = [ + group + for group in snapshot["groups"] + if group["path"] == family_path or group["path"] == dind_path + ] + workload_path = f"{dind_path}/docker/{probe_id}" + snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3)) + snapshot["workloads"] = [{ + "id": probe_id, + "path": workload_path, + "nestedPid": 77, + "cpuMilli": 4000, + "memoryBytes": 6 * 1024**3, + }] + monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot) + + result = envelope.attest_build_probe_workload(probe_id, 77) + + assert result["ready"] is True, result + assert result["runningRunnerContainers"] == 0 + assert result["probeWorkload"] == { + "id": probe_id, + "path": workload_path, + "nestedPid": 77, + "cpuMilli": 4000, + "memoryBytes": 6 * 1024**3, + "descendantGroupCount": 0, + } + assert "snapshot" not in result + assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False + + +def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch): + envelope = _module() + snapshot = _snapshot() + profile_names = { + profile.container_name: profile.id + for profile in envelope.RUNNER_PROFILES.values() + } + ids = {row["name"]: row["id"] for row in snapshot["containers"]} + pids = {name: 2000 + index for index, name in enumerate(ids)} + outer_items = {} + for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: + if name == envelope.DIND_NAME: + labels = { + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.service": "runner-dind", + } + host = { + "CgroupParent": "itworx-ci", + "NanoCpus": 6_000_000_000, + "Memory": 12 * 1024**3, + } + state = {"Running": True, "Pid": pids[name]} + else: + profile_id = profile_names[name] + labels = { + "com.docker.compose.project": "gitea-deploy-runners", + "com.docker.compose.service": f"runner-{profile_id}", + "tech.itworx.role": "gitea-actions-runner", + "tech.itworx.ci.profile": profile_id, + } + is_running = name in {"itworx-ci-checks", "itworx-ci-backend"} + host = { + "CgroupParent": "itworx-ci", + "NanoCpus": 500_000_000, + "Memory": 512 * 1024**2, + } + state = {"Running": is_running, "Pid": pids[name] if is_running else 0} + outer_items[name] = { + "Name": f"/{name}", + "Id": ids[name], + "Config": {"Labels": labels}, + "State": state, + "HostConfig": host, + } + + docker_commands = [] + + def command(args, **_kwargs): + docker_commands.append(args) + if "info" in args: + return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" + if "docker" in args and "ps" in args: + return "" + raise AssertionError(f"unexpected fixed Docker command: {args}") + + monkeypatch.setattr(envelope, "_run", command) + monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) + probe_calls = [] + + def fake_probe(operation, payload=None): + probe_calls.append((operation, payload)) + return snapshot + + monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) + real_read_text = Path.read_text + + def reject_local_proc_or_sys(path, *args, **kwargs): + if str(path).startswith(("/proc/", "/sys/fs/cgroup")): + raise AssertionError("the private controller must use the host helper") + return real_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys) + result = envelope.attest_resource_envelope( + ["backend"], + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert probe_calls and probe_calls[0][0] == "snapshot" + assert result["ready"] is True + assert result["familyCpuUsageUsec"] == 1200 + assert result["familyMemoryCurrentBytes"] == 4096 + dind_calls = [args for args in docker_commands if "exec" in args] + assert len(dind_calls) == 2 + assert all( + f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args + for args in dind_calls + ) + + +@pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"]) +def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support): + envelope = _module() + _install_classic_builder_collection( + envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000} + ) + original = envelope._run + + def command(args, **kwargs): + if "exec" in args and "info" in args: + return support + return original(args, **kwargs) + + monkeypatch.setattr(envelope, "_run", command) + with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"): + envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) + + +def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch): + envelope = _module() + snapshot, nested_items, probe_calls = _install_classic_builder_collection( + envelope, + monkeypatch, + {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}, + ) + + result = envelope.attest_resource_envelope( + ["build"], + expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, + ) + + assert result["ready"] is True, result + assert probe_calls[0][0] == "snapshot" + assert probe_calls[0][1]["workloads"] == [ + { + "id": nested_items[0]["Id"], + "cgroupParent": "docker", + "nestedPid": 70, + "cpuMilli": 1000, + "memoryBytes": 1024**3, + }, + { + "id": nested_items[1]["Id"], + "cgroupParent": "docker", + "nestedPid": 71, + "cpuMilli": 3000, + "memoryBytes": 5 * 1024**3, + }, + ] + assert snapshot["workloads"] == [ + { + "id": nested_items[0]["Id"], + "path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}", + "nestedPid": 70, + "cpuMilli": 1000, + "memoryBytes": 1024**3, + }, + { + "id": nested_items[1]["Id"], + "path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}", + "nestedPid": 71, + "cpuMilli": 3000, + "memoryBytes": 5 * 1024**3, + }, + ] + builder_group = next( + group + for group in snapshot["groups"] + if group["path"].endswith("/docker/" + nested_items[1]["Id"]) + ) + assert builder_group["cpuMax"] == "300000 100000" + assert builder_group["memoryMax"] == str(5 * 1024**3) + + +@pytest.mark.parametrize( + "builder_cpu_config", + [ + pytest.param( + {"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000}, + id="zero-quota", + ), + pytest.param( + {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0}, + id="zero-period", + ), + pytest.param( + {"NanoCpus": 0, "CpuQuota": 300_000}, + id="missing-period", + ), + pytest.param( + {"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000}, + id="boolean-quota", + ), + pytest.param( + {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True}, + id="boolean-period", + ), + pytest.param( + {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001}, + id="period-out-of-range", + ), + pytest.param( + {"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000}, + id="fractional-millicpu", + ), + pytest.param( + {"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000}, + id="conflicting-encodings", + ), + pytest.param( + {"CpuQuota": 300_000, "CpuPeriod": 100_000}, + id="missing-nanocpus-field", + ), + ], +) +def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations( + monkeypatch, builder_cpu_config +): + envelope = _module() + _snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection( + envelope, monkeypatch, builder_cpu_config + ) + + with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"): + envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) + + assert probe_calls == [] diff --git a/backend/tests/test_runner_specialization_controller_surface.py b/backend/tests/test_runner_specialization_controller_surface.py new file mode 100644 index 0000000..a89f2a9 --- /dev/null +++ b/backend/tests/test_runner_specialization_controller_surface.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import importlib.util +import io +import json +import sys +from email.message import Message +from pathlib import Path +from types import SimpleNamespace + +import pytest + +CONTROLLER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker" + + +@pytest.fixture +def controller(monkeypatch): + monkeypatch.syspath_prepend(str(CONTROLLER)) + spec = importlib.util.spec_from_file_location("specialization_surface_test", CONTROLLER / "status_server.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def handler(controller, monkeypatch, payload=None): + instance = object.__new__(controller.Handler) + body = json.dumps(payload or {"reason": "Specialize CI profiles"}).encode() + instance.headers = Message() + instance.headers["Content-Length"] = str(len(body)) + instance.rfile = io.BytesIO(body) + instance.path = controller.RUNNER_FLEET_SPECIALIZE_PATH + responses = [] + monkeypatch.setattr(instance, "_trusted_client", lambda: True) + monkeypatch.setattr(instance, "_send_json", lambda code, data: responses.append((code, json.loads(data)))) + signatures = [] + monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *args, **kwargs: signatures.append(kwargs) or True) + monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {"revision": "a" * 40}) + monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "a" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY}) + monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda revision: {"status": "already_deployed"}) + monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}}) + monkeypatch.setattr(controller, "_repository_path_for_delivery", lambda *args: Path("fixed-repo.git")) + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: True) + monkeypatch.setattr(controller, "runner_fleet_specialized_topology", lambda: {"ready": False}) + return instance, responses, signatures + + +@pytest.mark.parametrize("state", ["blocked", "active", "unknown"]) +def test_legacy_upgrade_cannot_recreate_runners_after_cutover(controller, monkeypatch, state): + instance, responses, _ = handler(controller, monkeypatch) + monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: state) + monkeypatch.setattr(controller, "execute_runner_fleet_v2", lambda _: pytest.fail("legacy executor must be fenced")) + instance._handle_runner_fleet_upgrade_v2() + assert responses == [(409, {"status": "legacy_fleet_upgrade_blocked"})] + + +def test_specialization_requires_matching_installed_controller_source(controller, monkeypatch): + instance, responses, _ = handler(controller, monkeypatch) + monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "b" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY}) + monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda _: pytest.fail("source mismatch must block mutation")) + instance.do_POST() + assert responses == [(409, {"status": "controller_source_mismatch"})] + + +def test_specialization_uses_only_signed_current_main_script(controller, monkeypatch): + instance, responses, signatures = handler(controller, monkeypatch) + commands = [] + scripts = [] + monkeypatch.setattr(controller.subprocess, "run", lambda command, **_: commands.append(command) or SimpleNamespace(returncode=0, stdout=b"exact source script")) + monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda script: scripts.append(script) or {"status": "specialized", "verifiedRunners": [], "resourceEnvelope": {"ready": True}}) + instance.do_POST() + assert signatures == [{"method": "POST", "path": controller.RUNNER_FLEET_SPECIALIZE_PATH}] + assert commands == [["git", "--git-dir=fixed-repo.git", "show", "a" * 40 + ":scripts/upgrade_ci_runner_fleet_specialized.sh"]] + assert scripts == [b"exact source script"] + assert responses[0][0] == 200 + assert responses[0][1]["revision"] == "a" * 40 + assert "burstRunning" not in responses[0][1] + + +def test_pending_protected_validation_blocks_specialization(controller, monkeypatch): + instance, responses, _ = handler(controller, monkeypatch) + monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _: {"status": "preflight_rejected"}) + monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda _: pytest.fail("pending validation must block mutation")) + instance.do_POST() + assert responses[0][0] == 409 + assert responses[0][1]["status"] == "preflight_rejected" + + +def test_caller_cannot_choose_script_image_or_runner(controller, monkeypatch): + instance, responses, _ = handler(controller, monkeypatch, {"reason": "Specialize CI profiles", "script": "untrusted.sh"}) + instance.do_POST() + assert responses == [(400, {"status": "invalid_request"})] + + +@pytest.mark.parametrize("state", ["blocked", "unknown"]) +def test_maintenance_never_falls_back_to_legacy_during_cutover(controller, monkeypatch, state): + monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: state) + monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("legacy maintenance must be fenced")) + assert controller._runner_maintenance_cycle() == "deferred" + + +def test_cutover_and_maintenance_share_single_owner_lock(controller, monkeypatch): + monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: pytest.fail("maintenance must not inspect during cutover")) + with controller._runner_fleet_upgrade_lock: + assert controller._runner_maintenance_cycle() == "migration_busy" + + +def test_active_maintenance_calls_only_admission_controller(controller, monkeypatch): + import runner_admission_controller + + calls = [] + monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: "active") + monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "cycle", lambda _: calls.append("admission")) + monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("legacy worker must stop")) + assert controller._runner_maintenance_cycle() == "specialized" + assert calls == ["admission"] + + +def test_statistics_get_returns_persisted_projection_without_a_cycle(controller, monkeypatch): + import runner_admission_controller + + instance, responses, _ = handler(controller, monkeypatch) + instance.path = "/v1/runner-fleet/statistics" + monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "cycle", lambda _: pytest.fail("GET may not start collection or mutate fleet")) + monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "snapshot", lambda _: {"schemaVersion": 1, "status": "unavailable"}) + instance.do_GET() + assert responses == [(200, {"schemaVersion": 1, "status": "unavailable"})] + + +def test_statistics_configuration_uses_same_protected_source_and_fixed_signed_route(controller, monkeypatch): + instance, responses, signatures = handler(controller, monkeypatch) + instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH + calls = [] + monkeypatch.setattr(controller, 'configure_runner_statistics_binding', + lambda **kwargs: calls.append(kwargs) or {'status': 'configured', 'credentialReturned': False}) + monkeypatch.setattr(controller, 'runner_fleet_specialized_topology', lambda: pytest.fail('configuration must not mutate fleet')) + instance.do_POST() + assert signatures == [{'method': 'POST', 'path': controller.RUNNER_STATISTICS_CONFIGURE_PATH}] + assert calls == [{'revision': 'a' * 40, 'projects': {controller.SELF_UPGRADE_REPOSITORY: {'branch': 'main'}}}] + assert responses == [(200, {'status': 'configured', 'credentialReturned': False})] + + +def test_statistics_configuration_source_mismatch_writes_nothing(controller, monkeypatch): + instance, responses, _ = handler(controller, monkeypatch) + instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH + monkeypatch.setattr(controller, '_require_controller_release', lambda: {'sourceRevision': 'b' * 40, 'sourceRepository': controller.SELF_UPGRADE_REPOSITORY}) + monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration')) + instance.do_POST() + assert responses == [(409, {'status': 'controller_source_mismatch'})] + + +def test_statistics_configuration_caller_cannot_select_credential_or_endpoint(controller, monkeypatch): + instance, responses, _ = handler(controller, monkeypatch, {'reason': 'configure stats', 'token': 'caller-credential'}) + instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH + monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration')) + instance.do_POST() + assert responses == [(400, {'status': 'invalid_request'})] diff --git a/backend/tests/test_runner_statistics_binding.py b/backend/tests/test_runner_statistics_binding.py new file mode 100644 index 0000000..d9f47ef --- /dev/null +++ b/backend/tests/test_runner_statistics_binding.py @@ -0,0 +1,391 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import tempfile +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif(os.name != 'posix', reason='Linux controller ownership/flock contract') + + +@pytest.fixture +def binding(tmp_path, monkeypatch): + path = Path(__file__).resolve().parents[2] / 'ops/gitea-deploy-broker/runner_statistics_binding.py' + spec = importlib.util.spec_from_file_location('binding_test', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + anchor = tmp_path / 'controller-anchor' + state = anchor / 'state' + (state / 'locks').mkdir(parents=True) + source = tmp_path / 'brain' + source.mkdir() + data = tmp_path / 'appops' + data.mkdir() + monkeypatch.setattr(module, 'ROOT_UID', os.getuid()) + if os.geteuid() != 0: + # The production controller is root. Keep portable unit coverage for + # non-root Linux runs while root-only tests assert 100:101. + monkeypatch.setattr(module, 'APPOPS_UID', os.geteuid()) + monkeypatch.setattr(module, 'APPOPS_GID', os.getegid()) + for key, value in {'STATE_ROOT': state, 'BINDING_ROOT': state / 'runner-statistics-binding', + 'PB_SOURCE': source, 'PB_ENV': source / '.env', 'APPOPS_DATA': data, + 'TOKEN_FILE': data / 'runner-statistics-token', + 'ADMIN_TOKEN_FILE': data / 'mcp-token', 'GLOBAL_LOCK': anchor / 'deploy.lock'}.items(): + monkeypatch.setattr(module, key, value) + module.PB_ENV.write_bytes(b'EXISTING_PRIVATE_VALUE=keep-me\nPROJECTBRAIN_APPOPS_MCP_URL=\nPROJECTBRAIN_APPOPS_MCP_TOKEN=\n') + module.PB_ENV.chmod(0o600) + module.ADMIN_TOKEN_FILE.write_text('admin-capability-separate-secret\n') + module.ADMIN_TOKEN_FILE.chmod(0o600) + module.project = {'jens/projectbrain': {'branch': 'main', 'slug': 'projectbrain', + 'source_dir': str(source), 'containers': ['projectbrain'], + 'deploy_script': 'scripts/deploy-unraid-single.sh', + 'validation': {'required': True, 'contexts': ['Managed validation / full (push)']}}} + return module + + +def invoke(binding): + return binding.configure(revision='a' * 40, projects=binding.project) + + +def test_configuration_is_separate_private_and_idempotent(binding): + before = binding.PB_ENV.read_bytes() + result = invoke(binding) + assert result['status'] == 'configured' + assert result['projectbrainDeliveryRequired'] is True + token = binding.TOKEN_FILE.read_text().strip() + assert len(token) == 64 + assert token != binding.ADMIN_TOKEN_FILE.read_text().strip() + assert token not in json.dumps(result) + assert binding._configured_values(binding.PB_ENV.read_bytes()) == { + binding.URL_KEY: binding.ENDPOINT, binding.TOKEN_KEY: token} + assert b'EXISTING_PRIVATE_VALUE=keep-me\n' in binding.PB_ENV.read_bytes() + assert (binding.BINDING_ROOT / 'environment-before').read_bytes() == before + for path in (binding.PB_ENV, binding.TOKEN_FILE, binding.BINDING_ROOT / 'receipt.json'): + assert path.stat().st_mode & 0o777 == 0o600 + token_info = binding.TOKEN_FILE.stat() + assert (token_info.st_uid, token_info.st_gid) == (binding.APPOPS_UID, binding.APPOPS_GID) + assert token_info.st_nlink == 1 + assert invoke(binding)['status'] == 'already_configured' + + +@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0, + reason='fixed AppOps ownership requires a privileged Linux controller') +def test_statistics_token_is_appops_readable_while_binding_state_stays_root_private(binding): + assert os.geteuid() != 100 + + result = invoke(binding) + + token_info = binding.TOKEN_FILE.stat() + assert (token_info.st_uid, token_info.st_gid) == (100, 101) + assert token_info.st_mode & 0o777 == 0o600 + assert token_info.st_nlink == 1 + + assert len(binding.TOKEN_FILE.read_text().strip()) == 64 + assert result['status'] == 'configured' + for path in (binding.PB_ENV, binding.BINDING_ROOT / 'receipt.json', + binding.BINDING_ROOT / 'environment-before'): + info = path.stat() + assert info.st_uid == binding.ROOT_UID + assert info.st_mode & 0o777 == 0o600 + assert info.st_nlink == 1 + assert invoke(binding)['status'] == 'already_configured' + token_info = binding.TOKEN_FILE.stat() + assert (token_info.st_uid, token_info.st_gid) == (100, 101) + assert token_info.st_mode & 0o777 == 0o600 + assert token_info.st_nlink == 1 + + +@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0, + reason='dropping to the fixed AppOps identity requires a privileged Linux controller') +def test_private_secret_reader_accepts_statistics_token_as_dropped_appops_identity(binding): + invoke(binding) + repository = Path(__file__).resolve().parents[2] + code = ( + 'import sys; from pathlib import Path; ' + 'sys.path.insert(0, sys.argv[2]); ' + 'from backend.app.core.config import _private_secret_file; ' + 'value = _private_secret_file(sys.argv[1]); ' + 'raise SystemExit(0 if value is not None and len(value) == 64 else 1)' + ) + + with tempfile.TemporaryDirectory(prefix='appops-statistics-reader-', dir='/tmp') as directory: + os.chmod(directory, 0o711) + token_copy = Path(directory) / 'runner-statistics-token' + token_copy.write_bytes(binding.TOKEN_FILE.read_bytes()) + source_info = binding.TOKEN_FILE.stat() + os.chown(token_copy, source_info.st_uid, source_info.st_gid) + token_copy.chmod(source_info.st_mode & 0o777) + + def drop_to_appops(): + os.setgroups([]) + os.setgid(101) + os.setuid(100) + + result = subprocess.run( + ['/usr/bin/python3', '-c', code, str(token_copy), str(repository)], + cwd=directory, preexec_fn=drop_to_appops, + capture_output=True, text=True, check=False, + ) + token = binding.TOKEN_FILE.read_text().strip() + assert token not in result.stdout and token not in result.stderr + assert result.stdout == '' + assert result.returncode == 0, ( + 'AppOps config private-secret reader rejected the fixed token: ' + + result.stderr[-500:] + ) + + +@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0, + reason='wrong-owner validation requires a privileged Linux controller') +def test_configured_root_owned_statistics_token_fails_closed(binding): + assert invoke(binding)['status'] == 'configured' + token = binding.TOKEN_FILE.read_bytes() + os.chown(binding.TOKEN_FILE, binding.ROOT_UID, os.getgid()) + + with pytest.raises(binding.StatisticsBindingError, match='file_untrusted'): + invoke(binding) + assert binding.TOKEN_FILE.read_bytes() == token + info = binding.TOKEN_FILE.stat() + assert (info.st_uid, info.st_gid) == (binding.ROOT_UID, os.getgid()) + + +@pytest.mark.parametrize('key', ['source_dir', 'deploy_script', 'containers', 'validation']) +def test_unknown_project_configuration_is_rejected_before_write(binding, key): + binding.project['jens/projectbrain'][key] = 'untrusted' + with pytest.raises(binding.StatisticsBindingError): + invoke(binding) + assert not binding.TOKEN_FILE.exists() + assert not binding.BINDING_ROOT.exists() + + +def test_existing_operator_configuration_is_preserved(binding): + before = b'PROJECTBRAIN_APPOPS_MCP_URL=https://operator.example/mcp\nOTHER=preserve\n' + binding.PB_ENV.write_bytes(before) + with pytest.raises(binding.StatisticsBindingError, match='existing_operator_configuration'): + invoke(binding) + assert binding.PB_ENV.read_bytes() == before + assert not binding.TOKEN_FILE.exists() + + +def test_existing_operator_token_is_never_adopted(binding): + binding.TOKEN_FILE.write_text('existing-operator-secret') + binding.TOKEN_FILE.chmod(0o600) + os.chown(binding.TOKEN_FILE, binding.APPOPS_UID, binding.APPOPS_GID) + with pytest.raises(binding.StatisticsBindingError, match='existing_operator_credential'): + invoke(binding) + assert binding.TOKEN_FILE.read_text() == 'existing-operator-secret' + + +@pytest.mark.parametrize('fault_path', ['token', 'environment', 'receipt']) +def test_crash_after_each_durable_boundary_recovers_same_credential(binding, monkeypatch, fault_path): + atomic = binding._atomic + interrupted = False + + def crash(path, value, expected): + nonlocal interrupted + atomic(path, value, expected) + selected = {'token': binding.TOKEN_FILE, 'environment': binding.PB_ENV, + 'receipt': binding.BINDING_ROOT / 'receipt.json'}[fault_path] + if path == selected and not interrupted: + interrupted = True + raise OSError('simulated interruption after fsync') + + monkeypatch.setattr(binding, '_atomic', crash) + with pytest.raises(OSError): + invoke(binding) + journal = json.loads((binding.BINDING_ROOT / 'receipt.json').read_bytes()) + expected = journal['token'] + monkeypatch.setattr(binding, '_atomic', atomic) + assert invoke(binding)['status'] == 'configured' + assert binding.TOKEN_FILE.read_text().strip() == expected + + +@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0, + reason='simulated ownership failure requires a privileged Linux controller') +def test_statistics_token_owner_failure_closes_fd_and_never_publishes(binding, monkeypatch, capsys): + opened_token_fds = [] + + def fail_chown(fd, uid, gid): + opened_token_fds.append(fd) + raise PermissionError('simulated AppOps ownership failure') + + monkeypatch.setattr(binding.os, 'fchown', fail_chown) + with pytest.raises(PermissionError): + invoke(binding) + + assert len(opened_token_fds) == 1 + with pytest.raises(OSError): + binding.os.fstat(opened_token_fds[0]) + assert not binding.TOKEN_FILE.exists() + assert [path.name for path in binding.APPOPS_DATA.iterdir()] == ['mcp-token'] + assert capsys.readouterr().out == '' + + +def test_crash_after_backup_before_receipt_has_safe_retry(binding, monkeypatch): + atomic = binding._atomic + + def crash(path, value, expected): + atomic(path, value, expected) + if path.name == 'environment-before': + raise OSError('interruption after backup') + + monkeypatch.setattr(binding, '_atomic', crash) + with pytest.raises(OSError): + invoke(binding) + monkeypatch.setattr(binding, '_atomic', atomic) + assert invoke(binding)['status'] == 'configured' + + +def test_pending_environment_cas_conflict_preserves_operator_change(binding, monkeypatch): + atomic = binding._atomic + + def crash(path, value, expected): + atomic(path, value, expected) + if path == binding.TOKEN_FILE: + raise OSError('interruption') + + monkeypatch.setattr(binding, '_atomic', crash) + with pytest.raises(OSError): + invoke(binding) + binding.PB_ENV.write_bytes(b'OPERATOR_CHANGED=value\n') + monkeypatch.setattr(binding, '_atomic', atomic) + with pytest.raises(binding.StatisticsBindingError, match='environment_cas_conflict'): + invoke(binding) + assert binding.PB_ENV.read_bytes() == b'OPERATOR_CHANGED=value\n' + + +def test_completed_binding_keeps_unrelated_later_operator_edits(binding): + invoke(binding) + with binding.PB_ENV.open('ab') as stream: + stream.write(b'NEW_OPERATOR_VALUE=preserved\n') + before = binding.PB_ENV.read_bytes() + assert invoke(binding)['status'] == 'already_configured' + assert binding.PB_ENV.read_bytes() == before + + +@pytest.mark.parametrize('unsafe', ['symlink', 'public_file', 'writable_anchor', 'duplicate_key']) +def test_unsafe_configuration_blocks_before_credential_write(binding, unsafe): + if unsafe == 'symlink': + target = binding.PB_ENV.with_name('other.env') + binding.PB_ENV.rename(target) + binding.PB_ENV.symlink_to(target) + elif unsafe == 'public_file': + binding.PB_ENV.chmod(0o644) + elif unsafe == 'writable_anchor': + binding.PB_SOURCE.chmod(0o777) + else: + binding.PB_ENV.write_text(f'{binding.URL_KEY}=\n{binding.URL_KEY}=\n') + with pytest.raises((binding.StatisticsBindingError, OSError)): + invoke(binding) + assert not binding.TOKEN_FILE.exists() + + +def test_active_deployment_lock_blocks_without_configuration_changes(binding): + import fcntl + + with binding.GLOBAL_LOCK.open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(binding.StatisticsBindingError, match='deployment_busy'): + invoke(binding) + assert not binding.TOKEN_FILE.exists() + + +@pytest.mark.parametrize('owner,accepted', [(100, True), (200, False)]) +def test_existing_fixed_admin_file_owner_contract(binding, monkeypatch, owner, accepted): + actual = binding.os.fstat + inode = binding.ADMIN_TOKEN_FILE.stat().st_ino + + def metadata(fd): + info = actual(fd) + if info.st_ino == inode: + fields = list(info) + fields[4] = owner + return os.stat_result(fields) + return info + + monkeypatch.setattr(binding.os, 'fstat', metadata) + if accepted: + assert invoke(binding)['status'] == 'configured' + else: + with pytest.raises(binding.StatisticsBindingError, match='file_untrusted'): + invoke(binding) + assert not binding.TOKEN_FILE.exists() + + +def test_parent_swap_cannot_redirect_a_root_read(binding, monkeypatch): + actual_open = binding.os.open + original = binding.PB_ENV.read_bytes() + moved = binding.PB_SOURCE.with_name('moved-brain') + attacker = binding.PB_SOURCE.with_name('attacker') + attacker.mkdir() + (attacker / '.env').write_bytes(b'ATTACKER=must-not-be-read\n') + (attacker / '.env').chmod(0o600) + swapped = False + + def swap(path, flags, *args, **kwargs): + nonlocal swapped + if path == '.env' and 'dir_fd' in kwargs and not swapped: + swapped = True + binding.PB_SOURCE.rename(moved) + binding.PB_SOURCE.symlink_to(attacker, target_is_directory=True) + return actual_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(binding.os, 'open', swap) + assert binding._read(binding.PB_ENV, binding.MAX_ENV_BYTES) == original + assert (attacker / '.env').read_bytes() == b'ATTACKER=must-not-be-read\n' + with pytest.raises(OSError): + binding._read(binding.PB_ENV, binding.MAX_ENV_BYTES) + + +def test_parent_swap_cannot_redirect_atomic_replace(binding, monkeypatch): + actual_replace = binding.os.replace + before = binding.PB_ENV.read_bytes() + updated = before + b'APPENDED=private\n' + moved = binding.PB_SOURCE.with_name('moved-brain') + attacker = binding.PB_SOURCE.with_name('attacker') + attacker.mkdir() + (attacker / '.env').write_bytes(b'ATTACKER=must-not-be-overwritten\n') + (attacker / '.env').chmod(0o600) + + def swap(source, destination, **kwargs): + assert source.startswith('.statistics-') + assert destination == '.env' + assert kwargs['src_dir_fd'] == kwargs['dst_dir_fd'] + binding.PB_SOURCE.rename(moved) + binding.PB_SOURCE.symlink_to(attacker, target_is_directory=True) + return actual_replace(source, destination, **kwargs) + + monkeypatch.setattr(binding.os, 'replace', swap) + binding._atomic(binding.PB_ENV, updated, before) + assert (moved / '.env').read_bytes() == updated + assert (attacker / '.env').read_bytes() == b'ATTACKER=must-not-be-overwritten\n' + with pytest.raises(OSError): + binding._read(binding.PB_ENV, binding.MAX_ENV_BYTES) + + +def test_parent_swap_cannot_replace_the_authentication_secret(binding, monkeypatch): + secret = b'original-administration-capability-1234567890' + binding.ADMIN_TOKEN_FILE.write_bytes(secret) + actual_open = binding.os.open + moved = binding.APPOPS_DATA.with_name('moved-appops') + attacker = binding.APPOPS_DATA.with_name('attacker-appops') + attacker.mkdir() + (attacker / 'mcp-token').write_bytes(b'attacker-chosen-authentication-capability-12345') + (attacker / 'mcp-token').chmod(0o600) + swapped = False + + def swap(path, flags, *args, **kwargs): + nonlocal swapped + if path == 'mcp-token' and 'dir_fd' in kwargs and not swapped: + swapped = True + binding.APPOPS_DATA.rename(moved) + binding.APPOPS_DATA.symlink_to(attacker, target_is_directory=True) + return actual_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(binding.os, 'open', swap) + assert binding.read_administration_secret() == secret + assert binding.read_administration_secret() is None diff --git a/backend/tests/test_runner_statistics_mcp.py b/backend/tests/test_runner_statistics_mcp.py new file mode 100644 index 0000000..b006ff5 --- /dev/null +++ b/backend/tests/test_runner_statistics_mcp.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException +from httpx import ASGITransport, AsyncClient +from mcp import Client + +from backend.app.core.config import Settings +from backend.app.core.security import require_service_request +from backend.app.runner_statistics_mcp import RUNNER_STATISTICS_TOOL, create_runner_statistics_mcp + + +STATISTICS_TOKEN = "runner-statistics-token-that-is-distinct-and-long" +GOVERNED_MCP_TOKEN = "governed-operations-token-that-is-distinct-and-long" + + +def _request(authorization: str | None = None): + from starlette.requests import Request + + headers = [(b"authorization", authorization.encode("ascii"))] if authorization else [] + return Request( + { + "type": "http", + "method": "POST", + "path": "/mcp-runner-statistics/", + "headers": headers, + "client": ("127.0.0.1", 12345), + } + ) + + +def _snapshot() -> dict[str, object]: + return { + "schemaVersion": 1, + "status": "healthy", + "scope": "ci-fleet", + "generatedAt": "2026-10-01T12:00:00+00:00", + "budget": { + "cpuLimit": 8, + "memoryLimitBytes": 17_179_869_184, + "activeJobLimit": 2, + "heavyJobLimit": 1, + "reservedCpu": 1, + "reservedMemoryBytes": 2_147_483_648, + "activeJobs": 0, + "heavyJobs": 0, + }, + "host": { + "cpuBusyPercent": None, + "loadRatio": None, + "availableMemoryBytes": None, + "cacheFreeBytes": None, + }, + "profiles": [], + "window": { + "startAt": None, + "endAt": None, + "completedJobs": 0, + "samples": 0, + "coverage": "unavailable", + }, + "history": [], + } + + +def test_separate_mcp_catalog_and_tool_are_read_only(): + server, _http_app = create_runner_statistics_mcp( + snapshot_provider=_snapshot, + allowed_hosts=["testserver"], + allowed_origins=[], + ) + + async def exercise() -> None: + async with Client(server) as client: + listing = await client.list_tools() + assert [tool.name for tool in listing.tools] == [RUNNER_STATISTICS_TOOL] + tool = listing.tools[0] + assert tool.annotations.read_only_hint is True + assert tool.annotations.destructive_hint is False + result = await client.call_tool(RUNNER_STATISTICS_TOOL, {}) + assert result.structured_content["scope"] == "ci-fleet" + unknown = await client.call_tool("container.restart", {"target": "anything"}) + assert unknown.is_error is True + + asyncio.run(exercise()) + + +def test_dedicated_service_auth_requires_strong_distinct_token(monkeypatch): + from backend.app.core import security + from backend.app.core.config import settings + + monkeypatch.setattr(security, "settings", settings) + monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: STATISTICS_TOKEN) + monkeypatch.setattr(settings, "mcp_token", GOVERNED_MCP_TOKEN) + assert require_service_request(_request(f"Bearer {STATISTICS_TOKEN}"), "mcp_runner_statistics") == "mcp_runner_statistics" + + with pytest.raises(HTTPException) as wrong: + require_service_request(_request(f"Bearer {GOVERNED_MCP_TOKEN}"), "mcp_runner_statistics") + assert wrong.value.status_code == 403 + + monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: GOVERNED_MCP_TOKEN) + with pytest.raises(HTTPException) as collision: + require_service_request(_request(f"Bearer {GOVERNED_MCP_TOKEN}"), "mcp_runner_statistics") + assert collision.value.status_code == 503 + + monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: "short") + with pytest.raises(HTTPException) as weak: + require_service_request(_request("Bearer short"), "mcp_runner_statistics") + assert weak.value.status_code == 503 + + +def test_statistics_token_file_is_private_and_uses_data_dir_default(tmp_path: Path, monkeypatch): + monkeypatch.delenv("APPOPS_RUNNER_STATISTICS_TOKEN_FILE", raising=False) + monkeypatch.delenv("APPOPS_RUNNER_STATISTICS_TOKEN", raising=False) + token_path = tmp_path / "runner-statistics-token" + token_path.write_text(STATISTICS_TOKEN, encoding="utf-8") + if __import__("os").name != "nt": + token_path.chmod(0o600) + + configured = Settings(data_dir=str(tmp_path), mcp_token=GOVERNED_MCP_TOKEN) + + assert configured.runner_statistics_token_file == str(token_path) + assert configured.read_runner_statistics_token() == STATISTICS_TOKEN + assert STATISTICS_TOKEN not in str(configured.redact_env_preview()) + + +def test_statistics_token_reader_rejects_oversized_and_symlink_files(tmp_path: Path): + from backend.app.core.config import _private_secret_file + + oversized = tmp_path / "oversized-token" + oversized.write_text("x" * 513, encoding="utf-8") + assert _private_secret_file(str(oversized)) is None + + target = tmp_path / "target-token" + target.write_text(STATISTICS_TOKEN, encoding="utf-8") + if __import__("os").name != "nt": + target.chmod(0o600) + linked = tmp_path / "linked-token" + try: + linked.symlink_to(target) + except (OSError, NotImplementedError): + pytest.skip("symlink creation is unavailable on this host") + assert _private_secret_file(str(linked)) is None + + +def test_token_file_can_be_provisioned_after_app_start_without_restart(tmp_path: Path, monkeypatch): + monkeypatch.delenv("APPOPS_RUNNER_STATISTICS_TOKEN_FILE", raising=False) + token_path = tmp_path / "runner-statistics-token" + configured = Settings(data_dir=str(tmp_path), mcp_token=GOVERNED_MCP_TOKEN) + + assert configured.read_runner_statistics_token() is None + token_path.write_text(STATISTICS_TOKEN, encoding="utf-8") + if __import__("os").name != "nt": + token_path.chmod(0o600) + assert configured.read_runner_statistics_token() == STATISTICS_TOKEN + + +def test_plaintext_environment_value_is_not_a_runner_statistics_credential(tmp_path: Path, monkeypatch): + monkeypatch.setenv("APPOPS_RUNNER_STATISTICS_TOKEN", STATISTICS_TOKEN) + monkeypatch.setenv("APPOPS_RUNNER_STATISTICS_TOKEN_FILE", str(tmp_path / "missing-token")) + configured = Settings(data_dir=str(tmp_path), mcp_token=GOVERNED_MCP_TOKEN) + assert configured.read_runner_statistics_token() is None + + +def test_readonly_transport_adds_only_validated_auth_public_host(monkeypatch): + from backend.app.core.config import settings + from backend.app.runner_statistics_mcp import _readonly_allowed_hosts, _readonly_allowed_origins + + monkeypatch.setattr(settings, "mcp_allowed_hosts", ["localhost:*"]) + monkeypatch.setattr(settings, "auth_public_url", "https://appops.example.test") + monkeypatch.setattr(settings, "frontend_origin", "https://frontend.example.test") + assert _readonly_allowed_hosts() == ["localhost:*", "appops.example.test"] + assert _readonly_allowed_origins() == ["https://frontend.example.test", "https://appops.example.test"] + assert settings.mcp_allowed_hosts == ["localhost:*"] + + monkeypatch.setattr(settings, "auth_public_url", "http://attacker.example.test") + assert _readonly_allowed_hosts() == ["localhost:*"] + assert _readonly_allowed_origins() == ["https://frontend.example.test"] + monkeypatch.setattr(settings, "auth_public_url", "https://attacker_host.example.test") + assert _readonly_allowed_hosts() == ["localhost:*"] + + +def test_transport_factory_uses_readonly_hosts_without_widening_full_mcp(monkeypatch): + from backend.app.core.config import settings + from mcp.server import MCPServer + from starlette.applications import Starlette + + captured: dict[str, object] = {} + + def capture_transport(self, **kwargs): + captured.update(kwargs) + return Starlette() + + monkeypatch.setattr(MCPServer, "streamable_http_app", capture_transport) + monkeypatch.setattr(settings, "mcp_allowed_hosts", ["localhost:*"]) + monkeypatch.setattr(settings, "auth_public_url", "https://appops.example.test") + create_runner_statistics_mcp() + + transport_security = captured["transport_security"] + assert transport_security.allowed_hosts == ["localhost:*", "appops.example.test"] + assert settings.mcp_allowed_hosts == ["localhost:*"] + + +def test_readonly_transport_preserves_explicit_port_and_ipv6(monkeypatch): + from backend.app.core.config import settings + from backend.app.runner_statistics_mcp import _readonly_allowed_hosts + + monkeypatch.setattr(settings, "mcp_allowed_hosts", []) + monkeypatch.setattr(settings, "auth_public_url", "https://[2001:db8::1]:9443") + assert _readonly_allowed_hosts() == ["[2001:db8::1]:9443"] + + +def test_main_http_boundary_rejects_cross_tokens_and_hides_missing_endpoint(monkeypatch): + from backend.app import main + from backend.app.core.config import settings + + def no_session_for_machine_token(_request): + raise HTTPException(401, "Sign in with Authentik or Microsoft Entra") + + monkeypatch.setattr(main, "authorize_request", no_session_for_machine_token) + monkeypatch.setattr(settings, "mcp_enabled", True) + monkeypatch.setattr(settings, "mcp_token", GOVERNED_MCP_TOKEN) + monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: STATISTICS_TOKEN) + monkeypatch.setattr(settings, "mcp_allowed_hosts", ["testserver"]) + + async def exercise() -> None: + transport = ASGITransport(app=main.app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + stats_with_main_token = await client.post( + "/mcp-runner-statistics/", + headers={"Authorization": f"Bearer {GOVERNED_MCP_TOKEN}"}, + json={}, + ) + main_with_stats_token = await client.post( + "/mcp/", + headers={"Authorization": f"Bearer {STATISTICS_TOKEN}"}, + json={}, + ) + native_api_with_stats_token = await client.get( + "/api", + headers={"Authorization": f"Bearer {STATISTICS_TOKEN}"}, + ) + assert stats_with_main_token.status_code == 403 + assert main_with_stats_token.status_code == 403 + assert native_api_with_stats_token.status_code == 401 + + monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: None) + disabled = await client.post("/mcp-runner-statistics/", json={}) + assert disabled.status_code == 404 + + asyncio.run(exercise()) + + +@pytest.mark.parametrize( + ("main_enabled", "expected"), + [(False, ["statistics"]), (True, ["main", "statistics"])], +) +def test_lifespan_always_runs_readonly_mcp_and_gates_governed_mcp(monkeypatch, main_enabled, expected): + from backend.app import main + from backend.app.core.config import settings + + entered: list[str] = [] + + @asynccontextmanager + async def run_manager(name: str): + entered.append(name) + try: + yield + finally: + entered.remove(name) + + monkeypatch.setattr(main, "appops_mcp", SimpleNamespace(session_manager=SimpleNamespace(run=lambda: run_manager("main")))) + monkeypatch.setattr(main, "run_runner_statistics_mcp", lambda: run_manager("statistics")) + monkeypatch.setattr(main, "init_db", lambda: None) + monkeypatch.setattr(main, "Session", lambda _engine: SimpleNamespace(__enter__=lambda self: self, __exit__=lambda *_args: False)) + monkeypatch.setattr(main, "restore_request_metrics", lambda _session: None) + monkeypatch.setattr(main, "restore_tool_metrics", lambda _session: None) + monkeypatch.setattr(main.knowledge_service, "ensure_index_current", lambda _session: None) + monkeypatch.setattr(main, "persist_observability", lambda: None) + monkeypatch.setattr(main, "set_session_running", lambda _value: None) + monkeypatch.setattr(settings, "mcp_enabled", main_enabled) + for field in ( + "notification_interval_seconds", + "connector_health_interval_seconds", + "observability_persist_interval_seconds", + "restore_drill_interval_seconds", + "mcp_canary_interval_seconds", + "discovery_interval_seconds", + ): + monkeypatch.setattr(settings, field, 0) + + async def exercise() -> None: + async with main.lifespan(main.app): + assert entered == expected + + asyncio.run(exercise()) diff --git a/backend/tests/test_runner_statistics_surface.py b/backend/tests/test_runner_statistics_surface.py new file mode 100644 index 0000000..cc17cd0 --- /dev/null +++ b/backend/tests/test_runner_statistics_surface.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import copy +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta + +import httpx +import pytest +from sqlmodel import Session, SQLModel, create_engine, select + +from backend.app.db.models import AuditEvent, JobRun +from backend.app.services import action_execution, deployment_broker, runner_statistics + + +def snapshot(): + at = datetime.now(UTC).isoformat() + return { + "schemaVersion": 1, "status": "healthy", "scope": "ci-fleet", "generatedAt": at, + "budget": {"cpuLimit": 8, "memoryLimitBytes": 16 * 1024**3, "activeJobLimit": 2, + "heavyJobLimit": 1, "reservedCpu": 2, "reservedMemoryBytes": 3 * 1024**3, + "activeJobs": 0, "heavyJobs": 0}, + "host": {"cpuBusyPercent": None, "loadRatio": None, + "availableMemoryBytes": None, "cacheFreeBytes": None}, + "profiles": [{"id": "checks", "displayName": "ITWorx CI ยท Checks", + "runnerName": "itworx-ci-checks", "labels": ["itworx-ci-checks"], + "status": "capacity_wait", "reason": "host_headroom", + "cpuLimit": 1, "memoryLimitBytes": 1536 * 1024**2, + "queueDepth": 1, "activeJobs": 0, "cpuUsed": None, + "memoryUsedBytes": None, "oldestQueuedSeconds": 12, + "completedJobs": 0, "p50WaitSeconds": None, "p95WaitSeconds": None, + "p50DurationSeconds": None, "p95DurationSeconds": None}], + "window": {"startAt": at, "endAt": at, "completedJobs": 0, "samples": 1, + "coverage": "bounded"}, + "history": [{"at": at, "activeJobs": 0, "queuedJobs": 1, + "cpuUsed": None, "memoryUsedBytes": None}], + } + + +@pytest.fixture(autouse=True) +def reset_cache(monkeypatch): + monkeypatch.setattr(runner_statistics, "_cached", None) + monkeypatch.setattr(runner_statistics, "_cached_at", 0.0) + + +def test_unknown_measurements_survive_and_private_fields_are_removed(): + payload = snapshot() + payload["rawJobs"] = [{"secret": "fixture-secret"}] + payload["profiles"][0]["workspacePath"] = "/private/credentials" + result = runner_statistics.sanitize_statistics(payload) + assert result["status"] == "healthy" + assert result["profiles"][0]["cpuUsed"] is None + assert result["profiles"][0]["queueDepth"] == 1 + assert "rawJobs" not in result + assert "workspacePath" not in result["profiles"][0] + + +def test_unattested_profile_limits_remain_unknown(): + payload = snapshot() + payload["status"] = "degraded" + payload["profiles"][0].update(cpuLimit=None, memoryLimitBytes=None) + result = runner_statistics.sanitize_statistics(payload) + assert result["status"] == "degraded" + assert result["profiles"][0]["cpuLimit"] is None + assert result["profiles"][0]["memoryLimitBytes"] is None + + +@pytest.mark.parametrize("mutation", [ + lambda p: p.update(schemaVersion=2), + lambda p: p.update(generatedAt="2026-10-01T12:00:00"), + lambda p: p.update(generatedAt=(datetime.now(UTC) + timedelta(minutes=1)).isoformat()), + lambda p: p["profiles"].append(copy.deepcopy(p["profiles"][0])), + lambda p: p["profiles"][0].update(cpuUsed=float("nan")), + lambda p: p["profiles"][0].update(activeJobs=True), + lambda p: p["profiles"][0].update(runnerName="secret-token"), + lambda p: p["profiles"][0].update(labels=["security-validation"]), + lambda p: p["profiles"][0].update(reason="Bearer fixture-secret-token"), + lambda p: p["window"].update(startAt=None), + lambda p: p["history"].append(copy.deepcopy(p["history"][0])), + lambda p: p.update(history=[p["history"][0]] * 289), +]) +def test_untrusted_or_ambiguous_statistics_fail_to_honest_unknown(mutation): + payload = snapshot() + mutation(payload) + result = runner_statistics.sanitize_statistics(payload) + assert result["status"] == "unavailable" + assert result["budget"]["cpuLimit"] is None + assert result["profiles"] == [] + + +def test_old_sample_is_degraded_without_retimestamping(): + payload = snapshot() + old = (datetime.now(UTC) - timedelta(minutes=2)).isoformat() + payload["generatedAt"] = old + payload["history"][0]["at"] = old + payload["window"].update(startAt=old, endAt=old) + result = runner_statistics.sanitize_statistics(payload) + assert result["status"] == "degraded" + assert result["generatedAt"] == old + + +def test_singleflight_cache_and_caller_mutation_isolation(monkeypatch): + calls = [] + + class Broker: + def is_configured(self): + return True + + def runner_fleet_statistics(self): + calls.append(1) + return snapshot() + + monkeypatch.setattr(runner_statistics, "DeploymentBrokerClient", Broker) + service = runner_statistics.RunnerStatisticsService() + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda _: service.snapshot(), range(8))) + assert calls == [1] + results[0]["budget"]["cpuLimit"] = 999 + assert service.snapshot()["budget"]["cpuLimit"] == 8 + + +@pytest.mark.parametrize("configured,raises,expected", [ + (False, False, "not_configured"), (True, True, "unavailable"), +]) +def test_unavailable_backend_never_leaks_error_or_invents_budget(monkeypatch, configured, raises, expected): + class Broker: + def is_configured(self): + return configured + + def runner_fleet_statistics(self): + assert raises + raise RuntimeError("Bearer fixture-private-secret /private/key") + + monkeypatch.setattr(runner_statistics, "DeploymentBrokerClient", Broker) + result = runner_statistics.RunnerStatisticsService().snapshot() + assert result["status"] == expected + assert result["budget"]["reservedCpu"] is None + assert "fixture-private" not in str(result) + + +def test_broker_statistics_fixed_read_path_has_deadline_and_no_credentials(monkeypatch): + calls = [] + real_client = httpx.Client + + def handle(request): + calls.append(request) + return httpx.Response(200, json=snapshot()) + + def client(**kwargs): + assert kwargs == {"timeout": 5, "verify": True, "follow_redirects": False} + return real_client(**kwargs, transport=httpx.MockTransport(handle)) + + monkeypatch.setattr(deployment_broker.httpx, "Client", client) + broker = deployment_broker.DeploymentBrokerClient(base_url="https://controller.test", verify_tls=True) + assert broker.runner_fleet_statistics()["schemaVersion"] == 1 + request = calls[0] + assert request.method == "GET" + assert str(request.url) == "https://controller.test/v1/runner-fleet/statistics" + assert "authorization" not in request.headers + assert not any(key.startswith("x-itworx") for key in request.headers) + assert broker._runner_fleet_read("/v1/arbitrary")["reason"] == "runner_fleet_path_invalid" + assert len(calls) == 1 + + +def test_specialize_broker_caller_cannot_choose_script_revision_or_target(monkeypatch): + calls = [] + broker = deployment_broker.DeploymentBrokerClient() + monkeypatch.setattr(broker, "_request", lambda *args, **kwargs: calls.append((args, kwargs)) or {}) + broker.runner_fleet_specialize(reason="x" * 1100) + args, kwargs = calls[0] + assert args == ("POST", "/v1/appops/runner-fleet/specialize") + assert kwargs["payload"] == {"reason": "x" * 1000} + + +@pytest.mark.parametrize("status,success", [("specialized", True), ("already_applied", True), + ("upgraded", False), ("failed", False)]) +def test_specialize_execution_persists_actual_success_or_failure(tmp_path, monkeypatch, status, success): + engine = create_engine(f"sqlite:///{tmp_path / 'specialize.db'}") + SQLModel.metadata.create_all(engine) + monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_fleet_specialize", + lambda self, **kwargs: {"status": status, "revision": "a" * 40}) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, action="runner.fleet.specialize", target="tower/runner-fleet-specialized", + source="chatgpt-mcp", risk="high", reason="approved fleet specialization", + original_policy_decision="allowed", + ) + job = session.get(JobRun, result["jobId"]) + audit = session.get(AuditEvent, result["auditId"]) + assert result["success"] is success + assert job.status == ("succeeded" if success else "failed") + assert audit.result == ("success" if success else "failed") + assert audit.target == "tower/runner-fleet-specialized" + assert ("verified" in result["message"]) is success + + +def test_wrong_specialization_target_is_blocked_before_job_or_broker(tmp_path, monkeypatch): + engine = create_engine(f"sqlite:///{tmp_path / 'blocked.db'}") + SQLModel.metadata.create_all(engine) + monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_fleet_specialize", + lambda *args, **kwargs: pytest.fail("must not call broker")) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, action="runner.fleet.specialize", target="other/runner", + source="chatgpt-mcp", risk="high", reason="wrong target", + original_policy_decision="allowed", + ) + assert result["policyDecision"] == "blocked" + assert session.exec(select(JobRun)).all() == [] + + +@pytest.mark.parametrize('status,success', [('configured', True), ('already_configured', True), + ('specialized', False), ('failed', False)]) +def test_statistics_configuration_persists_actual_status_without_secret(tmp_path, monkeypatch, status, success): + engine = create_engine(f"sqlite:///{tmp_path / 'binding.db'}") + SQLModel.metadata.create_all(engine) + monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, 'runner_statistics_configure', + lambda self, **kwargs: {'status': status, 'revision': 'a' * 40, 'projectbrainDeliveryRequired': True}) + with Session(engine) as session: + result = action_execution.execute_governed_action( + session, action='runner.statistics.configure', target='tower/projectbrain-runner-statistics', + source='chatgpt-mcp', risk='high', reason='configure read-only metrics', original_policy_decision='allowed') + assert result['success'] is success + assert session.get(JobRun, result['jobId']).status == ('succeeded' if success else 'failed') + assert session.get(AuditEvent, result['auditId']).target == 'tower/projectbrain-runner-statistics' + assert 'token' not in result + + +def test_statistics_configuration_broker_has_fixed_signed_body(monkeypatch): + calls = [] + broker = deployment_broker.DeploymentBrokerClient() + monkeypatch.setattr(broker, '_request', lambda *args, **kwargs: calls.append((args, kwargs)) or {}) + broker.runner_statistics_configure(reason='configure read-only stats') + assert calls[0][0] == ('POST', '/v1/appops/runner-statistics/configure') + assert calls[0][1]['payload'] == {'reason': 'configure read-only stats'} diff --git a/backend/tests/test_runner_task_image_producer.py b/backend/tests/test_runner_task_image_producer.py new file mode 100644 index 0000000..1734e67 --- /dev/null +++ b/backend/tests/test_runner_task_image_producer.py @@ -0,0 +1,230 @@ +"""Exercise the real producer script against Docker's inherited-label shape.""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +PROFILES = ("checks", "backend", "frontend", "security", "browser", "build") +# Captured from the SHA-verified immutable amd64 base image config c474d435... +BASE_LABELS = { + "io.buildah.version": "1.35.0", + "org.opencontainers.image.authors": "Gitea", + "org.opencontainers.image.created": "2026-10-01T00:01:57Z", + "org.opencontainers.image.description": "Official docker images used by Gitea runner to run workflows.", + "org.opencontainers.image.documentation": "https://gitea.com/gitea/runner-images/src/branch/main/README.md", + "org.opencontainers.image.licenses": "MIT", + "org.opencontainers.image.title": "Official Gitea Actions runner images", + "org.opencontainers.image.url": "https://gitea.com/gitea/runner-images", + "org.opencontainers.image.vendor": "Gitea", + "org.opencontainers.image.version": "v26.10.01", +} + + +def _run_producer(tmp_path: Path, mutation: str = ""): + repo = tmp_path / "repository" + repo.mkdir() + shutil.copytree(ROOT / "ops" / "runner-task-images", repo / "ops" / "runner-task-images") + code = repo / "ops" / "gitea-deploy-broker" + code.mkdir() + shutil.copyfile(ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py", code / "runner_profiles.py") + scripts = repo / "scripts" + scripts.mkdir() + shutil.copyfile( + ROOT / "scripts" / "normalize_ci_task_image_archive.py", + scripts / "normalize_ci_task_image_archive.py", + ) + for args in ( + ["init", "-q"], + ["add", "ops", "scripts"], + ["-c", "user.name=Producer test", "-c", "user.email=producer-test@localhost", "commit", "-qm", "fixture"], + ): + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], cwd=repo, text=True).strip() + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + docker = fake_bin / "docker" + docker.write_text( + f"#!{sys.executable}\n" + "import hashlib,io,json,os,sys,tarfile\n" + "from pathlib import Path\n" + "a=sys.argv[1:]; p=Path(os.environ['FAKE_DOCKER_ROWS'])\n" + "layer_buffer=io.BytesIO()\n" + "with tarfile.open(fileobj=layer_buffer,mode='w:') as layer_tar:\n" + " payload=b'producer fixture layer'; member=tarfile.TarInfo('fixture.txt'); member.size=len(payload); layer_tar.addfile(member,io.BytesIO(payload))\n" + "layer=layer_buffer.getvalue(); layer_sha=hashlib.sha256(layer).hexdigest()\n" + "if a==['info']: sys.exit(0)\n" + "if a[0]=='build':\n" + " builds=Path(os.environ['FAKE_DOCKER_BUILD_ARGS'])\n" + " argv=json.loads(builds.read_text()) if builds.exists() else []\n" + " argv.append(a); builds.write_text(json.dumps(argv))\n" + " pairs=[a[i+1].split('=',1) for i,x in enumerate(a) if x=='--build-arg']\n" + " b=dict(pairs); profile=b['PROFILE']; ref=a[a.index('--tag')+1]\n" + f" labels={BASE_LABELS!r}.copy()\n" + " labels.update({'org.opencontainers.image.source':'https://gitea.example.com/Jens/Unraid-AppOps-Gateway'," + "'org.opencontainers.image.revision':b['SOURCE_REVISION'],'tech.itworx.source-tree':b['SOURCE_TREE'],'tech.itworx.ci.profile':profile})\n" + " mutation=os.environ.get('FAKE_LABEL_MUTATION','')\n" + " if mutation=='extra': labels['unexpected.label']='unexpected'\n" + " if mutation=='missing-inherited': labels.pop('org.opencontainers.image.vendor')\n" + " if mutation=='changed-inherited': labels['org.opencontainers.image.created']='different'\n" + " if mutation=='wrong-source': labels['org.opencontainers.image.source']='https://example.invalid'\n" + " if mutation=='missing-profile': labels.pop('tech.itworx.ci.profile')\n" + " config={'architecture':'amd64','os':'linux','config':{'User':'ubuntu','Labels':labels},'rootfs':{'type':'layers','diff_ids':['sha256:'+layer_sha]}}\n" + " config_bytes=json.dumps(config,sort_keys=True,separators=(',',':')).encode(); image_id='sha256:'+hashlib.sha256(config_bytes).hexdigest()\n" + " rows=json.loads(p.read_text()) if p.exists() else []\n" + " rows.append({'Id':image_id,'Os':'linux','Architecture':'amd64','RepoTags':[ref],'Config':{'User':'ubuntu','Labels':labels}})\n" + " configs=Path(os.environ['FAKE_DOCKER_CONFIGS']); configs.mkdir(exist_ok=True); (configs/(profile+'.json')).write_bytes(config_bytes)\n" + " p.write_text(json.dumps(rows)); sys.exit(0)\n" + "if a[:2]==['image','inspect']: print(p.read_text()); sys.exit(0)\n" + "if a[:2]==['image','save']:\n" + " selected=set(a[a.index('--output')+2:]); rows=json.loads(p.read_text()); members={'blobs/sha256/'+layer_sha:layer}; manifest=[]\n" + " for row in rows:\n" + " if row['RepoTags'][0] not in selected: continue\n" + " reference=row['RepoTags'][0]; profile=reference.split('itworx-ci-',1)[1].split(':sha-',1)[0]; config=(Path(os.environ['FAKE_DOCKER_CONFIGS'])/(profile+'.json')).read_bytes(); config_sha=hashlib.sha256(config).hexdigest(); config_path='blobs/sha256/'+config_sha; members[config_path]=config\n" + " manifest.append({'Config':config_path,'RepoTags':[reference],'Layers':['blobs/sha256/'+layer_sha],'LayerSources':{'sha256:'+layer_sha:{'mediaType':'application/vnd.oci.image.layer.v1.tar','size':len(layer),'digest':'sha256:'+layer_sha}}})\n" + " members['manifest.json']=json.dumps(manifest,separators=(',',':')).encode(); members['oci-layout']=json.dumps({'imageLayoutVersion':'1.0.0'}).encode(); members['index.json']=json.dumps({'schemaVersion':2,'manifests':[]}).encode()\n" + " with tarfile.open(a[a.index('--output')+1],mode='w:') as archive:\n" + " for name,payload in members.items():\n" + " member=tarfile.TarInfo(name); member.size=len(payload); archive.addfile(member,io.BytesIO(payload))\n" + " sys.exit(0)\n" + "sys.exit('unexpected Docker invocation')\n", + encoding="utf-8", + ) + docker.chmod(0o755) + output = tmp_path / "output" + env = { + **os.environ, + "PATH": str(fake_bin) + os.pathsep + os.environ["PATH"], + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "Jens/Unraid-AppOps-Gateway", + "FAKE_DOCKER_ROWS": str(tmp_path / "docker-rows.json"), + "FAKE_DOCKER_BUILD_ARGS": str(tmp_path / "docker-build-args.json"), + "FAKE_DOCKER_CONFIGS": str(tmp_path / "docker-configs"), + "FAKE_LABEL_MUTATION": mutation, + } + Path(env["FAKE_DOCKER_CONFIGS"]).mkdir() + result = subprocess.run( + ["bash", str(ROOT / "scripts" / "build_ci_task_images.sh"), revision, "123", str(output)], + cwd=repo, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + return result, output, revision, tree + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +def test_producer_accepts_exact_pinned_base_metadata_and_keeps_provenance_manifest(tmp_path): + result, output, revision, tree = _run_producer(tmp_path) + assert result.returncode == 0, result.stderr + build_args = json.loads((tmp_path / "docker-build-args.json").read_text()) + assert len(build_args) == len(PROFILES) + expected_limits = { + "--cpu-period=100000", + "--cpu-quota=300000", + "--memory=5120m", + "--memory-swap=5120m", + "--cgroup-parent=docker", + "--rm=true", + "--force-rm=true", + } + profiles = [ + next( + args[index + 1] + for index, arg in enumerate(args[:-1]) + if arg == "--build-arg" and args[index + 1].startswith("PROFILE=") + ) + for args in build_args + ] + assert profiles == [f"PROFILE={profile}" for profile in PROFILES] + for args in build_args: + resource_flags = [ + arg + for arg in args + if arg.startswith( + ( + "--cpu-period", + "--cpu-quota", + "--memory", + "--cgroup-parent", + "--rm", + "--force-rm", + "--cpus", + ) + ) + ] + assert set(resource_flags) == expected_limits + assert len(resource_flags) == len(expected_limits) + manifest = json.loads((output / "task-image-manifest.json").read_text()) + assert manifest["sourceRevision"] == revision + assert manifest["sourceTree"] == tree + assert tuple(row["profile"] for row in manifest["images"]) == PROFILES + for row in manifest["images"]: + assert row["labels"] == { + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.revision": revision, + "tech.itworx.source-tree": tree, + "tech.itworx.ci.profile": row["profile"], + } + archive_path = output / "task-images.tar" + archive_bytes = archive_path.read_bytes() + assert manifest["dockerTarSha256"] == hashlib.sha256(archive_bytes).hexdigest() + assert manifest["dockerTarBytes"] == len(archive_bytes) + inspected = { + image["RepoTags"][0]: image + for image in json.loads((tmp_path / "docker-rows.json").read_text()) + } + with tarfile.open(archive_path, mode="r:") as archive: + members = {member.name: member for member in archive} + normalized = json.loads(archive.extractfile(members["manifest.json"]).read()) + assert len(normalized) == len(PROFILES) + assert all(set(item) == {"Config", "RepoTags", "Layers"} for item in normalized) + for item, row in zip(normalized, manifest["images"], strict=True): + assert item["RepoTags"] == [row["reference"]] + assert item["Config"] == row["imageId"].removeprefix("sha256:") + ".json" + assert len(item["Layers"]) == 1 + image = inspected[row["reference"]] + config_bytes = archive.extractfile(members[item["Config"]]).read() + config = json.loads(config_bytes) + assert "sha256:" + hashlib.sha256(config_bytes).hexdigest() == image["Id"] == row["imageId"] + assert config["config"]["User"] == "ubuntu" + assert config["config"]["Labels"] == image["Config"]["Labels"] == { + **BASE_LABELS, + **row["labels"], + } + assert config["rootfs"]["type"] == "layers" + assert len(config["rootfs"]["diff_ids"]) == 1 + diff_id = config["rootfs"]["diff_ids"][0] + layer_name = item["Layers"][0] + assert layer_name == diff_id.removeprefix("sha256:") + "/layer.tar" + layer_bytes = archive.extractfile(members[layer_name]).read() + assert "sha256:" + hashlib.sha256(layer_bytes).hexdigest() == diff_id + assert not any(name.startswith("blobs/") for name in members) + assert not (output / ".task-images.raw.tar").exists() + assert not (output / "images.inspect.json").exists() + + +@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script") +@pytest.mark.parametrize("mutation", ("extra", "missing-inherited", "changed-inherited", "wrong-source", "missing-profile")) +def test_producer_rejects_any_untrusted_image_label_change(tmp_path, mutation): + result, output, _revision, _tree = _run_producer(tmp_path, mutation) + assert result.returncode != 0 + expected_error = ( + "task image archive normalization failed: image inspection provenance invalid" + if mutation in {"wrong-source", "missing-profile"} + else "image identity verification failed" + ) + assert expected_error in result.stderr + assert not (output / "task-image-manifest.json").exists()