Jens/Unraid-AppOps-Gateway-Public
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Publish curated Unraid-AppOps-Gateway source from a506550ef83ef2d81651e37c98a89f7ee330c730

Browse Source
This commit is contained in:
Public Source Publisher committed 2026-10-03 13:35:10 +00:00
1 parent 2facb8de34
commit 767e66921e
68 files changed
+15919 -308
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

PUBLIC_SOURCE_EXPORT.md
+1 -1
View File
@@ -1,3 +1,3 @@
# Public source export
This source snapshot was generated from the private canonical repository at revision `2e53f628d284213b90ca2f9c5e121671f5933574`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
This source snapshot was generated from the private canonical repository at revision `a506550ef83ef2d81651e37c98a89f7ee330c730`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
PUBLIC_SOURCE_MANIFEST.json
+202 -87
View File
@@ -1,6 +1,6 @@
{
"schemaVersion": 1,
"sourceRevision": "2e53f628d284213b90ca2f9c5e121671f5933574",
"sourceRevision": "a506550ef83ef2d81651e37c98a89f7ee330c730",
"files": [
{
"path": ".dockerignore",
@@ -294,8 +294,8 @@
},
{
"path": "backend/app/connectors/gitea_actions.py",
"sha256": "711a92fcfdb0a27bd192d587d9b2597d2c64852f4343929a0e93f95042467323",
"bytes": 18661
"sha256": "b367bebc60e2542c7d7763c78e4eabfa98bca5d2be753e387c8faa269fdbe330",
"bytes": 18440
},
{
"path": "backend/app/connectors/gpu_metrics.py",
@@ -314,8 +314,8 @@
},
{
"path": "backend/app/connectors/portainer.py",
"sha256": "b22b3d36294d2aaa1dcbcc0e140d515893ec0ba58f095c5f6ccb6d82f161a517",
"bytes": 19001
"sha256": "dcbfb3926929b10e08190944acf0217a71d9696f0c8e91b24fe832bc71034f8f",
"bytes": 19655
},
{
"path": "backend/app/connectors/prowlarr.py",
@@ -354,8 +354,8 @@
},
{
"path": "backend/app/core/config.py",
"sha256": "c95e402679a238042c13d816d00a81ba4374d146eb13aab0b24ddb2f215fe8ea",
"bytes": 23746
"sha256": "83441a676b4e6f415f80b283897bb613805ecbf82a6d405d767532e086dbc5fc",
"bytes": 25697
},
{
"path": "backend/app/core/network_security.py",
@@ -369,8 +369,8 @@
},
{
"path": "backend/app/core/security.py",
"sha256": "2ddc677fe34e4818ba33cb287ad57df2caeb43849940f4d1ec760dab83be7bc3",
"bytes": 3257
"sha256": "abc3de16a802253ea3fa7e0bdaed51efb4d99a82e353ec062fdadb6c20e5e313",
"bytes": 3837
},
{
"path": "backend/app/core/time.py",
@@ -399,13 +399,18 @@
},
{
"path": "backend/app/main.py",
"sha256": "07ff0720575a518131ea6a80027cb515210bdf433f721c79050bbad0bddbf137",
"bytes": 17347
"sha256": "17aa07c08bbe9eecbc0c100a6b573ef7db4b682e9bd7c6e1cb5982f72966d853",
"bytes": 18104
},
{
"path": "backend/app/mcp_server.py",
"sha256": "75779ae0a366cca415c04fc6fe24f38d99c338939874acf17be6616cc12b4f25",
"bytes": 35237
"sha256": "7cb748e5350231bfe938c91867b7eef408dd3589ba4e7ec078e07589338554c6",
"bytes": 35626
},
{
"path": "backend/app/runner_statistics_mcp.py",
"sha256": "72b72bd512c8ace97ce3eea54ee4babaa0156044535f793411cd0228eb294ec8",
"bytes": 5860
},
{
"path": "backend/app/schemas/ai_remediation.py",
@@ -444,13 +449,13 @@
},
{
"path": "backend/app/services/action_execution.py",
"sha256": "fb32be9b89ac8235f826aabd8cb6ca0068fc9f5f2e5fc95a9963dac8c1df2cb3",
"bytes": 83876
"sha256": "3ba0a33cd6947dee1d7961e63e6e9a4b247eda3ea8253537b9f879a57cf063ac",
"bytes": 99269
},
{
"path": "backend/app/services/action_registry.py",
"sha256": "b34a8217b0bbdaf1a9cda8d7bc310f5577a3dea0abf4bcc95d106a8227dd8ea9",
"bytes": 20499
"sha256": "11add43608abc05d51d89d29d772fe17f6d429a032c471376ad8f6d8400976b2",
"bytes": 21035
},
{
"path": "backend/app/services/ai_remediation_planner.py",
@@ -492,10 +497,15 @@
"sha256": "320b7edc1cc47701216897929bb4197fe7cf496a61ef4efae477658482aaf185",
"bytes": 9150
},
{
"path": "backend/app/services/controller_artifact_helper_spec.py",
"sha256": "1da47a3f061ef0b89158c9f646bb7018ce88b81c0687d8b27abe65b74df67fea",
"bytes": 4948
},
{
"path": "backend/app/services/controller_artifact_promoter.py",
"sha256": "6e740205449af2adc1fbbf9adf15989532fc6a92e252c8dcc81e7b4d644e58cd",
"bytes": 16389
"sha256": "bc06a28941c111b5a0f6eed96e407259626c0b9fc1ba5ecb8953befc83b66304",
"bytes": 13602
},
{
"path": "backend/app/services/controller_bootstrap_import.py",
@@ -504,8 +514,8 @@
},
{
"path": "backend/app/services/deployment_broker.py",
"sha256": "2feb43b65684f3f6a2dba688a47c18117d907ca34426919270f67e4de42db453",
"bytes": 23266
"sha256": "829fe26916967dad3c6907989a5a8825a0aabbc294aa2cf600e701bc76c5e1e3",
"bytes": 24904
},
{
"path": "backend/app/services/deployment_control_plane.py",
@@ -514,8 +524,8 @@
},
{
"path": "backend/app/services/deployment_target_registry.py",
"sha256": "c3a454fee8613439d28728d97f7db2d80e0854bd8b48e9d4f72378d1b6f6068b",
"bytes": 20175
"sha256": "322208e3793306d72d47685ee1567a568a64078abe7547de3f2d0721ec888851",
"bytes": 20168
},
{
"path": "backend/app/services/discovery_scheduler.py",
@@ -599,8 +609,8 @@
},
{
"path": "backend/app/services/media_codec_dashboard_deployment_reconciliation.py",
"sha256": "78648b078b7525e15a8ac136e13914db1fae6835325901a3ba5b4965e7f1adbe",
"bytes": 43243
"sha256": "1f61df61d796e9d8aafa2f6dab137ccb8c5422e6e69a681d33362fe584c566e2",
"bytes": 43236
},
{
"path": "backend/app/services/media_pipeline_service.py",
@@ -644,13 +654,13 @@
},
{
"path": "backend/app/services/portfolio_deployment_enrollment.py",
"sha256": "04e7a3b9a603f9cf50476d9e3f2e67e46d8e169b0a6952eb3753dc6d059d6418",
"bytes": 33458
"sha256": "55acd53bf2a7b412935f61a60e5b21ad27eb591e6bfe0741fe85d4105e8a4d1e",
"bytes": 33452
},
{
"path": "backend/app/services/portfolio_deployment_recovery.py",
"sha256": "c1adb3734ca68eb961b61def1925cdb08a82a562767813064c170f16b2754865",
"bytes": 17107
"sha256": "f7bf495631ec35359f7adc47642b2ae2089ba2c9c8e73447664b6edcb9a7a8fb",
"bytes": 17108
},
{
"path": "backend/app/services/preflight_service.py",
@@ -664,18 +674,18 @@
},
{
"path": "backend/app/services/privatemuse_deployment_enrollment.py",
"sha256": "523834b875641603fb3f3aedf29bcd72ec3bcfa2a9c5f2c587961adddae96f19",
"bytes": 21876
"sha256": "fa12901f31b5ce13b6ad32669e7012906d1d713d6881e7d410b0ab10645e7116",
"bytes": 21869
},
{
"path": "backend/app/services/privatemuse_deployment_recovery.py",
"sha256": "36b5a74971ef746712b3d557d3a35001334838e6b892ffbce9da4ac713b38b04",
"bytes": 47799
"sha256": "df9cde501fa48bcad15387596c4c870cb8651ee8def93b92b65d274f1d7d4c9c",
"bytes": 47800
},
{
"path": "backend/app/services/privatemuse_webhook_repair.py",
"sha256": "114cf8f6ee8cc2e93cdadaacc1034209dd25c11407cac6740a61d14db7c25d8c",
"bytes": 11884
"sha256": "79a7ae42f57bb3e4b556230b9c7624fff8a072e80565c938992e242b6bf14839",
"bytes": 11886
},
{
"path": "backend/app/services/repopulse_adoption_journal.py",
@@ -689,8 +699,8 @@
},
{
"path": "backend/app/services/repopulse_adoption_preflight.py",
"sha256": "b92cdccfdd0458bd5a9debc49b12887105fc93d19b169027fa40a2b661d104e0",
"bytes": 37153
"sha256": "b993d8c12a16ffc720b207d9d5191ea41f603786e6c1f085895d431c799a02c8",
"bytes": 37146
},
{
"path": "backend/app/services/repopulse_adoption_runner.py",
@@ -732,10 +742,15 @@
"sha256": "2a10a4996456f5dec30b466fcb5c3fc375a31e6dd512df9ddf7c7ed055019ff3",
"bytes": 3212
},
{
"path": "backend/app/services/runner_statistics.py",
"sha256": "1f132221db4bcd2a7e122f0c0838ce2c8597e969dc912853444eb9a7fe52cbbf",
"bytes": 7735
},
{
"path": "backend/app/services/runnerless_target_provisioning.py",
"sha256": "99b503697365a750379bd6d15da3ee89d9fcc97c77c8aac2be6a98730cd2c3a9",
"bytes": 17552
"sha256": "ddadcbf82d6b37df18f415f969891ecf31726aef7a9d880c7becf9cc96eb6ffc",
"bytes": 17602
},
{
"path": "backend/app/services/runtime_service.py",
@@ -774,8 +789,8 @@
},
{
"path": "backend/app/services/workmesh_first_deployment.py",
"sha256": "4af175ab68a572779ada153783f417e1045321418ed41b87b4c226b2ed2a7a27",
"bytes": 22488
"sha256": "1ce6581b378bb9ee341c93047e8e5430f80fb053462bf83ad50d766b98abe06f",
"bytes": 22487
},
{
"path": "backend/app/services/workmesh_promotion.py",
@@ -809,38 +824,53 @@
},
{
"path": "backend/tests/test_appops_live_state_recovery.py",
"sha256": "33b5fa25e441128f84c4e518cef803fa9229ffe04a6ec4e980a3bb2707175959",
"bytes": 12368
"sha256": "e65bf9968fba65318295ea3d2395da8f7a567657d8050029bd8db323cad5a0dc",
"bytes": 12658
},
{
"path": "backend/tests/test_appops_migration_bridge.py",
"sha256": "11af3dfaf72d35affa1b26a52d7850c8e4d44d36183728f326e5e615bcbde102",
"bytes": 11481
},
{
"path": "backend/tests/test_appops_self_upgrade_controller.py",
"sha256": "ad6d8fba2213c1c8018bc9ff1c2ebd61684792c27fa00a07195c5d6aea99d69f",
"bytes": 20392
"sha256": "b41b936e6351a9e2a5a947b7d69e13eb05d2eb6b02bd5bee1b4d3fdfa2c63eea",
"bytes": 23121
},
{
"path": "backend/tests/test_authentik_connector.py",
"sha256": "1ea6eaea54f0ce6f5a1e92a6c338edb880eff4c6bc43f09a32ea7a0e4a537870",
"bytes": 28252
},
{
"path": "backend/tests/test_ci_job_docker.py",
"sha256": "7292a3ad3e52e934e6a410fdea8adb6a00dc90fe8de28d8159205ec069951d73",
"bytes": 27439
},
{
"path": "backend/tests/test_container_restart.py",
"sha256": "2b84085e0e35d4405f39a6b5274b0ba123191c5855306bc0bd7bb67f6c3b7822",
"bytes": 108066
"sha256": "6bbe8c44a07a0609873148c5b8a0775d375c92c2a5f44ffbfeafb1960d462cab",
"bytes": 108190
},
{
"path": "backend/tests/test_controller_artifact_bootstrap.py",
"sha256": "fcb29fae7778e00a3c84e6bcdb51a20ffaea73c24b5a8bd4304058825e30d359",
"bytes": 42896
},
{
"path": "backend/tests/test_controller_artifact_promoter.py",
"sha256": "671f62d61ec1281644d0b49500141dd4a50ef75b7bf82accbe80dd0bf16d04ec",
"bytes": 3922
"sha256": "8eee2bdaf60422824f605637b1820c431fdacea305e1b68b3618fffa0dc000ee",
"bytes": 7570
},
{
"path": "backend/tests/test_controller_bootstrap_import_host.py",
"sha256": "541f786746b42447fb738f14343fb6d1cb0097d10a5e4949f0e940e111a6aec3",
"bytes": 1577
"sha256": "58c153b8e3d1b016006b0bed5c98f9aa069d58e670d65edac368833ae38dd6f1",
"bytes": 3060
},
{
"path": "backend/tests/test_controller_config_paths.py",
"sha256": "9cbe851651abec8c5a792d9843ceca308993e71f67bdcbc081ae7eb9a96a0888",
"bytes": 5207
"sha256": "1081ecf7206e59be24aebf92284d3833743b1a84eeb1cefc3891a340dea883ae",
"bytes": 8254
},
{
"path": "backend/tests/test_controller_host_dependencies.py",
@@ -849,8 +879,8 @@
},
{
"path": "backend/tests/test_controller_hub_release_volume.py",
"sha256": "8c7dd4f95407c9b274f8ee6ca9c98088ee0e842afa0c537cbd1d7311ec9bfa30",
"bytes": 32553
"sha256": "2aedc6c8f97a9e6aaa58ab43c79c2189360e4f254519314558f77c2de72b25ee",
"bytes": 41287
},
{
"path": "backend/tests/test_controller_root_procedure_contract.py",
@@ -859,8 +889,8 @@
},
{
"path": "backend/tests/test_controller_state_reconciliation.py",
"sha256": "d00b8bcf388e17f0c1b8999d4b511fede7212b8ef83282905328d1b0b4b70f76",
"bytes": 10523
"sha256": "d9ddea9971414954a0b34738ef8de6bb330a867f7f41376063e473855ff5d17e",
"bytes": 10957
},
{
"path": "backend/tests/test_deluge_connector.py",
@@ -869,8 +899,8 @@
},
{
"path": "backend/tests/test_deploy_broker.py",
"sha256": "6bc23cb5883961316acae07daf12ec261e1c21e229bf5ee8ebc56ff245f6a34d",
"bytes": 309532
"sha256": "56dffec32ac7383b9d9cf0f82bfb7e586458cb2780bfefb0bfc5785b97bf4736",
"bytes": 381065
},
{
"path": "backend/tests/test_deployment_auto_rollback.py",
@@ -894,8 +924,8 @@
},
{
"path": "backend/tests/test_deployment_target_registry.py",
"sha256": "32c90b7c065183bbc0595f568e1729fc98c787a9fa8a158b082d15f90700fde8",
"bytes": 9437
"sha256": "2c28d05cbf6156d4ab53e2b52a770e2b0bda34138a3d715ba5d97bca0b9e8ce7",
"bytes": 9429
},
{
"path": "backend/tests/test_dockdeck_classifier.py",
@@ -929,8 +959,8 @@
},
{
"path": "backend/tests/test_gitea_actions.py",
"sha256": "b2c99ac7e5548b9394cce6ce6f8a3fd0e4b7bdf6c72006c95edb6aac108c35fd",
"bytes": 10766
"sha256": "7c4bebd43553b6de6738bdcb70153640a8ec6f2e855c09eac35a40345995b1d8",
"bytes": 12826
},
{
"path": "backend/tests/test_gitea_project_auto_registration.py",
@@ -947,6 +977,31 @@
"sha256": "718603f23478261e178bc4377fcea35322a59ff5cb2912e7d4a685fd9da7c2e4",
"bytes": 8311
},
{
"path": "backend/tests/test_host_guard_memory_threshold_installer.py",
"sha256": "c50a26646d73a5fe0279b5d9923f121c31fc2c59ee9b8f21cb2a34478729147c",
"bytes": 7846
},
{
"path": "backend/tests/test_hub_candidate_delivery.py",
"sha256": "582c987db50715930b79f0fe07d88e9acc79c41905c973e9935c865695a7e714",
"bytes": 9323
},
{
"path": "backend/tests/test_hub_candidate_import.py",
"sha256": "178a07b0dc9489545723c8ff5ecbfd7060aa0c1491ba6c1ced50edd7f3481e68",
"bytes": 30015
},
{
"path": "backend/tests/test_hub_candidate_transport.py",
"sha256": "da0b302e0bf45d87f0b3339b2c1392aec22e650f45468ab236fc37f448d9f9ea",
"bytes": 9353
},
{
"path": "backend/tests/test_hub_permission_preparation.py",
"sha256": "3ba8efe338a21b100e4bc20e4fbffe043eb57c411a4143c00332fdb368db5011",
"bytes": 28482
},
{
"path": "backend/tests/test_inventory_discovery.py",
"sha256": "db7a7fbe23ef01f2a4e2e58ca72514889b9740a04374851b4a414118908bc594",
@@ -984,8 +1039,8 @@
},
{
"path": "backend/tests/test_mcp_hub_candidate_preparation.py",
"sha256": "aa5525214e034464ac3586252ba5bad09f59c96c861cb9449a3dc86ae7a91c74",
"bytes": 22525
"sha256": "ed0d60e8f0e4371622405dc200e80ecfa3c4574ad2a9439373032f997a1af414",
"bytes": 27390
},
{
"path": "backend/tests/test_mcp_hub_deployment_reconciliation.py",
@@ -994,8 +1049,13 @@
},
{
"path": "backend/tests/test_mcp_hub_governed_topology.py",
"sha256": "117c07dfb34e9ab0fac5675dd51585906b33d7bf5ea0a059d2d8f6eb4d2897a7",
"bytes": 118901
"sha256": "9e306fbce479f15610a87ad666023de7deae588dc01f69be6024140d5df2158d",
"bytes": 125396
},
{
"path": "backend/tests/test_mcp_hub_legacy_registry_compatibility.py",
"sha256": "f382839f7ac5ec5a038f305bd9204e01514b73ad2b6116e6d388ec2d4f728b25",
"bytes": 4101
},
{
"path": "backend/tests/test_mcp_hub_policy_reconciliation.py",
@@ -1009,8 +1069,8 @@
},
{
"path": "backend/tests/test_mcp_hub_release_store.py",
"sha256": "91dbd0da6a131f76a04c51f93e31a77f4acd057f788f4b1a6fce75d37c23525e",
"bytes": 18199
"sha256": "4483020ccfe8cd31cc00ac4e9030a792af176e2d54929316a954bf738237f571",
"bytes": 21241
},
{
"path": "backend/tests/test_mcp_rag_n8n_integrations.py",
@@ -1019,8 +1079,8 @@
},
{
"path": "backend/tests/test_media_codec_dashboard_deployment_reconciliation.py",
"sha256": "a7fd78d28e191f3370aec320c88dff150682d3ca8acfcb5766b59ed6978e6e9c",
"bytes": 9161
"sha256": "7933221aedbaa0dd1b1c4e0fcde9987ec60d6b9f390ab5b3f59529e681a8d35a",
"bytes": 9088
},
{
"path": "backend/tests/test_media_codec_dashboard_reconcile_diagnostics.py",
@@ -1069,13 +1129,13 @@
},
{
"path": "backend/tests/test_portainer_restart_timeout.py",
"sha256": "3502178913145313afe376ccde8edcfc452ca723fa161aa572debba0b2f57a96",
"bytes": 1339
"sha256": "6ac7e5234de0d044b77c35390da14a26d26c99f21ecf35dd2e690ec8bcf28f3a",
"bytes": 4354
},
{
"path": "backend/tests/test_portfolio_deployment_enrollment.py",
"sha256": "3d70cc965d56db374ab88d1df3a25ac497807d0b1f0c3d385789b20fd8fe57e4",
"bytes": 6734
"sha256": "6ff8929444d3455db66955270dcd57750f9b3c4ab55c96daccc127386a68cb2e",
"bytes": 6727
},
{
"path": "backend/tests/test_portfolio_deployment_recovery.py",
@@ -1099,8 +1159,8 @@
},
{
"path": "backend/tests/test_privatemuse_deployment_enrollment.py",
"sha256": "2739d9e0d31db4eaec6c18c2da07a0ba16200d379274a8d9a47516110518241e",
"bytes": 7490
"sha256": "ec086f9dc6560066e7fe21a6052b0ec435ef9fcbd19f3a231a2d8cb07e1bf0e0",
"bytes": 7482
},
{
"path": "backend/tests/test_privatemuse_deployment_recovery.py",
@@ -1119,8 +1179,8 @@
},
{
"path": "backend/tests/test_privatemuse_webhook_repair.py",
"sha256": "2c6ba79e8cd3d63faf6103a8578d9cb8e9ecb7278008e9ad36037cc08a02c2a7",
"bytes": 6391
"sha256": "aa04a057b29302ae585751a5d4c5cdbb268e55350b6b6d20575c78808492a778",
"bytes": 6392
},
{
"path": "backend/tests/test_production_runtime.py",
@@ -1204,8 +1264,8 @@
},
{
"path": "backend/tests/test_repopulse_target_attestation.py",
"sha256": "6c0c61262aceb58b924edab70bfa079e4b46b222897e00c72e39f7a73bd5c9e6",
"bytes": 11564
"sha256": "1fccd197fbb8abe3c13187401b16ed21a44cc0f0137966f6427678a77a4d81f3",
"bytes": 14015
},
{
"path": "backend/tests/test_repopulse_target_credential.py",
@@ -1232,20 +1292,35 @@
"sha256": "7ea8409b3129938fd57390db26ba2e6068af90cf46db31204a0f0cd567c254ed",
"bytes": 3242
},
{
"path": "backend/tests/test_runner_admission_controller.py",
"sha256": "dcfb8e7a1d1ee86d6bfadef71bbd8be629a7f70a26baafb37bb134c61e78491d",
"bytes": 57459
},
{
"path": "backend/tests/test_runner_admission_policy.py",
"sha256": "e4bdf5b454b5e17d1785b2db82819c1e43150fe84de1bf0fd242288352f4ee26",
"bytes": 5951
},
{
"path": "backend/tests/test_runner_capacity_controller.py",
"sha256": "d757a661f06507ec7c685103de274c0dd26a987def676c70dd8b44be6e9c265d",
"bytes": 6909
"sha256": "fbc1025db04ab98f5aaeee7c94bb6b5dfb1a5652bfe696d92f2ac5f631620448",
"bytes": 18051
},
{
"path": "backend/tests/test_runner_capacity_policy.py",
"sha256": "662bc7a4a9f0ec4cd721e032068e08f4530409aa2fc03636a04077d4d9b5e6f3",
"bytes": 5292
},
{
"path": "backend/tests/test_runner_controller_artifact.py",
"sha256": "ec3a53cebab1fae3a68817798d72b2831f35eda4c46fbb3d07ad84fc5eb55ab3",
"bytes": 2830
},
{
"path": "backend/tests/test_runner_fleet.py",
"sha256": "a442a54748caf77f912d05b7ed9d68645db502d1a2a96cd1838501343bccdf90",
"bytes": 5966
"sha256": "88bfd354ceb8b7f7fbbd68142ed8b0dd14133de5844c89cedea72cb3909781db",
"bytes": 8826
},
{
"path": "backend/tests/test_runner_fleet_cutover_safety.py",
@@ -1254,19 +1329,59 @@
},
{
"path": "backend/tests/test_runner_fleet_scheduler_recovery.py",
"sha256": "567c59526c430c61c7823a984fb8c2a7eb17c5614e3e0a28a2b25b9af85fc55f",
"bytes": 34952
"sha256": "cc2e5c2b1076df701e4124e62fc9173b0ea3badb148e99ea5d54f84f40619568",
"bytes": 50645
},
{
"path": "backend/tests/test_runner_fleet_specialize.py",
"sha256": "e2b142ed5656e118156d44baf6f1f8568ba25297ea9af13b817861ce3c4fd4da",
"bytes": 107149
},
{
"path": "backend/tests/test_runner_fleet_v2_execution_surface.py",
"sha256": "62dcc692085ea9af004c4618e0c770701396be02254daf06bd5b55f56b5119fa",
"bytes": 2337
},
{
"path": "backend/tests/test_runner_profiles.py",
"sha256": "45a8dba20d1e8d778dc90c20a997630037e2eaa1d462fa62f48671285aa82e0e",
"bytes": 3111
},
{
"path": "backend/tests/test_runner_resource_envelope.py",
"sha256": "a0e469bc980dbec76cedda9a1ae5011fc1444d49ffed4c067268a10f1905a5a1",
"bytes": 40504
},
{
"path": "backend/tests/test_runner_scheduler_liveness_recovery.py",
"sha256": "86e9a1e336206506f48e149fedf21e79f9c7ad54771dea4f53ec039ac81548af",
"bytes": 3824
},
{
"path": "backend/tests/test_runner_specialization_controller_surface.py",
"sha256": "dc26476686237318953bf335865ad0901635044629826690af0e1d98d786612b",
"bytes": 9174
},
{
"path": "backend/tests/test_runner_statistics_binding.py",
"sha256": "2a684cc43097a0ac86270140a2f47273be23d33964ed6c75e4bb9e063a79b228",
"bytes": 16674
},
{
"path": "backend/tests/test_runner_statistics_mcp.py",
"sha256": "330d1ba91fb47d932a03b3657782e92087e6f93bd7b9fa3120e2d2e4f502e9f6",
"bytes": 12567
},
{
"path": "backend/tests/test_runner_statistics_surface.py",
"sha256": "a5786c51bf0aff6982b8491d4f3c5c81e1e249ad5aeb3398d720402b712b15bd",
"bytes": 11212
},
{
"path": "backend/tests/test_runner_task_image_producer.py",
"sha256": "c55f54bf58fa81649ddb31a8953abadf299c2fe32d1b4411dd59889981cc7376",
"bytes": 12112
},
{
"path": "backend/tests/test_secret_storage.py",
"sha256": "3e02994542763f4f7b2137356f883c48a17dc9ddb0af0efcd218e0d1c9ac6dce",
backend/app/connectors/gitea_actions.py
+7 -11
View File
@@ -217,7 +217,9 @@ class GiteaActionsConnector(BaseConnector):
"order": "asc",
},
)
pending_payload = self._request(
# Keep pending inventory in the all-or-error fetch, but it is not
# runnable queue pressure and must not drive runner diagnostics.
self._request(
"/api/v1/admin/actions/jobs",
{
"status": "pending",
@@ -267,9 +269,10 @@ class GiteaActionsConnector(BaseConnector):
]
queued_jobs = [
item
for payload in (queued_payload, pending_payload)
for item in (
payload.get("jobs") if isinstance(payload.get("jobs"), list) else []
queued_payload.get("jobs")
if isinstance(queued_payload.get("jobs"), list)
else []
)
if isinstance(item, dict)
]
@@ -462,8 +465,7 @@ class GiteaActionsConnector(BaseConnector):
"offline": sum(item["status"] != "online" for item in runners),
},
"queue": {
"queued": int(queued_payload.get("total_count", 0) or 0)
+ int(pending_payload.get("total_count", 0) or 0),
"queued": int(queued_payload.get("total_count", 0) or 0),
"inProgress": int(
active_payload.get("total_count", len(active_jobs)) or 0
),
@@ -476,12 +478,6 @@ class GiteaActionsConnector(BaseConnector):
if isinstance(queued_payload.get("jobs"), list)
else []
)
or int(pending_payload.get("total_count", 0) or 0)
> len(
pending_payload.get("jobs")
if isinstance(pending_payload.get("jobs"), list)
else []
)
),
"blockedByRunnerLabels": sorted(set(blocked_labels))[:20],
"staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20],
backend/app/connectors/portainer.py
+19 -1
View File
@@ -11,8 +11,22 @@ from ..core.config import settings
from ..core.redaction import redact_message
from .base import BaseConnector
_PORTAINER_DEFAULT_TIMEOUT_SECONDS = 8.0
_CONTROLLER_OWNED_CI_CONTAINER_NAMES = frozenset({
'itworx-ci-checks',
'itworx-ci-backend',
'itworx-ci-frontend',
'itworx-ci-security',
'itworx-ci-browser',
'itworx-ci-build',
'itworx-security-runner',
'itworx-validation-runner-2',
'itworx-validation-runner-3',
'itworx-docker-runner',
'itworx-browser-runner',
'itworx-validation-runner-burst-1',
'gitea-runner-dind',
})
@dataclass
@@ -213,6 +227,10 @@ class PortainerConnector(BaseConnector):
if not selected:
return {'status': 'not_found', 'reason': f'Container {target} not found'}
selected_name = str(selected.get('name', '')).strip().strip('/').lower()
if selected_name in _CONTROLLER_OWNED_CI_CONTAINER_NAMES:
return {'status': 'blocked', 'reason': 'controller_owned_ci_lifecycle'}
container_name = selected.get('id') or selected.get('name')
if not container_name:
return {'status': 'error', 'reason': 'Unable to resolve container identifier'}
backend/app/core/config.py
+46
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
import stat
from typing import Any
@@ -34,6 +36,38 @@ def _secret_value(name: str, file_name: str) -> str | None:
return None
def _private_secret_file(path: str | None) -> str | None:
"""Read a small owner-private regular secret file without following links."""
if not path:
return None
descriptor: int | None = None
try:
source = Path(path)
before = source.lstat()
if not source.is_absolute() or source.is_symlink() or not stat.S_ISREG(before.st_mode) or before.st_size > 512:
return None
descriptor = os.open(
source,
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0),
)
opened = os.fstat(descriptor)
if not stat.S_ISREG(opened.st_mode) or opened.st_size > 512:
return None
if os.name != "nt" and (
opened.st_uid != os.geteuid() or stat.S_IMODE(opened.st_mode) & 0o077
):
return None
value = os.read(descriptor, 513)
if len(value) > 512:
return None
return value.decode("utf-8").strip() or None
except (OSError, UnicodeDecodeError):
return None
finally:
if descriptor is not None:
os.close(descriptor)
def _env_list(name: str, default: list[str] | None = None) -> list[str]:
value = os.getenv(name)
if value is None:
@@ -86,6 +120,7 @@ class Settings:
mcp_token: str | None = None
mcp_token_file: str | None = None
mcp_allowed_hosts: list[str] | None = None
runner_statistics_token_file: str | None = None
n8n_enabled: bool = False
n8n_token: str | None = None
n8n_token_file: str | None = None
@@ -120,6 +155,11 @@ class Settings:
self.mcp_token = token_file.read().strip() or None
except OSError:
self.mcp_token = None
default_runner_statistics_token_file = str(Path(self.data_dir) / "runner-statistics-token")
self.runner_statistics_token_file = _env_value(
"APPOPS_RUNNER_STATISTICS_TOKEN_FILE",
self.runner_statistics_token_file or default_runner_statistics_token_file,
)
allowed_hosts = _env_value("APPOPS_MCP_ALLOWED_HOSTS", "localhost:*,127.0.0.1:*,198.18.10.150:*") or ""
self.mcp_allowed_hosts = [item.strip() for item in allowed_hosts.split(",") if item.strip()]
self.n8n_enabled = _env_bool("APPOPS_ENABLE_N8N", self.n8n_enabled)
@@ -267,6 +307,10 @@ class Settings:
"APPOPS_VERIFY_TLS": self.verify_tls,
}
def read_runner_statistics_token(self) -> str | None:
"""Read the fixed private credential at request time for no-restart provisioning."""
return _private_secret_file(self.runner_statistics_token_file)
def redact_env_preview(self) -> dict[str, Any]:
return {
"APPOPS_BIND_HOST": self.bind_host,
@@ -291,6 +335,8 @@ class Settings:
"APPOPS_MCP_TOKEN_FILE": bool(self.mcp_token_file),
"APPOPS_MCP_TOKEN_CONFIGURED": bool(self.mcp_token),
"APPOPS_MCP_ALLOWED_HOSTS": list(self.mcp_allowed_hosts or []),
"APPOPS_RUNNER_STATISTICS_TOKEN_FILE": bool(self.runner_statistics_token_file),
"APPOPS_RUNNER_STATISTICS_TOKEN_CONFIGURED": bool(self.read_runner_statistics_token()),
"APPOPS_ENABLE_N8N": self.n8n_enabled,
"APPOPS_N8N_TOKEN_FILE": bool(self.n8n_token_file),
"APPOPS_N8N_TOKEN_CONFIGURED": bool(self.n8n_token),
backend/app/core/security.py
+9
View File
@@ -53,6 +53,15 @@ def require_service_request(request: Request, service: str) -> str:
enabled, expected = settings.mcp_enabled, settings.mcp_token
elif service == "n8n":
enabled, expected = settings.n8n_enabled, settings.n8n_token
elif service == "mcp_runner_statistics":
expected = settings.read_runner_statistics_token()
if not expected:
raise HTTPException(404, "runner statistics MCP integration is disabled")
if not token_is_strong(expected):
raise HTTPException(503, "runner statistics MCP authentication is not safely configured")
if settings.mcp_token and hmac.compare_digest(expected, settings.mcp_token):
raise HTTPException(503, "runner statistics MCP token must be separate from the governed MCP token")
enabled = True
elif service == "voice":
enabled, expected = True, settings.voice_token or os.getenv("APP_OPS_VOICE_TOKEN")
else:
backend/app/main.py
+13 -3
View File
@@ -73,6 +73,7 @@ from .mcp_server import (
restore_tool_metrics,
set_session_running,
)
from .runner_statistics_mcp import runner_statistics_mcp_http_app, run_runner_statistics_mcp
from .services import knowledge_service
from .services.discovery_scheduler import run_periodic_discovery
from .services.maintenance_scheduler import (
@@ -130,6 +131,7 @@ async def lifespan(_: FastAPI):
if settings.mcp_enabled:
await stack.enter_async_context(appops_mcp.session_manager.run())
set_session_running(True)
await stack.enter_async_context(run_runner_statistics_mcp())
try:
yield
finally:
@@ -182,6 +184,7 @@ class MachineBodyLimitMiddleware:
method = str(scope.get("method", "GET")).upper()
is_machine_post = method not in _SAFE_METHODS and (
path == "/mcp" or path.startswith("/mcp/")
or path == "/mcp-runner-statistics" or path.startswith("/mcp-runner-statistics/")
or path == "/api/integrations/n8n" or path.startswith("/api/integrations/n8n/")
)
if scope["type"] != "http" or not is_machine_post:
@@ -243,17 +246,23 @@ async def private_operator_boundary(request, call_next):
is_api = path == "/api" or path.startswith("/api/")
is_voice = path.startswith("/api/voice/")
is_mcp = path == "/mcp" or path.startswith("/mcp/")
is_runner_statistics_mcp = path == "/mcp-runner-statistics" or path.startswith("/mcp-runner-statistics/")
is_n8n = path == "/api/integrations/n8n" or path.startswith("/api/integrations/n8n/")
is_public_auth = request.method in _SAFE_METHODS and (
path in {"/api/auth/session", "/api/auth/providers", "/api/health"}
or path.startswith(("/api/auth/login/", "/api/auth/callback/"))
)
if is_mcp or is_n8n:
if is_runner_statistics_mcp:
try:
require_service_request(request, "mcp_runner_statistics")
except HTTPException as exc:
return finalize_response(JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}))
elif is_mcp or is_n8n:
try:
require_service_request(request, "mcp" if is_mcp else "n8n")
except HTTPException as exc:
return finalize_response(JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}))
if is_mcp or is_n8n:
if is_mcp or is_runner_statistics_mcp or is_n8n:
content_length = request.headers.get("content-length")
if content_length and content_length.isdigit() and int(content_length) > 1_048_576:
return finalize_response(JSONResponse(status_code=413, content={"detail": "Request body exceeds 1 MiB limit"}))
@@ -297,7 +306,7 @@ async def private_operator_boundary(request, call_next):
response = await call_next(request)
finally:
actor_identity.reset(actor_token)
if is_api:
if is_api or is_runner_statistics_mcp:
response.headers.setdefault("Cache-Control", "no-store")
return finalize_response(response)
@@ -356,6 +365,7 @@ app.include_router(integrations_router)
app.include_router(n8n_router)
app.include_router(knowledge_router)
app.mount("/mcp", mcp_http_app)
app.mount("/mcp-runner-statistics", runner_statistics_mcp_http_app)
@app.get('/api')
backend/app/mcp_server.py
+8
View File
@@ -46,6 +46,7 @@ from .services.privatemuse_artifact_installer import ArtifactInstallError
from .services.privatemuse_artifact_installer import install as install_artifact_file
from .services.privatemuse_artifact_installer import preflight as artifact_preflight
from .services.runtime_service import runtime_info
from .services.runner_statistics import RunnerStatisticsService
_SESSION_RUNNING = False
_SESSION_VERIFIED_AT: str | None = None
@@ -233,6 +234,13 @@ def gitea_actions_runners_status() -> dict[str, Any]:
return _json(GiteaActionsConnector().snapshot())
@appops_mcp.tool(name="gitea.actions.runners.statistics", annotations=_READ_ONLY_TOOL)
@_tracked("gitea.actions.runners.statistics")
def gitea_actions_runner_statistics() -> dict[str, Any]:
"""Read bounded CI profile budgets, measured usage and job statistics."""
return _json(RunnerStatisticsService().snapshot())
def _json(value: Any) -> Any:
if hasattr(value, "model_dump"):
return _json(value.model_dump(mode="json"))
backend/app/runner_statistics_mcp.py
+164
View File
@@ -0,0 +1,164 @@
"""Dedicated one-tool MCP surface for bounded, read-only CI statistics."""
from __future__ import annotations
from collections.abc import Callable, Sequence
from contextlib import asynccontextmanager
import ipaddress
import re
from typing import Any
from urllib.parse import urlsplit
from mcp.server import MCPServer
from mcp.server.transport_security import TransportSecuritySettings
from mcp.types import ToolAnnotations
from starlette.applications import Starlette
from starlette.responses import Response
from starlette.types import ASGIApp, Receive, Scope, Send
from .core.config import settings
from .core.version import APP_VERSION
from .services.runner_statistics import RunnerStatisticsService, sanitize_statistics, unavailable_statistics
RUNNER_STATISTICS_TOOL = "gitea.actions.runners.statistics"
_READ_ONLY_TOOL = ToolAnnotations(
read_only_hint=True,
destructive_hint=False,
idempotent_hint=True,
open_world_hint=False,
)
_DNS_HOST_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)(?:\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?))*$")
def _validated_auth_public_host() -> tuple[str, int | None] | None:
try:
parsed = urlsplit(settings.auth_public_url)
hostname = parsed.hostname
port = parsed.port
except ValueError:
return None
if (
parsed.scheme.lower() != "https"
or not hostname
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
or (port is not None and not 1 <= port <= 65535)
):
return None
try:
normalized = hostname.rstrip(".").encode("idna").decode("ascii").lower()
except UnicodeError:
return None
if not normalized:
return None
if ":" in normalized:
try:
ipaddress.IPv6Address(normalized)
except ValueError:
return None
elif len(normalized) > 253 or not _DNS_HOST_PATTERN.fullmatch(normalized):
return None
return normalized, port
def _readonly_allowed_hosts() -> list[str]:
"""Add only the validated public auth host to this read-only transport."""
hosts = list(settings.mcp_allowed_hosts or [])
validated = _validated_auth_public_host()
if validated is None:
return hosts
normalized, port = validated
host_value = f"[{normalized}]" if ":" in normalized else normalized
candidates = [f"{host_value}:{port}" if port is not None else host_value]
for candidate in candidates:
if candidate not in hosts:
hosts.append(candidate)
return hosts
def _readonly_allowed_origins() -> list[str]:
"""Keep existing frontend origin and admit only the validated auth origin."""
origins = [settings.frontend_origin] if settings.frontend_origin else []
validated = _validated_auth_public_host()
if validated is None:
return origins
normalized, port = validated
host_value = f"[{normalized}]" if ":" in normalized else normalized
netloc = f"{host_value}:{port}" if port is not None else host_value
origin = f"https://{netloc}"
if origin not in origins:
origins.append(origin)
return origins
def create_runner_statistics_mcp(
*,
snapshot_provider: Callable[[], object] | None = None,
allowed_hosts: Sequence[str] | None = None,
allowed_origins: Sequence[str] | None = None,
) -> tuple[MCPServer, Starlette]:
"""Build an isolated MCP server exposing only the sanitized statistics tool."""
provider = snapshot_provider or RunnerStatisticsService().snapshot
server = MCPServer(
"AppOps CI Runner Statistics",
version=APP_VERSION,
instructions=(
"Read-only CI runner profile statistics. This server exposes no operational, "
"administrative, deployment, filesystem, or mutation tools."
),
)
@server.tool(name=RUNNER_STATISTICS_TOOL, annotations=_READ_ONLY_TOOL)
def gitea_actions_runner_statistics() -> dict[str, Any]:
"""Read bounded runner budgets, measured usage, and job statistics."""
try:
return sanitize_statistics(provider())
except Exception: # noqa: BLE001 - keep internal broker details off the MCP surface.
return unavailable_statistics()
transport = server.streamable_http_app(
streamable_http_path="/",
stateless_http=True,
json_response=True,
max_request_body_size=1_048_576,
transport_security=TransportSecuritySettings(
enable_dns_rebinding_protection=True,
allowed_hosts=_readonly_allowed_hosts() if allowed_hosts is None else list(allowed_hosts),
allowed_origins=_readonly_allowed_origins() if allowed_origins is None else list(allowed_origins),
),
)
return server, transport
class _ReplaceableASGIApp:
"""Keep the mounted path stable while giving each app lifespan a fresh manager."""
def __init__(self) -> None:
self._app: ASGIApp | None = None
def replace(self, app: ASGIApp | None) -> None:
self._app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if self._app is None:
response = Response("Runner statistics MCP is unavailable", status_code=503)
await response(scope, receive, send)
return
await self._app(scope, receive, send)
runner_statistics_mcp_http_app = _ReplaceableASGIApp()
@asynccontextmanager
async def run_runner_statistics_mcp():
"""Run a fresh stateless transport manager for this FastAPI lifespan."""
server, transport = create_runner_statistics_mcp()
runner_statistics_mcp_http_app.replace(transport)
try:
async with server.session_manager.run():
yield
finally:
runner_statistics_mcp_http_app.replace(None)
backend/app/services/action_execution.py
+355 -10
View File
@@ -53,6 +53,9 @@ APPROVAL_EXECUTOR_ACTIONS = {
"container.restart",
"controller.artifact.promote",
"runner.fleet.upgrade_v2",
"runner.fleet.specialize",
"runner.statistics.configure",
"hub.migration.prepare_permissions",
"deployment.targets.sync",
"deployment.runnerless.provision",
"deployment.target.deploy_workmesh",
@@ -1648,18 +1651,27 @@ def _execute_runner_fleet_upgrade_v2(
risk: str,
reason: str,
original_policy_decision: str,
specialized: bool = False,
statistics_configuration: bool = False,
) -> dict[str, Any]:
from .deployment_broker import DeploymentBrokerClient
action = "runner.fleet.upgrade_v2"
target = "tower/runner-fleet-v2"
action = ("runner.statistics.configure" if statistics_configuration else
"runner.fleet.specialize" if specialized else "runner.fleet.upgrade_v2")
target = ("tower/projectbrain-runner-statistics" if statistics_configuration else
"tower/runner-fleet-specialized" if specialized else "tower/runner-fleet-v2")
steps = [
"Validate exact fixed runner-fleet target",
"Call HMAC-signed root controller endpoint",
"Prove protected main and successful push validation",
"Execute exact-revision fleet-v2 migration script",
"Execute exact-revision specialized migration script" if specialized
else "Execute exact-revision fleet-v2 migration script",
"Verify specialized runner topology",
]
if statistics_configuration:
steps = ["Verify fixed read-only statistics target", "Prove protected controller/runtime source",
"Lock production configuration", "Provision private independent credential with durable CAS",
"Configure ProjectBrain for its governed delivery"]
job = create_job(
session,
action_id=action,
@@ -1670,24 +1682,31 @@ def _execute_runner_fleet_upgrade_v2(
steps=steps,
)
start_job(session, job)
set_job_progress(session, job, 25, "Requesting fixed Tower runner-fleet v2 migration.")
set_job_progress(session, job, 25, f"Requesting {action} through the fixed controller endpoint.")
try:
result = DeploymentBrokerClient().runner_fleet_upgrade_v2(reason=reason)
broker = DeploymentBrokerClient()
result = (broker.runner_statistics_configure(reason=reason) if statistics_configuration else
broker.runner_fleet_specialize(reason=reason) if specialized else broker.runner_fleet_upgrade_v2(reason=reason))
status = str(result.get("status") or "error")
success = status in {"upgraded", "already_applied"}
success = status in ({"configured", "already_configured"} if statistics_configuration else
{"specialized", "already_applied"} if specialized else {"upgraded", "already_applied"})
message = str(
result.get("reason")
or (
"Tower runner-fleet v2 migration verified."
if success
else "Tower runner-fleet v2 migration was rejected or failed."
("ProjectBrain read-only statistics configuration verified." if statistics_configuration else
"Tower runner specialization verified." if specialized
else "Tower runner-fleet v2 migration verified.")
if success else
("ProjectBrain statistics configuration was rejected or failed." if statistics_configuration else
"Tower runner specialization was rejected or failed." if specialized
else "Tower runner-fleet v2 migration was rejected or failed.")
)
)
except Exception as exc:
result = {"status": "error"}
status = "error"
success = False
message = f"{type(exc).__name__}: runner fleet v2 migration failed"
message = f"{type(exc).__name__}: fixed controller operation failed"
completed = complete_job(
session,
@@ -1711,6 +1730,7 @@ def _execute_runner_fleet_upgrade_v2(
"status": status,
"revision": result.get("revision"),
"verifiedRunners": result.get("verifiedRunners"),
"projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"),
},
redacted_metadata={
"executor": action,
@@ -1732,6 +1752,253 @@ def _execute_runner_fleet_upgrade_v2(
"controllerStatus": status,
"revision": result.get("revision"),
"verifiedRunners": result.get("verifiedRunners"),
"projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"),
}
def _execute_hub_migration_prepare_permissions(
session: Session,
*,
source: str,
risk: str,
reason: str,
original_policy_decision: str,
) -> dict[str, Any]:
from .deployment_broker import DeploymentBrokerClient
action = "hub.migration.prepare_permissions"
target = "tower/itworx-mcp-hub-permissions"
state_file = "/mnt/cache/appdata/gitea-deploy-runners/state/itworx-mcp-hub.json"
fixed_targets = {state_file}
steps = [
"Validate the exact fixed MCP Hub permission target",
"Authenticate the fixed root-controller request with HMAC and replay protection",
"Verify the running controller is the protected current main release with full-push validation",
"Reinspect the fixed Hub state file and shared deployment lock",
"Apply only the fixed state-file mode correction and verify its final metadata",
]
job = create_job(
session,
action_id=action,
target=target,
source=source,
risk=risk,
policy_decision="allowed",
steps=steps,
)
start_job(session, job)
set_job_progress(session, job, 25, "Requesting fixed MCP Hub permission preparation through the controller.")
try:
result = DeploymentBrokerClient().hub_migration_prepare_permissions(reason=reason)
status = str(result.get("status") or "blocked")
revision_value = result.get("revision")
revision = (
revision_value
if isinstance(revision_value, str)
and len(revision_value) == 40
and all(character in "0123456789abcdef" for character in revision_value)
else None
)
changed_value = result.get("changedTargets", [])
changed_targets = (
changed_value
if isinstance(changed_value, list)
and all(isinstance(path, str) and path in fixed_targets for path in changed_value)
and len(changed_value) == len(set(changed_value))
else None
)
changed_details_value = result.get("changedTargetDetails", [])
target_names = {
state_file: "state_file",
}
changed_details = (
changed_details_value
if isinstance(changed_details_value, list)
and all(
isinstance(item, dict)
and set(item) == {"name", "path", "observedMode", "sameIdentityAtPath"}
and isinstance(item.get("path"), str)
and item.get("path") in fixed_targets
and item.get("name") == target_names.get(item.get("path"))
and isinstance(item.get("observedMode"), str)
and len(item["observedMode"]) == 4
and item["observedMode"].startswith("0")
and all(character in "01234567" for character in item["observedMode"])
and isinstance(item.get("sameIdentityAtPath"), bool)
and changed_targets is not None
and item["path"] in changed_targets
for item in changed_details_value
)
and len({item["path"] for item in changed_details_value}) == len(changed_details_value)
else None
)
uncertain_value = result.get("uncertainTargets", [])
uncertain_targets = (
uncertain_value
if isinstance(uncertain_value, list)
and all(isinstance(path, str) and path in fixed_targets for path in uncertain_value)
and len(uncertain_value) == len(set(uncertain_value))
else None
)
expected_verified = {
"state_file": {
"path": state_file,
"uid": 0,
"gid": 0,
"mode": "0600",
"aclPresent": False,
},
}
verified_value = result.get("verifiedTargets")
verified_targets = None
if isinstance(verified_value, list) and len(verified_value) == len(expected_verified):
by_name = {
item.get("name"): item
for item in verified_value
if isinstance(item, dict) and isinstance(item.get("name"), str)
}
if set(by_name) == set(expected_verified) and all(
all(by_name[name].get(key) == value for key, value in expected.items())
for name, expected in expected_verified.items()
):
verified_targets = [
{"name": name, **expected_verified[name]}
for name in ("state_file",)
]
has_controller_projection = all(
key in result
for key in (
"revision",
"changedTargets",
"changedTargetDetails",
"uncertainTargets",
)
)
success = (
status in {"prepared", "already_compliant"}
and has_controller_projection
and revision is not None
and changed_targets is not None
and changed_details is not None
and uncertain_targets == []
and verified_targets is not None
and (bool(changed_targets) if status == "prepared" else not changed_targets)
)
if success:
message = (
"The fixed MCP Hub state-file permission was prepared and verified."
if status == "prepared"
else "The fixed MCP Hub state-file permission was already compliant and verified."
)
elif (
status == "partial"
and has_controller_projection
and revision is not None
and changed_targets is not None
and changed_details is not None
and uncertain_targets is not None
and (changed_targets or uncertain_targets)
and len(changed_details) == len(changed_targets)
and {item["path"] for item in changed_details} == set(changed_targets)
):
message = "Permission preparation stopped partway; inspect the exact changed and uncertain fixed targets before retrying."
elif (
status in {
"blocked",
"unauthorized",
"missing_config",
"missing_secret",
"runtime_identity_unavailable",
"controller_source_mismatch",
"preflight_rejected",
"configuration_unavailable",
"exact_source_unavailable",
"invalid_request",
"invalid_body_size",
"authentication_rejected",
"forbidden",
}
and changed_targets == []
and changed_details == []
and uncertain_targets == []
):
status = "blocked"
message = "The fixed MCP Hub permission request was rejected before any target change."
else:
status = "outcome_unknown"
changed_targets = []
changed_details = []
uncertain_targets = sorted(fixed_targets)
verified_targets = None
message = (
"The controller request outcome is unknown; the fixed Hub state file may have changed. "
"Inspect it before retrying."
)
except Exception:
result = {"status": "outcome_unknown"}
status = "outcome_unknown"
revision = None
changed_targets = []
changed_details = []
uncertain_targets = sorted(fixed_targets)
verified_targets = None
success = False
message = (
"The controller request outcome is unknown; the fixed Hub state file may have changed. "
"Inspect it before retrying."
)
completed = complete_job(
session,
job,
status="succeeded" if success else "failed",
result_summary=message,
)
audit = log_event(
session,
source="action",
identity=source,
action_id=action,
target=target,
target_type="mcp-hub-permissions",
risk=risk,
policy_decision="allowed",
reason=reason,
result="success" if success else "outcome_unknown" if status == "outcome_unknown" else "failed",
execution_plan={"steps": steps, "jobId": completed.id},
after_state={
"status": status,
"revision": revision,
"changedTargets": changed_targets,
"changedTargetDetails": changed_details,
"uncertainTargets": uncertain_targets,
"verifiedTargets": verified_targets,
},
redacted_metadata={
"executor": action,
"integration": "mcp" if source == "chatgpt-mcp" else "operator",
"originalPolicyDecision": original_policy_decision,
"targetSelectionCallerSelectable": False,
"pathOrCommandCallerSelectable": False,
},
)
return {
"success": success,
"status": "success" if success else status,
"result": "success" if success else "outcome_unknown" if status == "outcome_unknown" else "failed",
"message": message,
"jobId": completed.id,
"auditId": audit.id,
"policyDecision": "allowed",
"originalPolicyDecision": original_policy_decision,
"controllerStatus": status,
"revision": revision,
"changedTargets": changed_targets,
"changedTargetDetails": changed_details,
"uncertainTargets": uncertain_targets,
"verifiedTargets": verified_targets,
}
@@ -1809,6 +2076,24 @@ def execute_governed_action(
original_policy_decision=original_policy_decision,
)
if action == "hub.migration.prepare_permissions":
if target.strip().lower() != "tower/itworx-mcp-hub-permissions":
return {
"success": False,
"status": "failed",
"result": "blocked",
"message": "MCP Hub permission preparation accepts only tower/itworx-mcp-hub-permissions.",
"policyDecision": "blocked",
"originalPolicyDecision": original_policy_decision,
}
return _execute_hub_migration_prepare_permissions(
session,
source=source,
risk=risk,
reason=reason,
original_policy_decision=original_policy_decision,
)
if action == "runner.fleet.upgrade_v2":
if target.strip().lower() != "tower/runner-fleet-v2":
return {
@@ -1827,6 +2112,28 @@ def execute_governed_action(
original_policy_decision=original_policy_decision,
)
if action == "runner.statistics.configure":
if target.strip().lower() != "tower/projectbrain-runner-statistics":
return {"success": False, "status": "failed", "result": "blocked",
"message": "Statistics configuration accepts only tower/projectbrain-runner-statistics.",
"policyDecision": "blocked", "originalPolicyDecision": original_policy_decision}
return _execute_runner_fleet_upgrade_v2(
session, source=source, risk=risk, reason=reason,
original_policy_decision=original_policy_decision, statistics_configuration=True,
)
if action == "runner.fleet.specialize":
if target.strip().lower() != "tower/runner-fleet-specialized":
return {
"success": False, "status": "failed", "result": "blocked",
"message": "Runner specialization accepts only tower/runner-fleet-specialized.",
"policyDecision": "blocked", "originalPolicyDecision": original_policy_decision,
}
return _execute_runner_fleet_upgrade_v2(
session, source=source, risk=risk, reason=reason,
original_policy_decision=original_policy_decision, specialized=True,
)
if action == "deployment.runnerless.provision":
return _execute_runnerless_target_provisioning(
session,
@@ -2287,6 +2594,44 @@ def execute_approved_action(session: Session, approval: Approval) -> dict[str, A
original_policy_decision="approval_required",
)
if approval.action == "runner.statistics.configure":
if approval.target.strip().lower() != "tower/projectbrain-runner-statistics":
return {"success": False, "status": "failed", "result": "blocked",
"message": "Statistics configuration accepts only tower/projectbrain-runner-statistics."}
return _execute_runner_fleet_upgrade_v2(
session, source="approved_operator", risk=approval.risk,
reason=approval.reason or f"Approved through {approval.id}.",
original_policy_decision="approval_required", statistics_configuration=True,
)
if approval.action == "runner.fleet.specialize":
if approval.target.strip().lower() != "tower/runner-fleet-specialized":
return {
"success": False, "status": "failed", "result": "blocked",
"message": "Runner specialization accepts only tower/runner-fleet-specialized.",
}
return _execute_runner_fleet_upgrade_v2(
session, source="approved_operator", risk=approval.risk,
reason=approval.reason or f"Approved through {approval.id}.",
original_policy_decision="approval_required", specialized=True,
)
if approval.action == "hub.migration.prepare_permissions":
if approval.target.strip().lower() != "tower/itworx-mcp-hub-permissions":
return {
"success": False,
"status": "failed",
"result": "blocked",
"message": "MCP Hub permission preparation accepts only tower/itworx-mcp-hub-permissions.",
}
return _execute_hub_migration_prepare_permissions(
session,
source="approved_operator",
risk=approval.risk,
reason=approval.reason or f"Approved through {approval.id}.",
original_policy_decision="approval_required",
)
if approval.action in {"unraid.vm.start", "unraid.vm.resume", "unraid.vm.pause"}:
from .unraid_vm_service import execute_approved_vm_action
backend/app/services/action_registry.py
+12
View File
@@ -7,8 +7,11 @@ ACTIONS = [
'container.restart',
'container.stop',
'runner.fleet.upgrade_v2',
'runner.fleet.specialize',
'runner.statistics.configure',
'controller.artifact.promote',
'controller.managed-storage.initialize',
'hub.migration.prepare_permissions',
'gateway.clear_cache',
'stack.status',
'stack.redeploy',
@@ -214,8 +217,11 @@ ACTION_RISK = {
'container.restart': 'medium',
'container.stop': 'high',
'runner.fleet.upgrade_v2': 'high',
'runner.fleet.specialize': 'high',
'runner.statistics.configure': 'high',
'controller.artifact.promote': 'high',
'controller.managed-storage.initialize': 'high',
'hub.migration.prepare_permissions': 'high',
'gateway.clear_cache': 'high',
'stack.status': 'low',
'stack.redeploy': 'high',
@@ -529,8 +535,14 @@ def action_default_target(action: str) -> str:
return 'tower/deployment-controller'
if action == 'controller.managed-storage.initialize':
return 'tower/deployment-controller'
if action == 'hub.migration.prepare_permissions':
return 'tower/itworx-mcp-hub-permissions'
if action == 'runner.fleet.upgrade_v2':
return 'tower/runner-fleet-v2'
if action == 'runner.fleet.specialize':
return 'tower/runner-fleet-specialized'
if action == 'runner.statistics.configure':
return 'tower/projectbrain-runner-statistics'
if action == 'deployment.targets.sync':
return 'jens/*'
if action == 'deployment.target.deploy_workmesh':
backend/app/services/controller_artifact_helper_spec.py
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import re
from typing import Any
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
HELPER_NAME = "itworx-controller-artifact-promoter-modern"
CONTROLLER_VERSION = "2.1.14"
HELPER_LABEL = "itworx.controller.promoter"
_REVISION = re.compile(r"^[0-9a-f]{40}$")
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
_CONTAINER_ID = re.compile(r"^[0-9a-f]{64}$")
_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$")
def helper_container_spec(
*,
revision: str,
image_id: str,
current_controller_id: str | None = None,
current_controller_revision: str | None = None,
current_controller_tree: str | None = None,
current_controller_version: str | None = None,
) -> dict[str, Any]:
if not _REVISION.fullmatch(revision):
raise ValueError("controller_promoter_revision_invalid")
if not _IMAGE_ID.fullmatch(image_id):
raise ValueError("controller_promoter_image_invalid")
bootstrap_values = (
current_controller_id,
current_controller_revision,
current_controller_tree,
current_controller_version,
)
if any(value is not None for value in bootstrap_values):
if (
not _CONTAINER_ID.fullmatch(current_controller_id or "")
or not _REVISION.fullmatch(current_controller_revision or "")
or not _REVISION.fullmatch(current_controller_tree or "")
or not _VERSION.fullmatch(current_controller_version or "")
):
raise ValueError("controller_promoter_bootstrap_binding_invalid")
command = f"""set -Eeuo pipefail
umask 077
revision='{revision}'
version='{CONTROLLER_VERSION}'
image_id='{image_id}'
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
[[ \"$(stat -c '%a:%u:%g' /tmp)\" == '1777:0:0' ]]
work=\"$(mktemp -d -p /tmp itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX)\"
[[ -d \"$work\" && ! -L \"$work\" && \"$(stat -c '%u:%a' \"$work\")\" == '0:700' ]]
cleanup() {{ rm -rf -- \"$work\"; }}
trap cleanup EXIT
command -v git >/dev/null
command -v docker >/dev/null
docker compose version >/dev/null
command -v python3 >/dev/null
command -v tar >/dev/null
command -v curl >/dev/null
git --git-dir=\"$repo\" cat-file -e \"$revision^{{commit}}\"
tree=\"$(git --git-dir=\"$repo\" rev-parse \"$revision^{{tree}}\")\"
git --git-dir=\"$repo\" archive --format=tar \"$revision\" \\
ops/gitea-deploy-broker \\
scripts/prepare_controller_artifact.py \\
scripts/reconcile_controller_state.py \\
scripts/prepare_controller_hub_release_volume.py \\
scripts/vendor \\
scripts/run_repopulse_adoption.py \
scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"
chmod 0600 \"$work/controller.tar\"
archive_sha=\"$(sha256sum \"$work/controller.tar\" | cut -d ' ' -f 1)\"
git --git-dir=\"$repo\" show \"$revision:scripts/upgrade_deployment_controller.sh\" > \"$work/upgrade.sh\"
chmod 0700 \"$work/upgrade.sh\"
bash \"$work/upgrade.sh\" \\
--archive \"$work/controller.tar\" \\
--archive-sha256 \"$archive_sha\" \\
--source-repository '{SOURCE_REPOSITORY}' \\
--source-revision \"$revision\" \\
--source-tree \"$tree\" \\
--controller-version \"$version\" \\
--image-id \"$image_id\" \\
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
"""
labels = {
HELPER_LABEL: "true",
"itworx.source.repository": SOURCE_REPOSITORY,
"itworx.source.revision": revision,
"itworx.controller.version": CONTROLLER_VERSION,
}
if current_controller_id is not None:
labels.update(
{
"itworx.controller.bootstrap.target.id": current_controller_id,
"itworx.controller.bootstrap.target.image": image_id,
"itworx.controller.bootstrap.target.revision": current_controller_revision or "",
"itworx.controller.bootstrap.target.tree": current_controller_tree or "",
"itworx.controller.bootstrap.target.version": current_controller_version or "",
}
)
return {
"Image": image_id,
"User": "0:0",
"Cmd": ["/bin/bash", "-lc", command],
"Labels": labels,
"HostConfig": {
"Binds": [
"/mnt/cache:/mnt/cache",
"/mnt/user:/mnt/user",
"/var/lib/docker:/var/lib/docker:ro",
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
"/var/run/docker.sock:/var/run/docker.sock",
],
"NetworkMode": "host",
"Privileged": False,
"ReadonlyRootfs": True,
"CpuPeriod": 100000,
"CpuQuota": 100000,
"Memory": 1024 * 1024 * 1024,
"PidsLimit": 128,
"Tmpfs": {"/tmp": "rw,nosuid,nodev,size=268435456,mode=1777"},
"SecurityOpt": ["no-new-privileges:true"],
"RestartPolicy": {"Name": "no"},
},
}
backend/app/services/controller_artifact_promoter.py
+9 -78
View File
@@ -11,19 +11,22 @@ import httpx
from ..connectors.portainer import PortainerConnector
from ..core.config import settings
from ..core.redaction import redact_message
from .controller_artifact_helper_spec import (
CONTROLLER_VERSION,
HELPER_LABEL,
HELPER_NAME,
SOURCE_REPOSITORY as SOURCE_REPOSITORY,
_IMAGE_ID,
_REVISION,
helper_container_spec,
)
from .deployment_broker import DeploymentBrokerClient
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
GITEA_OWNER = "Jens"
GITEA_REPOSITORY = "Unraid-AppOps-Gateway"
REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"
HELPER_NAME = "itworx-controller-artifact-promoter"
CONTROLLER_CONTAINER = "gitea-deploy-control"
CONTROLLER_VERSION = "2.1.13"
HELPER_LABEL = "itworx.controller.promoter"
_HELPER_TIMEOUT_SECONDS = 240
_REVISION = re.compile(r"^[0-9a-f]{40}$")
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$")
@@ -31,78 +34,6 @@ class ControllerArtifactPromotionError(RuntimeError):
pass
def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]:
if not _REVISION.fullmatch(revision):
raise ValueError("controller_promoter_revision_invalid")
if not _IMAGE_ID.fullmatch(image_id):
raise ValueError("controller_promoter_image_invalid")
command = f"""set -Eeuo pipefail
umask 077
revision='{revision}'
version='{CONTROLLER_VERSION}'
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
work="$(mktemp -d -p /tmp 'itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX')"
[[ -d "$work" && ! -L "$work" && "$(stat -c '%u:%a' "$work")" == '0:700' ]]
cleanup() {{ rm -rf -- "$work"; }}
trap cleanup EXIT
command -v git >/dev/null
command -v docker >/dev/null
docker compose version >/dev/null
command -v python3 >/dev/null
command -v tar >/dev/null
command -v curl >/dev/null
git --git-dir="$repo" cat-file -e "$revision^{{commit}}"
tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")"
git --git-dir="$repo" archive --format=tar "$revision" \
ops/gitea-deploy-broker \
scripts/prepare_controller_artifact.py \
scripts/reconcile_controller_state.py \
scripts/run_repopulse_adoption.py \
scripts/prepare_controller_hub_release_volume.py \
scripts/vendor \
scripts/upgrade_deployment_controller.sh > "$work/controller.tar"
chmod 0600 "$work/controller.tar"
archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)"
git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh"
chmod 0700 "$work/upgrade.sh"
bash "$work/upgrade.sh" \
--archive "$work/controller.tar" \
--archive-sha256 "$archive_sha" \
--source-repository '{SOURCE_REPOSITORY}' \
--source-revision "$revision" \
--source-tree "$tree" \
--controller-version "$version" \
--image-id '{image_id}' \
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
"""
return {
"Image": image_id,
"User": "0:0",
"Cmd": ["/bin/bash", "-lc", command],
"Labels": {
HELPER_LABEL: "true",
"itworx.source.repository": SOURCE_REPOSITORY,
"itworx.source.revision": revision,
"itworx.controller.version": CONTROLLER_VERSION,
},
"HostConfig": {
"Binds": [
"/mnt/cache:/mnt/cache",
"/mnt/user:/mnt/user",
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
"/var/lib/docker:/var/lib/docker:ro",
"/var/run/docker.sock:/var/run/docker.sock",
],
"NetworkMode": "host",
"Privileged": False,
"SecurityOpt": ["no-new-privileges:true"],
"RestartPolicy": {"Name": "no"},
},
}
class ControllerArtifactPromoter:
def __init__(
self,
backend/app/services/deployment_broker.py
+32 -2
View File
@@ -551,6 +551,27 @@ class DeploymentBrokerClient:
payload={"reason": reason[:1000]},
)
def runner_fleet_specialize(self, *, reason: str) -> dict[str, Any]:
"""Request the fixed protected-source CI specialization migration."""
return self._request(
"POST", "/v1/appops/runner-fleet/specialize",
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
)
def runner_statistics_configure(self, *, reason: str) -> dict[str, Any]:
"""Provision the fixed independent read-only ProjectBrain capability."""
return self._request(
"POST", "/v1/appops/runner-statistics/configure",
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
)
def hub_migration_prepare_permissions(self, *, reason: str) -> dict[str, Any]:
"""Prepare only the fixed Hub backup and controller-state permissions."""
return self._request(
"POST", "/v1/appops/hub-migration/prepare-permissions",
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
)
def runner_fleet_status(self) -> dict[str, Any]:
"""Read the controller's private, bounded CI sandbox projection.
@@ -558,16 +579,25 @@ class DeploymentBrokerClient:
derives the payload from fixed runner identities and its private DinD
daemon; AppOps never receives a Docker socket or a cleanup selector.
"""
return self._runner_fleet_read("/v1/runner-fleet")
def runner_fleet_statistics(self) -> dict[str, Any]:
"""Read the persisted profile projection without scheduling work."""
return self._runner_fleet_read("/v1/runner-fleet/statistics", timeout_seconds=min(self.timeout_seconds, 5))
def _runner_fleet_read(self, path: str, *, timeout_seconds: float | None = None) -> dict[str, Any]:
if path not in {"/v1/runner-fleet", "/v1/runner-fleet/statistics"}:
return {"status": "unavailable", "reason": "runner_fleet_path_invalid"}
if not self.is_configured():
return {"status": "missing_config", "reason": self.configuration_error()}
try:
with httpx.Client(
timeout=self.timeout_seconds,
timeout=self.timeout_seconds if timeout_seconds is None else timeout_seconds,
verify=self._verify_option(),
follow_redirects=False,
) as client, client.stream(
"GET",
f"{self.base_url}/v1/runner-fleet",
f"{self.base_url}{path}",
headers={"Accept": "application/json"},
) as response:
if response.status_code != 200:
backend/app/services/deployment_target_registry.py
+2 -2
View File
@@ -32,8 +32,8 @@ import subprocess
import sys
import time
path = Path("/opt/gitea-deploy/projects.json")
host_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
path = Path("/controller-config/projects.json")
host_path = Path("/controller-hub-releases/controller-config/projects.json")
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
repository, revision, branch, raw_project = sys.argv[1:5]
project = json.loads(raw_project)
backend/app/services/media_codec_dashboard_deployment_reconciliation.py
+2 -2
View File
@@ -29,8 +29,8 @@ import sys
import time
repository, expected_container = sys.argv[1:3]
registry_path = Path("/opt/gitea-deploy/projects.json")
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "media-codec-dashboard.json"
project_lock_path = state_root / "locks" / "media-codec-dashboard.lock"
backend/app/services/portfolio_deployment_enrollment.py
+3 -3
View File
@@ -50,7 +50,7 @@ from pathlib import Path
import sys
repository = sys.argv[1]
path = Path("/opt/gitea-deploy/projects.json")
path = Path("/controller-config/projects.json")
try:
projects = json.loads(path.read_text(encoding="utf-8"))
except Exception:
@@ -145,8 +145,8 @@ import subprocess
import sys
import time
registry_path = Path("/opt/gitea-deploy/projects.json")
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "portfolio.json"
project_lock_path = state_root / "locks" / "portfolio.lock"
backend/app/services/portfolio_deployment_recovery.py
+1 -1
View File
@@ -46,7 +46,7 @@ if (
raise SystemExit(40)
controller_path = Path("/opt/gitea-deploy/deploy.py")
registry_path = Path("/opt/gitea-deploy/projects.json")
registry_path = Path("/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "portfolio.json"
project_lock_path = state_root / "locks" / "portfolio.lock"
backend/app/services/privatemuse_deployment_enrollment.py
+2 -2
View File
@@ -46,8 +46,8 @@ import subprocess
import sys
import time
registry_path = Path("/opt/gitea-deploy/projects.json")
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "privatemuse.json"
project_lock_path = state_root / "locks" / "privatemuse.lock"
backend/app/services/privatemuse_deployment_recovery.py
+1 -1
View File
@@ -64,7 +64,7 @@ if (
raise SystemExit(40)
controller_path = Path("/opt/gitea-deploy/deploy.py")
registry_path = Path("/opt/gitea-deploy/projects.json")
registry_path = Path("/controller-config/projects.json")
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
state_path = state_root / "privatemuse.json"
project_lock_path = state_root / "locks" / "privatemuse.lock"
backend/app/services/privatemuse_webhook_repair.py
+2 -2
View File
@@ -26,8 +26,8 @@ repository, endpoint = sys.argv[1:3]
if repository != "jens/privatemuse" or endpoint != "http://198.19.0.1:1299/v1/gitea/push/jens/privatemuse":
raise SystemExit(40)
projects_path = Path("/opt/gitea-deploy/projects.json")
secret_root = Path("/opt/gitea-deploy/webhook-secrets")
projects_path = Path("/controller-config/projects.json")
secret_root = Path("/controller-config/webhook-secrets")
secret_path = secret_root / "jens__privatemuse"
admin_token_path = Path("/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token")
expected_validation = {
backend/app/services/repopulse_adoption_preflight.py
+2 -2
View File
@@ -75,8 +75,8 @@ CONTAINER = "repopulse"
VOLUME = "repopulse_data"
APPDATA_ROOT = Path("/srv/example/appdata/repopulse")
SCAN_ROOT = Path("/mnt/cache/appdata/gitea-deploy-runners/sources")
REGISTRY = Path("/opt/gitea-deploy/projects.json")
HOST_REGISTRY = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
REGISTRY = Path("/controller-config/projects.json")
HOST_REGISTRY = Path("/controller-hub-releases/controller-config/projects.json")
STATE = Path("/mnt/cache/appdata/gitea-deploy-runners/state/repopulse.json")
BARE_REPO = Path("/gitea-repositories/jens/repopulse.git")
SHA = re.compile(r"^[0-9a-f]{40}$")
backend/app/services/runner_statistics.py
+181
View File
@@ -0,0 +1,181 @@
"""Bounded read-only CI statistics; absent measurements remain unknown."""
from __future__ import annotations
import copy
import re
import threading
import time
from datetime import UTC, datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from .deployment_broker import DeploymentBrokerClient
class _StatisticsModel(BaseModel):
model_config = ConfigDict(extra="ignore", allow_inf_nan=False, strict=True)
class _Budget(_StatisticsModel):
cpuLimit: float | int | None = Field(default=None, ge=0, le=20)
memoryLimitBytes: int | None = Field(default=None, ge=0)
activeJobLimit: int | None = Field(default=None, ge=0, le=6)
heavyJobLimit: int | None = Field(default=None, ge=0, le=6)
reservedCpu: float | int | None = Field(default=None, ge=0)
reservedMemoryBytes: int | None = Field(default=None, ge=0)
activeJobs: int | None = Field(default=None, ge=0)
heavyJobs: int | None = Field(default=None, ge=0)
class _Host(_StatisticsModel):
cpuBusyPercent: float | int | None = Field(default=None, ge=0, le=100)
loadRatio: float | int | None = Field(default=None, ge=0)
availableMemoryBytes: int | None = Field(default=None, ge=0)
cacheFreeBytes: int | None = Field(default=None, ge=0)
class _Profile(_StatisticsModel):
id: Literal["checks", "backend", "frontend", "security", "browser", "build"]
displayName: str = Field(min_length=1, max_length=80)
labels: list[str] = Field(max_length=8)
runnerName: str = Field(min_length=1, max_length=80)
status: str = Field(min_length=1, max_length=80)
queueDepth: int | None = Field(default=None, ge=0)
activeJobs: int | None = Field(default=None, ge=0)
oldestQueuedSeconds: float | int | None = Field(default=None, ge=0)
reason: str | None = Field(default=None, max_length=120)
cpuLimit: float | int | None = Field(default=None, ge=0, le=8)
memoryLimitBytes: int | None = Field(default=None, ge=0)
cpuUsed: float | int | None = Field(default=None, ge=0)
memoryUsedBytes: int | None = Field(default=None, ge=0)
completedJobs: int = Field(default=0, ge=0)
p50WaitSeconds: float | int | None = Field(default=None, ge=0)
p95WaitSeconds: float | int | None = Field(default=None, ge=0)
p50DurationSeconds: float | int | None = Field(default=None, ge=0)
p95DurationSeconds: float | int | None = Field(default=None, ge=0)
lastDurationSeconds: float | int | None = Field(default=None, ge=0)
class _Window(_StatisticsModel):
startAt: str | None = Field(default=None, max_length=40)
endAt: str | None = Field(default=None, max_length=40)
completedJobs: int = Field(default=0, ge=0)
samples: int = Field(default=0, ge=0)
coverage: Literal["bounded", "complete", "unavailable"] = "unavailable"
class _History(_StatisticsModel):
at: str = Field(min_length=1, max_length=40)
activeJobs: int | None = Field(default=None, ge=0)
queuedJobs: int | None = Field(default=None, ge=0)
cpuUsed: float | int | None = Field(default=None, ge=0)
memoryUsedBytes: int | None = Field(default=None, ge=0)
class RunnerStatistics(_StatisticsModel):
schemaVersion: Literal[1]
status: Literal["healthy", "degraded", "unavailable", "not_configured"]
scope: Literal["ci-fleet"]
generatedAt: str = Field(min_length=1, max_length=40)
budget: _Budget
host: _Host
profiles: list[_Profile] = Field(max_length=6)
window: _Window
history: list[_History] = Field(max_length=288)
def _valid_timestamp(value: str) -> bool:
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return False
return parsed.tzinfo is not None and 0 < parsed.timestamp() <= time.time() + 15
def unavailable_statistics(*, configured: bool = True) -> dict[str, object]:
return RunnerStatistics(
schemaVersion=1,
status="unavailable" if configured else "not_configured",
scope="ci-fleet",
generatedAt=datetime.now(UTC).isoformat(),
budget=_Budget(),
host=_Host(),
profiles=[],
window=_Window(),
history=[],
).model_dump(mode="json")
def sanitize_statistics(payload: object) -> dict[str, object]:
"""Accept only the public contract, never raw paths, credentials or jobs."""
try:
snapshot = RunnerStatistics.model_validate(payload)
except (ValidationError, TypeError, ValueError):
return unavailable_statistics()
timestamps = [snapshot.generatedAt, *(sample.at for sample in snapshot.history)]
timestamps.extend(value for value in (snapshot.window.startAt, snapshot.window.endAt) if value)
if any(not _valid_timestamp(value) for value in timestamps):
return unavailable_statistics()
if len({profile.id for profile in snapshot.profiles}) != len(snapshot.profiles):
return unavailable_statistics()
for profile in snapshot.profiles:
allowed_labels = {f"itworx-ci-{profile.id}"}
if profile.id in {"backend", "frontend"}:
allowed_labels.update({"linux-validation", "ubuntu-latest"})
elif profile.id == "build":
allowed_labels.add("docker-validation")
if (
profile.displayName != f"ITWorx CI · {profile.id.title()}"
or profile.runnerName != f"itworx-ci-{profile.id}"
or not profile.labels
or len(set(profile.labels)) != len(profile.labels)
or not set(profile.labels).issubset(allowed_labels)
or re.fullmatch(r"[a-z][a-z0-9_-]{0,79}", profile.status) is None
or (profile.reason is not None and re.fullmatch(r"[a-z][a-z0-9_-]{0,119}", profile.reason) is None)
):
return unavailable_statistics()
sample_time = datetime.fromisoformat(snapshot.generatedAt.replace("Z", "+00:00")).timestamp()
history_times = [datetime.fromisoformat(sample.at.replace("Z", "+00:00")).timestamp() for sample in snapshot.history]
if history_times != sorted(set(history_times)) or any(at > sample_time + 15 for at in history_times):
return unavailable_statistics()
window = snapshot.window
if (window.startAt is None) != (window.endAt is None):
return unavailable_statistics()
if window.startAt is not None and window.endAt is not None:
start = datetime.fromisoformat(window.startAt.replace("Z", "+00:00")).timestamp()
end = datetime.fromisoformat(window.endAt.replace("Z", "+00:00")).timestamp()
if not start <= end <= sample_time + 15:
return unavailable_statistics()
elif window.coverage != "unavailable":
return unavailable_statistics()
if time.time() - sample_time > 90 and snapshot.status == "healthy":
snapshot.status = "degraded"
return snapshot.model_dump(mode="json")
_lock = threading.Lock()
_cached: dict[str, object] | None = None
_cached_at = 0.0
_CACHE_SECONDS = 30.0
class RunnerStatisticsService:
def snapshot(self) -> dict[str, object]:
global _cached, _cached_at
with _lock:
now = time.monotonic()
if _cached is not None and now - _cached_at < _CACHE_SECONDS:
return copy.deepcopy(_cached)
broker = DeploymentBrokerClient()
if not broker.is_configured():
snapshot = unavailable_statistics(configured=False)
else:
try:
snapshot = sanitize_statistics(broker.runner_fleet_statistics())
except Exception:
snapshot = unavailable_statistics()
_cached = snapshot
_cached_at = time.monotonic()
return copy.deepcopy(snapshot)
backend/app/services/runnerless_target_provisioning.py
+2 -2
View File
@@ -36,8 +36,8 @@ if (
):
raise SystemExit(40)
projects_path = Path("/opt/gitea-deploy/projects.json")
secret_root = Path("/opt/gitea-deploy/webhook-secrets")
projects_path = Path("/controller-hub-releases/controller-config/projects.json")
secret_root = Path("/controller-hub-releases/controller-config/webhook-secrets")
admin_token_path = Path(
"/mnt/cache/appdata/gitea-deploy-runners/secrets/runner_guard_admin_token"
)
backend/app/services/workmesh_first_deployment.py
+2 -2
View File
@@ -53,8 +53,8 @@ containers = [
]
ports = {1255, 1256, 1257, 1258}
controller_root = Path("/opt/gitea-deploy")
registry_path = controller_root / "projects.json"
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
registry_path = Path("/controller-config/projects.json")
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
env_dir = Path("/srv/example/appdata/workmesh")
env_path = env_dir / ".env"
backend/tests/test_appops_live_state_recovery.py
+5
View File
@@ -123,9 +123,14 @@ def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery):
before = r.path.read_bytes()
compose = (r.repo / "docker-compose.yml").read_bytes()
result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
read_state, read_raw, read_sha256 = r.m.read_appops_state_preimage(r.project)
state = json.loads(r.path.read_bytes())
backup = r.path.parent / result["backupFile"]
assert backup.read_bytes() == before
assert read_state == state
assert read_raw == r.path.read_bytes()
assert read_sha256 == hashlib.sha256(read_raw).hexdigest()
assert os.name == "nt" or (backup.stat().st_mode & 0o777) == 0o600
assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest()
assert state["last_attempt"] == r.prior["last_attempt"]
assert state["live_revision"] == OBSERVED
backend/tests/test_appops_migration_bridge.py
+198
View File
@@ -0,0 +1,198 @@
from __future__ import annotations
import copy
import json
import subprocess
import pytest
from test_deploy_broker import (
_appops_known_legacy_state,
_appops_rollback_intent_fixture,
deploy_module as deploy_module,
)
@pytest.fixture
def bridge(deploy_module, monkeypatch, tmp_path):
m = deploy_module
revision = "c" * 40
state_root = tmp_path / "state"
state_root.mkdir(mode=0o700)
monkeypatch.setattr(m, "STATE_ROOT", state_root)
monkeypatch.setattr(m, "APPOPS_LEGACY_INTENT_ROOT", state_root / "appops-legacy")
monkeypatch.setattr(m, "APPOPS_LEGACY_INTENT_PATH", state_root / "appops-legacy" / "unraid-appops-gateway.json")
prior = _appops_known_legacy_state()
monkeypatch.setattr(m, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", m._canonical_json_sha256(prior["last_attempt"]["metadata"]))
monkeypatch.setattr(m, "APPOPS_LEGACY_PROJECTION_SHA256", m._canonical_json_sha256(m._appops_legacy_state_projection(prior)))
repository, metadata, source, green, backup, sample = _appops_rollback_intent_fixture(m, revision, "e" * 64)
project = {"slug": "unraid-appops-gateway", "compose_project": "unraid-appops-gateway",
"_repository_name": repository, "branch": "main", "gitea_repository_id": 9,
"containers": ["Unraid-AppOps-Gateway"], "compose_files": ["docker-compose.yml"],
"verify_oci_revision": True, "environment": {"APPOPS_DATA_PATH": str(m.APPOPS_DATA_ROOT)}}
runtime = {"containerId": m.APPOPS_LEGACY_OLD_CONTAINER_ID, "imageId": m.APPOPS_LEGACY_OLD_IMAGE_ID,
"imageRef": m.APPOPS_LEGACY_OLD_IMAGE_REF, "sourceRepository": repository}
artifacts = m._appops_legacy_state_override(prior, runtime)["live_artifacts"]
artifacts["compose_sha256"] = ""
monkeypatch.setattr(m, "require_host_root", lambda: None)
monkeypatch.setattr(m, "_appops_recovery_config_unchanged", lambda _p: None)
monkeypatch.setattr(m, "validate_commit", lambda _r, _b, rev: rev)
monkeypatch.setattr(m, "validate_recorded_receipt", lambda *_a: None)
monkeypatch.setattr(m, "_appops_recovery_identity", lambda *_a: runtime.copy())
monkeypatch.setattr(m, "deployment_container_artifacts", lambda _p: copy.deepcopy(artifacts["containers"]))
monkeypatch.setattr(m, "deployment_artifacts", lambda _p: copy.deepcopy(artifacts))
monkeypatch.setattr(m, "verify_appops_green_delivery", lambda rev: green if rev == revision else pytest.fail("wrong candidate"))
monkeypatch.setattr(m, "verify_appops_legacy_source", lambda *_a: source)
monkeypatch.setattr(m, "revalidate_deployment_candidate", lambda *_a: None)
proof = sample["migrationProof"]["runtime"]
monkeypatch.setattr(m, "appops_legacy_compose_model", lambda *_a: ({"name": project["compose_project"]}, proof["resolvedComposeSha256"], proof["resolvedEnvironmentSha256"]))
monkeypatch.setattr(m, "_appops_legacy_inspect_runtime", lambda *_a, **_k: ({}, {}, runtime, None))
monkeypatch.setattr(m, "appops_legacy_runtime_hashes", lambda *_a: proof)
events = []
monkeypatch.setattr(m, "capture_appops_data_backup", lambda *_a: events.append("backup") or copy.deepcopy(backup))
monkeypatch.setattr(m, "verify_appops_data_backup", lambda *_a: None)
monkeypatch.setattr(m, "capture_compose_rollback", lambda *_a, **_k: events.append("snapshot") or copy.deepcopy(sample["rollbackSnapshot"]))
monkeypatch.setattr(m, "_appops_legacy_tag_snapshot", lambda *_a: events.append("tag"))
path = state_root / "unraid-appops-gateway.json"
path.write_text(json.dumps(prior), encoding="utf-8")
path.chmod(0o644)
return m, project, tmp_path, revision, metadata, path, events
def prepare_bridge(bridge):
m, project, repo, revision, metadata, _path, _events = bridge
state, raw, digest = m.read_appops_state_preimage(project)
return m.prepare_appops_legacy_migration(project, project["_repository_name"], repo, revision, metadata, state, raw, digest)
def test_reattested_state_prepares_durable_migration_and_completed_rollback_retry(bridge):
m, project, repo, revision, metadata, path, events = bridge
m.reattest_appops_live_state(project, repo, m.APPOPS_LEGACY_OLD_REVISION)
state, raw, digest = m.read_appops_state_preimage(project)
migration = prepare_bridge(bridge)
assert events == ["backup", "snapshot", "tag"]
intent, intent_hash = m.read_appops_legacy_intent()
assert intent["stateSha256"] == digest
assert intent["migrationProof"]["recovery"]["auditSha256"] == m._canonical_json_sha256(state["recovery"])
assert m._appops_legacy_pending_state_is_bound(state, raw, digest, intent, revision, project, repo)
wrong_intent = copy.deepcopy(intent)
wrong_intent["migrationProof"]["recovery"]["auditSha256"] = "f" * 64
assert not m._appops_legacy_pending_state_is_bound(state, raw, digest, wrong_intent, revision, project, repo)
project["_compose_rollback"] = migration["snapshot"]
project["_rollback_restored_revision"] = m.APPOPS_LEGACY_OLD_REVISION
m.write_state(project["_repository_name"], project, m.APPOPS_LEGACY_OLD_REVISION, "success",
m.APPOPS_LEGACY_ROLLBACK_STATE_MESSAGE, live_legacy_provenance=True,
live_receipt={}, rollback_snapshot={}, attempt_revision=revision,
attempt_metadata={"operation": "rollback", "from_revision": revision}, expected_state_sha256=digest)
restored, restored_raw, restored_hash = m.read_appops_state_preimage(project)
assert m._appops_legacy_rollback_state_matches(restored, project, revision, intent)
tampered = copy.deepcopy(restored)
tampered["recovery"]["at_epoch"] += 1
assert not m._appops_legacy_rollback_state_matches(tampered, project, revision, intent)
completed = m._appops_legacy_rebind_observed_state(intent, restored_hash)
completed["status"] = "rolled_back"
m.write_appops_legacy_intent(completed, expected_sha256=intent_hash)
assert m.appops_legacy_rollback_state_anchor(restored, restored_raw, completed, revision)
events.clear()
retried = prepare_bridge(bridge)
assert events == ["tag"]
assert retried["intent"]["migrationProof"] == intent["migrationProof"]
assert retried["intent"]["status"] == "deploying"
def test_candidate_crash_reconciles_the_reattested_preimage_to_a_real_receipt(bridge, monkeypatch):
m, project, repo, revision, metadata, path, _events = bridge
m.reattest_appops_live_state(project, repo, m.APPOPS_LEGACY_OLD_REVISION)
original, raw, digest = m.read_appops_state_preimage(project)
prepare_bridge(bridge)
receipt = {"repository": project["_repository_name"], "revision": revision,
"tree": metadata["tree"], "build": metadata}
monkeypatch.setattr(m, "verify_deployment", lambda *_a: receipt)
result = m.reconcile_appops_legacy_candidate(
project, project["_repository_name"], repo, revision, metadata, original, raw, digest,
)
assert result == receipt
state = json.loads(path.read_bytes())
assert state["live_revision"] == revision
assert state["live_receipt"] == receipt
assert state["live_legacy_provenance"] is False
assert state["recovery"] == original["recovery"]
assert m.read_appops_legacy_intent()[0]["status"] == "deployed"
@pytest.mark.parametrize("damage", ["backup-bytes", "backup-mode", "backup-link", "audit", "state"])
def test_reattest_bridge_rejects_tampering_before_backup_or_docker(bridge, damage):
m, project, repo, _revision, _metadata, path, events = bridge
result = m.reattest_appops_live_state(project, repo, m.APPOPS_LEGACY_OLD_REVISION)
backup = path.parent / result["backupFile"]
if damage == "backup-bytes":
backup.write_bytes(b"{}")
elif damage == "backup-mode":
backup.chmod(0o644)
elif damage == "backup-link":
target = backup.with_suffix(".saved")
backup.rename(target)
backup.symlink_to(target)
else:
state = json.loads(path.read_bytes())
if damage == "audit":
state["recovery"]["observed_image_id"] = "sha256:" + "f" * 64
else:
state["live_artifacts"]["containers"]["Unraid-AppOps-Gateway"]["image_id"] = "sha256:" + "f" * 64
path.write_text(json.dumps(state))
with pytest.raises(m.AppOpsLegacyNoMutation):
prepare_bridge(bridge)
assert events == []
assert not m.APPOPS_LEGACY_INTENT_PATH.exists()
def prior_record(m, revision):
semantic = m._canonical_json_sha256({"gitea_repository_id": 9, "ref": "refs/heads/main", "revision": revision})
record = {"schema": "gitea-webhook-inbox.v2", "semantic_sha256": semantic,
"body_sha256": "a" * 64, "delivery": "12345678-1234-4234-8234-123456789012",
"gitea_repository_id": 9, "ref": "refs/heads/main", "repository": "jens/unraid-appops-gateway",
"revision": revision, "source": "gitea_webhook", "status": "failed", "reason": "deployment_failed",
"validation": {"validation_gate_version": 1, "required_contexts": ["Managed validation / full (push)"],
"timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30,
"last_result": "pass", "observed": [{"context": "Managed validation / full (push)", "status": "success"}]},
"audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"},
{"stage": "deployment_failed", "reason": "deployment_failed"}]}
inbox = m.STATE_ROOT / "webhook-inbox"
inbox.mkdir(mode=0o700)
marker = inbox / f"{semantic}.json"
marker.write_text(json.dumps(record))
marker.chmod(0o600)
return marker, record
@pytest.mark.parametrize("damage", [None, "deployed", "pending", "missing", "metadata", "ancestry"])
def test_prior_main_attempt_requires_private_green_failure_and_git_proof(bridge, monkeypatch, damage):
m, _project, _repo, _revision, metadata, path, events = bridge
prior_revision = "b" * 40
state = json.loads(path.read_bytes())
prior_metadata = {**metadata, "revision": prior_revision, "id": prior_revision[:12]}
state["last_attempt"].update(revision=prior_revision, metadata=prior_metadata)
path.write_text(json.dumps(state))
marker, record = prior_record(m, prior_revision)
monkeypatch.setattr(m, "deployment_metadata", lambda *_a, **_k: prior_metadata if damage != "metadata" else {**prior_metadata, "tree": "e" * 40})
def ancestry(command, **_kwargs):
assert command[-4:] == ["merge-base", "--is-ancestor", prior_revision, metadata["revision"]]
if damage == "ancestry":
raise subprocess.CalledProcessError(1, command)
monkeypatch.setattr(m.subprocess, "run", ancestry)
if damage == "deployed":
record["status"] = "deployed" # Retain the old failure audit deliberately.
elif damage == "pending":
record["validation"]["observed"][0]["status"] = "pending"
marker.write_text(json.dumps(record))
if damage == "missing":
marker.unlink()
if damage is not None:
with pytest.raises(m.AppOpsLegacyNoMutation):
prepare_bridge(bridge)
assert events == []
else:
migration = prepare_bridge(bridge)
assert migration["intent"]["migrationProof"]["priorAttempt"]["metadataSha256"] == m._canonical_json_sha256(prior_metadata)
assert events == ["backup", "snapshot", "tag"]
backend/tests/test_appops_self_upgrade_controller.py
+74 -29
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import subprocess
import sys
import tarfile
@@ -53,7 +54,10 @@ def _project():
"containers": ["Unraid-AppOps-Gateway"],
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"environment": {"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216"},
"environment": {
"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216",
"APPOPS_PUBLISH_HOST": "198.18.10.150",
},
"verify_oci_revision": True,
"validation": {
"validation_gate_version": 1, "required": True,
@@ -71,6 +75,9 @@ def _ready(controller, monkeypatch):
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", []))
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
"revision": "a" * 40, "imageId": "sha256:" + "b" * 64,
"containerId": "c" * 64,
"imageRef": "unraid-appops-gateway:production",
"sourceTree": "d" * 40,
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
})
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3))
@@ -150,24 +157,65 @@ def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controll
def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch):
monkeypatch.setattr(controller, "_gitea_validation_settings", lambda: ("https://gitea.example.com/api/v1", b"t" * 32))
payload = [{
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}],
"Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40,
"org.opencontainers.image.source": "jens/unraid-appops-gateway"}},
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
"Id": "1" * 64,
"Mounts": [
{"Type": "bind", "Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True},
{"Type": "bind", "Destination": "/mnt/privatemuse-comfyui-models", "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models", "RW": True},
],
"Config": {
"Image": "unraid-appops-gateway:platform-5553c04ce497",
"Hostname": "unraid-appops-gateway",
"Labels": {
"com.docker.compose.project": "unraid-appops-gateway",
"com.docker.compose.service": "appops-gateway",
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
"org.opencontainers.image.revision": "5553c04ce497c08a35ed301052dc5e0b02e5d9b4",
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
"org.opencontainers.image.source-tree": "1624e1748053f998d8427c8c8a4203277d22f13c",
"org.opencontainers.image.version": "1.8.0-platform-5553c04ce497",
"org.opencontainers.image.created": "2026-10-01T16:32:01Z",
},
},
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
"NetworkSettings": {
"Ports": {"1216/tcp": [{"HostIp": "198.18.10.150", "HostPort": "1216"}]},
"Networks": {"itworx-mcp-hub-npm": {}, "unraid-appops-gateway_default": {}},
},
"Image": "sha256:" + "b" * 64,
}]
image = [{"Id": payload[0]["Image"], "Config": {"Labels": {
key: value for key, value in payload[0]["Config"]["Labels"].items()
if key.startswith("org.opencontainers.image.")
}}}]
names = ["Unraid-AppOps-Gateway"]
def fake_run(args, **kwargs):
return subprocess.CompletedProcess(args, 0, json.dumps(payload) if args[1] == "inspect" else "\n".join(names), "")
if args[1] == "inspect":
result = payload
elif args[1] == "image":
result = image
else:
result = "\n".join(names)
return subprocess.CompletedProcess(args, 0, json.dumps(result) if isinstance(result, list) else result, "")
monkeypatch.setattr(controller.subprocess, "run", fake_run)
assert controller._self_upgrade_runtime()["revision"] == "a" * 40
runtime = controller._self_upgrade_runtime()
assert runtime["revision"] == "5553c04ce497c08a35ed301052dc5e0b02e5d9b4"
assert runtime["imageRef"] == "unraid-appops-gateway:platform-5553c04ce497"
assert runtime["sourceTree"] == "1624e1748053f998d8427c8c8a4203277d22f13c"
payload[0]["Mounts"][0]["Source"] = "/wrong"
assert controller._self_upgrade_runtime() is None
payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216"
payload[0]["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "127.0.0.1"
assert controller._self_upgrade_runtime() is None
payload[0]["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "198.18.10.150"
image[0]["Id"] = "sha256:" + "0" * 64
assert controller._self_upgrade_runtime() is None
image[0]["Id"] = payload[0]["Image"]
names.append("Unraid-AppOps-Gateway-copy")
assert controller._self_upgrade_runtime() is None
@@ -184,30 +232,13 @@ def test_runtime_requires_exact_container_mount_port_and_health(controller, monk
("http://gitea.example.com/Jens/Unraid-AppOps-Gateway", False),
],
)
def test_runtime_accepts_only_fixed_appops_oci_sources(controller, monkeypatch, source, accepted):
def test_source_validation_accepts_only_fixed_appops_oci_sources(controller, monkeypatch, source, accepted):
monkeypatch.setattr(
controller,
"_gitea_validation_settings",
lambda: ("https://gitea.example.com/api/v1", b"t" * 32),
)
payload = [{
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}],
"Config": {"Labels": {
"org.opencontainers.image.revision": "a" * 40,
"org.opencontainers.image.source": source,
}},
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
"Image": "sha256:" + "b" * 64,
}]
def fake_run(args, **kwargs):
stdout = json.dumps(payload) if args[1] == "inspect" else "Unraid-AppOps-Gateway"
return subprocess.CompletedProcess(args, 0, stdout, "")
monkeypatch.setattr(controller.subprocess, "run", fake_run)
assert (controller._self_upgrade_runtime() is not None) is accepted
assert controller._self_upgrade_source_is_valid(source) is accepted
def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch):
@@ -219,12 +250,16 @@ def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch):
mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345")
@pytest.mark.skipif(sys.platform == 'win32', reason='Linux controller pinned directory descriptors')
def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path):
secret = b"fixed-test-self-upgrade-capability-0123456789"
token_file = tmp_path / "mcp-token"
token_file.write_bytes(secret + b"\n")
token_file.chmod(0o600)
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
import runner_statistics_binding
monkeypatch.setattr(runner_statistics_binding, 'APPOPS_DATA', tmp_path)
monkeypatch.setattr(runner_statistics_binding, 'ADMIN_TOKEN_FILE', token_file)
monkeypatch.setattr(runner_statistics_binding, 'ROOT_UID', os.getuid())
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
stamp = str(int(time.time()))
nonce = "a" * 32
@@ -245,6 +280,7 @@ def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, mo
)
@pytest.mark.skipif(sys.platform == 'win32', reason='Linux controller pinned directory descriptors')
def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path):
_ready(controller, monkeypatch)
monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests")
@@ -254,7 +290,10 @@ def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller,
token_file = tmp_path / "mcp-token"
token_file.write_bytes(secret + b"\n")
token_file.chmod(0o600)
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
import runner_statistics_binding
monkeypatch.setattr(runner_statistics_binding, 'APPOPS_DATA', tmp_path)
monkeypatch.setattr(runner_statistics_binding, 'ADMIN_TOKEN_FILE', token_file)
monkeypatch.setattr(runner_statistics_binding, 'ROOT_UID', os.getuid())
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
@@ -333,6 +372,9 @@ def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller,
)
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
"revision": "c" * 40, "imageId": "sha256:" + "b" * 64,
"containerId": "d" * 64,
"imageRef": "unraid-appops-gateway:production",
"sourceTree": "e" * 40,
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
})
monkeypatch.setattr(controller, "_deployment_state", lambda slug: {})
@@ -389,7 +431,10 @@ def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch,
archive = Path(receipt["archive"])
assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"]
assert receipt["previousRevision"] == "a" * 40
assert json.loads((archive.parent / "receipt.json").read_text()) == receipt
stored_receipt = json.loads((archive.parent / "receipt.json").read_text())
assert {**stored_receipt, "receiptSha256": hashlib.sha256(
json.dumps(stored_receipt, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()} == receipt
with tarfile.open(archive, "r:gz") as backup:
assert backup.extractfile("./state.txt").read() == b"preserved"
backend/tests/test_ci_job_docker.py
+550
View File
@@ -0,0 +1,550 @@
"""Job-local Docker producer contract without starting a container daemon."""
from __future__ import annotations
import hashlib
import runpy
import io
import json
import os
import shutil
import subprocess
import sys
import tarfile
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
PROFILES = ("checks", "backend", "frontend", "security", "browser", "build")
BASE_LABELS = {
"io.buildah.version": "1.35.0",
"org.opencontainers.image.authors": "Gitea",
"org.opencontainers.image.created": "2026-10-01T00:01:57Z",
"org.opencontainers.image.description": "Official docker images used by Gitea runner to run workflows.",
"org.opencontainers.image.documentation": "https://gitea.com/gitea/runner-images/src/branch/main/README.md",
"org.opencontainers.image.licenses": "MIT",
"org.opencontainers.image.title": "Official Gitea Actions runner images",
"org.opencontainers.image.url": "https://gitea.com/gitea/runner-images",
"org.opencontainers.image.vendor": "Gitea",
"org.opencontainers.image.version": "v26.10.01",
}
def _job_local_producer(
tmp_path: Path,
*,
wrong_driver: bool = False,
wrong_security: bool = False,
job_local: bool = True,
):
repo = tmp_path / "repository"
repo.mkdir()
shutil.copytree(ROOT / "ops" / "runner-task-images", repo / "ops" / "runner-task-images")
scripts = repo / "scripts"
scripts.mkdir()
shutil.copyfile(ROOT / "scripts" / "normalize_ci_task_image_archive.py", scripts / "normalize_ci_task_image_archive.py")
code = repo / "ops" / "gitea-deploy-broker"
code.mkdir()
shutil.copyfile(ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py", code / "runner_profiles.py")
for args in (
["init", "-q"],
["add", "ops", "scripts"],
["-c", "user.name=Producer test", "-c", "user.email=producer-test@localhost", "commit", "-qm", "fixture"],
):
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
docker = fake_bin / "docker"
docker.write_text(
f"#!{sys.executable}\n"
"import hashlib,io,json,os,sys,tarfile\n"
"from pathlib import Path\n"
"a=sys.argv[1:]; rows=Path(os.environ['FAKE_DOCKER_ROWS']); calls=Path(os.environ['FAKE_DOCKER_CALLS'])\n"
"history=json.loads(calls.read_text()) if calls.exists() else []\n"
"history.append({'argv':a,'host':os.environ.get('DOCKER_HOST'),'context':os.environ.get('DOCKER_CONTEXT'),'tls':os.environ.get('DOCKER_TLS_VERIFY'),'buildkit':os.environ.get('DOCKER_BUILDKIT')})\n"
"calls.write_text(json.dumps(history))\n"
"layerbuf=io.BytesIO()\n"
"with tarfile.open(fileobj=layerbuf,mode='w:') as t:\n"
" p=b'producer fixture layer'; x=tarfile.TarInfo('fixture.txt'); x.size=len(p); t.addfile(x,io.BytesIO(p))\n"
"layer=layerbuf.getvalue(); layer_sha=hashlib.sha256(layer).hexdigest()\n"
"if a[:1]==['info']:\n"
" fmt=a[a.index('--format')+1] if '--format' in a else ''\n"
f" driver={('vfs' if wrong_driver else 'overlay2')!r}\n"
" security='[\"name=seccomp,profile=builtin\",\"name=rootless\",\"name=cgroupns\"]'\n"
" if os.environ.get('FAKE_DOCKER_WRONG_SECURITY')=='1': security='[\"name=seccomp,profile=builtin\",\"name=rootless\",\"name=cgroupns\",\"name=unknown\"]'\n"
" if fmt=='{{.Driver}}|{{.CgroupDriver}}|{{.DockerRootDir}}|{{json .SecurityOptions}}': print(driver+'|none|/home/itworx-builder/.local/share/docker|'+security); sys.exit(0)\n"
" sys.exit(0)\n"
"if a[0]=='build':\n"
" builds=Path(os.environ['FAKE_DOCKER_BUILDS']); argv=json.loads(builds.read_text()) if builds.exists() else []\n"
" pairs=[a[i+1].split('=',1) for i,x in enumerate(a) if x=='--build-arg']\n"
" is_build=dict(pairs).get('PROFILE')=='build'\n"
" staged=Path(a[-1]).is_dir() and all((Path(a[-1])/n).is_file() for n in ['job_docker.py','itworx-ci-job-docker-probe','normalize_ci_task_image_archive.py']) if is_build else True\n"
" argv.append({'argv':a,'staged':staged}); builds.write_text(json.dumps(argv))\n"
" b=dict(pairs); profile=b['PROFILE']; ref=a[a.index('--tag')+1]\n"
f" labels={BASE_LABELS!r}.copy()\n"
" labels.update({'org.opencontainers.image.source':'https://gitea.example.com/Jens/Unraid-AppOps-Gateway','org.opencontainers.image.revision':b['SOURCE_REVISION'],'tech.itworx.source-tree':b['SOURCE_TREE'],'tech.itworx.ci.profile':profile})\n"
" config={'architecture':'amd64','os':'linux','config':{'Labels':labels},'rootfs':{'type':'layers','diff_ids':['sha256:'+layer_sha]}}\n"
" config_bytes=json.dumps(config,sort_keys=True,separators=(',',':')).encode(); image_id='sha256:'+hashlib.sha256(config_bytes).hexdigest()\n"
" current=json.loads(rows.read_text()) if rows.exists() else []\n"
" current.append({'Id':image_id,'Os':'linux','Architecture':'amd64','RepoTags':[ref],'Config':{'Labels':labels}})\n"
" Path(os.environ['FAKE_DOCKER_CONFIGS']+'/'+profile+'.json').write_bytes(config_bytes)\n"
" rows.write_text(json.dumps(current)); sys.exit(0)\n"
"if a[:2]==['image','inspect']: print(rows.read_text()); sys.exit(0)\n"
"if a[:2]==['image','save']:\n"
" selected=set(a[a.index('--output')+2:]); current=[r for r in json.loads(rows.read_text()) if r['RepoTags'][0] in selected]; items=[]; members={'blobs/sha256/'+layer_sha:layer}\n"
" for row in current:\n"
" profile=row['Config']['Labels']['tech.itworx.ci.profile']; config=Path(os.environ['FAKE_DOCKER_CONFIGS']+'/'+profile+'.json').read_bytes(); config_sha=hashlib.sha256(config).hexdigest(); config_path='blobs/sha256/'+config_sha; members[config_path]=config\n"
" items.append({'Config':config_path,'RepoTags':row['RepoTags'],'Layers':['blobs/sha256/'+layer_sha],'LayerSources':{'sha256:'+layer_sha:{'mediaType':'application/vnd.oci.image.layer.v1.tar','size':len(layer),'digest':'sha256:'+layer_sha}}})\n"
" members['manifest.json']=json.dumps(items,separators=(',',':')).encode(); members['oci-layout']=b'{\"imageLayoutVersion\":\"1.0.0\"}'; members['index.json']=b'{\"schemaVersion\":2,\"manifests\":[]}'\n"
" with tarfile.open(a[a.index('--output')+1],mode='w:') as t:\n"
" for name,payload in members.items():\n"
" x=tarfile.TarInfo(name); x.size=len(payload); t.addfile(x,io.BytesIO(payload))\n"
" sys.exit(0)\n"
"sys.exit('unexpected Docker invocation')\n",
encoding="utf-8",
)
docker.chmod(0o755)
output = tmp_path / "output"
env = {
**os.environ,
"PATH": str(fake_bin) + os.pathsep + os.environ["PATH"],
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
"GITHUB_REPOSITORY": "Jens/Unraid-AppOps-Gateway",
"DOCKER_HOST": "tcp://shared-dind:2375",
"DOCKER_CONTEXT": "shared-context",
"DOCKER_TLS_VERIFY": "1",
"FAKE_DOCKER_ROWS": str(tmp_path / "docker-rows.json"),
"FAKE_DOCKER_CALLS": str(tmp_path / "docker-calls.json"),
"FAKE_DOCKER_BUILDS": str(tmp_path / "docker-builds.json"),
"FAKE_DOCKER_CONFIGS": str(tmp_path / "docker-configs"),
"FAKE_DOCKER_WRONG_SECURITY": "1" if wrong_security else "0",
}
Path(env["FAKE_DOCKER_CONFIGS"]).mkdir()
command = ["bash", str(ROOT / "scripts" / "build_ci_task_images.sh"), revision, "123", str(output)]
if job_local:
command.append("--job-local")
result = subprocess.run(
command,
cwd=repo,
env=env,
capture_output=True,
text=True,
timeout=30,
)
tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], cwd=repo, text=True).strip()
return result, output, tmp_path, wrong_driver, revision, tree
def _oci_task_image_fixture(
tmp_path: Path,
*,
layer_media_type: str = "application/vnd.oci.image.layer.v1.tar",
profiles: tuple[str, ...] = PROFILES,
):
layer_buffer = io.BytesIO()
with tarfile.open(fileobj=layer_buffer, mode="w:") as layer_tar:
payload = b"task image rootfs fixture"
member = tarfile.TarInfo("fixture.txt")
member.size = len(payload)
layer_tar.addfile(member, io.BytesIO(payload))
layer_bytes = layer_buffer.getvalue()
layer_sha = hashlib.sha256(layer_bytes).hexdigest()
labels_base = {
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
"org.opencontainers.image.revision": "a" * 40,
"tech.itworx.source-tree": "b" * 40,
}
rows = []
source_manifest = []
members: dict[str, bytes] = {f"blobs/sha256/{layer_sha}": layer_bytes}
for profile in profiles:
labels = {**labels_base, "tech.itworx.ci.profile": profile}
config = {
"architecture": "amd64",
"os": "linux",
"config": {"Labels": labels},
"rootfs": {"type": "layers", "diff_ids": [f"sha256:{layer_sha}"]},
}
config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode()
config_sha = hashlib.sha256(config_bytes).hexdigest()
config_path = f"blobs/sha256/{config_sha}"
members[config_path] = config_bytes
reference = f"localhost/itworx-ci-{profile}:sha-{'a' * 40}"
rows.append({
"Id": f"sha256:{config_sha}",
"Os": "linux",
"Architecture": "amd64",
"RepoTags": [reference],
"Config": {"Labels": labels},
})
source_manifest.append({
"Config": config_path,
"RepoTags": [reference],
"Layers": [f"blobs/sha256/{layer_sha}"],
"LayerSources": {
f"sha256:{layer_sha}": {
"mediaType": layer_media_type,
"size": len(layer_bytes),
"digest": f"sha256:{layer_sha}",
},
},
})
members["manifest.json"] = json.dumps(source_manifest, separators=(",", ":")).encode()
members["oci-layout"] = b'{"imageLayoutVersion":"1.0.0"}'
members["index.json"] = b'{"schemaVersion":2,"manifests":[]}'
source = tmp_path / "source-oci.tar"
with tarfile.open(source, mode="w:") as archive:
for name, payload in members.items():
info = tarfile.TarInfo(name)
info.size = len(payload)
archive.addfile(info, io.BytesIO(payload))
inspection = tmp_path / "inspection.json"
inspection.write_text(json.dumps(rows), encoding="utf-8")
output = tmp_path / "normalized.tar"
return source, inspection, output, rows, layer_bytes
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_archive_normalizer_converts_uncompressed_oci_blobs_to_legacy_docker_save_layout(tmp_path):
source, inspection, output, rows, layer_bytes = _oci_task_image_fixture(tmp_path)
result = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "normalize_ci_task_image_archive.py"), str(source), str(output), str(inspection)],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, result.stderr
with tarfile.open(output, mode="r:") as archive:
members = {member.name: member for member in archive}
normalized = json.loads(archive.extractfile(members["manifest.json"]).read())
assert len(normalized) == len(PROFILES)
assert all(set(item) == {"Config", "RepoTags", "Layers"} for item in normalized)
for item, expected in zip(normalized, rows, strict=True):
assert item["Config"] == expected["Id"].removeprefix("sha256:") + ".json"
assert item["RepoTags"] == expected["RepoTags"]
assert len(item["Layers"]) == 1
assert item["Layers"][0].endswith("/layer.tar")
config_bytes = archive.extractfile(members[item["Config"]]).read()
assert "sha256:" + hashlib.sha256(config_bytes).hexdigest() == expected["Id"]
assert json.loads(config_bytes)["config"]["Labels"] == expected["Config"]["Labels"]
assert archive.extractfile(members[item["Layers"][0]]).read() == layer_bytes
assert not any(name.startswith("blobs/") for name in members)
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_archive_normalizer_rejects_compressed_oci_layers_before_emitting_output(tmp_path):
source, inspection, output, _rows, _layer_bytes = _oci_task_image_fixture(
tmp_path,
layer_media_type="application/vnd.oci.image.layer.v1.tar+gzip",
)
result = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "normalize_ci_task_image_archive.py"), str(source), str(output), str(inspection)],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode != 0
assert "compressed or unsupported layer media type" in result.stderr
assert not output.exists()
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_archive_normalizer_probe_mode_accepts_and_emits_one_build_image(tmp_path):
source, inspection, output, rows, _layer_bytes = _oci_task_image_fixture(tmp_path, profiles=("build",))
result = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "normalize_ci_task_image_archive.py"), "--probe", str(source), str(output), str(inspection)],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, result.stderr
with tarfile.open(output, mode="r:") as archive:
manifest = json.loads(archive.extractfile("manifest.json").read())
assert len(manifest) == 1
assert set(manifest[0]) == {"Config", "RepoTags", "Layers"}
assert manifest[0]["RepoTags"] == rows[0]["RepoTags"]
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_job_local_mode_builds_all_profiles_through_private_rootless_engine_without_sibling_caps(tmp_path):
result, output, scratch, _wrong_driver, revision, tree = _job_local_producer(tmp_path)
assert result.returncode == 0, result.stderr
calls = json.loads((scratch / "docker-calls.json").read_text())
assert calls[0] == {
"argv": ["info", "--format", "{{.Driver}}|{{.CgroupDriver}}|{{.DockerRootDir}}|{{json .SecurityOptions}}"],
"host": "unix:///run/itworx-job-docker/docker.sock",
"context": None,
"tls": None,
"buildkit": "1",
}
builds = json.loads((scratch / "docker-builds.json").read_text())
assert len(builds) == len(PROFILES)
assert all(row["staged"] for row in builds)
build_arguments = [row["argv"] for row in builds]
assert [next(args[i + 1].split("=", 1)[1] for i, arg in enumerate(args[:-1]) if arg == "--build-arg" and args[i + 1].startswith("PROFILE=")) for args in build_arguments] == list(PROFILES)
for args in build_arguments:
assert not any(arg.startswith(("--cpu-period", "--cpu-quota", "--memory", "--cgroup-parent", "--cpus")) for arg in args)
assert "--rm=true" in args and "--force-rm=true" in args
manifest = json.loads((output / "task-image-manifest.json").read_text())
assert manifest["apiVersion"] == "itworx.runner-task-images/v1"
assert manifest["sourceRevision"] == revision
assert manifest["sourceTree"] == tree
assert tuple(row["profile"] for row in manifest["images"]) == PROFILES
assert manifest["dockerTarSha256"] == hashlib.sha256((output / "task-images.tar").read_bytes()).hexdigest()
for row in manifest["images"]:
assert row["labels"] == {
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
"org.opencontainers.image.revision": revision,
"tech.itworx.source-tree": tree,
"tech.itworx.ci.profile": row["profile"],
}
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_job_local_mode_rejects_non_overlay_or_non_rootless_server_before_build(tmp_path):
result, output, scratch, _wrong_driver, _revision, _tree = _job_local_producer(tmp_path, wrong_driver=True)
assert result.returncode != 0
assert "job-local Docker daemon verification failed" in result.stderr
assert not (scratch / "docker-builds.json").exists()
assert not (output / "task-image-manifest.json").exists()
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_job_local_mode_rejects_unknown_daemon_security_options_before_build(tmp_path):
result, output, scratch, _wrong_driver, _revision, _tree = _job_local_producer(tmp_path, wrong_security=True)
assert result.returncode != 0
assert "job-local Docker daemon verification failed" in result.stderr
assert not (scratch / "docker-builds.json").exists()
assert not (output / "task-image-manifest.json").exists()
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_bootstrap_mode_keeps_classic_shared_daemon_caps_and_archive_contract(tmp_path):
result, output, scratch, _wrong_driver, _revision, _tree = _job_local_producer(tmp_path, job_local=False)
assert result.returncode == 0, result.stderr
calls = json.loads((scratch / "docker-calls.json").read_text())
assert calls[0]["argv"] == ["info"]
assert calls[0]["host"] == "tcp://shared-dind:2375"
assert calls[0]["buildkit"] == "0"
builds = json.loads((scratch / "docker-builds.json").read_text())
assert len(builds) == len(PROFILES)
assert all(row["staged"] for row in builds)
build_arguments = [row["argv"] for row in builds]
assert all(any(arg.startswith("--cpu-quota=") for arg in args) for args in build_arguments)
assert all(any(arg.startswith("--memory=5120m") for arg in args) for args in build_arguments)
assert (output / "task-images.tar").is_file()
with tarfile.open(output / "task-images.tar", mode="r:") as archive:
manifest = json.loads(archive.extractfile("manifest.json").read())
assert len(manifest) == len(PROFILES)
assert all(set(item) == {"Config", "RepoTags", "Layers"} for item in manifest)
def test_build_image_uses_only_minimal_file_capabilities_for_uidmap_helpers():
dockerfile = (ROOT / "ops" / "runner-task-images" / "build" / "Dockerfile").read_text(encoding="utf-8")
pins = json.loads((ROOT / "ops" / "runner-task-images" / "pins.json").read_text(encoding="utf-8"))
assert pins["jobLocalDocker"]["uidmapFileCapabilities"] == {
"newuidmap": "cap_setuid=ep",
"newgidmap": "cap_setgid=ep",
"setuidBits": False,
}
assert '"$setcap_bin" cap_setuid=ep "$newuidmap_bin"' in dockerfile
assert '"$setcap_bin" cap_setgid=ep "$newgidmap_bin"' in dockerfile
assert 'test "$("$getcap_bin" "$newuidmap_bin")" = "$newuidmap_bin cap_setuid=ep"' in dockerfile
assert 'test "$("$getcap_bin" "$newgidmap_bin")" = "$newgidmap_bin cap_setgid=ep"' in dockerfile
assert '! test -u "$newuidmap_bin"' in dockerfile
assert '! test -u "$newgidmap_bin"' in dockerfile
def test_build_image_declares_exactly_one_private_daemon_data_volume():
dockerfile = (ROOT / "ops" / "runner-task-images" / "build" / "Dockerfile").read_text(encoding="utf-8")
pins = json.loads((ROOT / "ops" / "runner-task-images" / "pins.json").read_text(encoding="utf-8"))
volumes = [line.strip() for line in dockerfile.splitlines() if line.strip().startswith("VOLUME")]
assert volumes == ['VOLUME ["/home/itworx-builder/.local/share/docker"]']
assert pins["jobLocalDocker"]["dataRoot"] == "/home/itworx-builder/.local/share/docker"
assert pins["jobLocalDocker"]["anonymousDataVolume"] == volumes[0].split('"')[1]
@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only")
@pytest.mark.parametrize("mode", [0o700, 0o710])
def test_job_docker_accepts_private_data_root_modes_used_before_and_after_daemon_start(tmp_path, monkeypatch, mode):
sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build"))
import job_docker
runtime = tmp_path / "runtime"
data_root = tmp_path / "data"
runtime.mkdir(mode=0o700)
data_root.mkdir(mode=mode)
os.chmod(runtime, 0o700)
os.chmod(data_root, mode)
data_info = data_root.stat()
monkeypatch.setattr(job_docker, "BUILDER_UID", data_info.st_uid)
monkeypatch.setattr(job_docker, "BUILDER_GID", data_info.st_gid)
monkeypatch.setattr(job_docker, "RUNTIME", runtime)
monkeypatch.setattr(job_docker, "DATA_ROOT", data_root)
job_docker._ensure_runtime()
@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only")
@pytest.mark.parametrize("mode", [0o711, 0o750, 0o777])
def test_job_docker_rejects_unexpected_private_data_root_modes(tmp_path, monkeypatch, mode):
sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build"))
import job_docker
runtime = tmp_path / "runtime"
data_root = tmp_path / "data"
runtime.mkdir(mode=0o700)
data_root.mkdir(mode=mode)
os.chmod(runtime, 0o700)
os.chmod(data_root, mode)
data_info = data_root.stat()
monkeypatch.setattr(job_docker, "BUILDER_UID", data_info.st_uid)
monkeypatch.setattr(job_docker, "BUILDER_GID", data_info.st_gid)
monkeypatch.setattr(job_docker, "RUNTIME", runtime)
monkeypatch.setattr(job_docker, "DATA_ROOT", data_root)
with pytest.raises(job_docker.WrapperError, match="data directory ownership invalid"):
job_docker._ensure_runtime()
@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only")
def test_job_docker_rejects_symlink_private_data_root(tmp_path, monkeypatch):
sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build"))
import job_docker
runtime = tmp_path / "runtime"
real_data_root = tmp_path / "real-data"
data_root_link = tmp_path / "data-link"
runtime.mkdir(mode=0o700)
real_data_root.mkdir(mode=0o700)
os.chmod(runtime, 0o700)
os.chmod(real_data_root, 0o700)
data_root_link.symlink_to(real_data_root, target_is_directory=True)
real_data_info = real_data_root.stat()
monkeypatch.setattr(job_docker, "BUILDER_UID", real_data_info.st_uid)
monkeypatch.setattr(job_docker, "BUILDER_GID", real_data_info.st_gid)
monkeypatch.setattr(job_docker, "RUNTIME", runtime)
monkeypatch.setattr(job_docker, "DATA_ROOT", data_root_link)
with pytest.raises(job_docker.WrapperError, match="data directory ownership invalid"):
job_docker._ensure_runtime()
def test_private_socket_ping_reads_the_complete_bounded_docker_response(monkeypatch):
sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build"))
import job_docker
class ChunkedSocket:
def __init__(self, response: bytes):
self.response = response
self.offset = 0
self.recv_calls = 0
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def settimeout(self, _timeout):
pass
def connect(self, _path):
pass
def sendall(self, _request):
pass
def recv(self, size):
self.recv_calls += 1
end = min(self.offset + min(size, 256), len(self.response))
chunk = self.response[self.offset:end]
self.offset = end
return chunk
response = (
b"HTTP/1.1 200 OK\r\nX-Long-Header: "
+ b"x" * 300
+ b"\r\nContent-Length: 2\r\n\r\nOK"
)
assert len(response) > 256
wrapper_socket = ChunkedSocket(response)
monkeypatch.setattr(
job_docker,
"socket",
type("SocketModule", (), {
"AF_UNIX": 1,
"SOCK_STREAM": 2,
"socket": lambda *_args: wrapper_socket,
}),
)
assert job_docker._socket_ping(Path("/run/itworx-job-docker/docker.sock"))
assert wrapper_socket.recv_calls == 2
probe = runpy.run_path(str(ROOT / "ops" / "runner-task-images" / "build" / "itworx-ci-job-docker-probe"))
probe_socket = ChunkedSocket(response)
probe["_socket_ready"].__globals__["socket"] = type("SocketModule", (), {
"AF_UNIX": 1,
"SOCK_STREAM": 2,
"socket": lambda *_args: probe_socket,
})
assert probe["_socket_ready"]()
assert probe_socket.recv_calls == 2
@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only")
def test_job_docker_wrapper_sanitizes_remote_endpoint_environment_and_rejects_cli_overrides(monkeypatch):
sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build"))
import job_docker
monkeypatch.setenv("DOCKER_HOST", "tcp://shared-dind:2375")
monkeypatch.setenv("DOCKER_CONTEXT", "shared-context")
monkeypatch.setenv("DOCKER_TLS_VERIFY", "1")
monkeypatch.setenv("DOCKER_CERT_PATH", "/shared/certs")
monkeypatch.setenv("DOCKER_CONFIG", "/job/config")
environment = job_docker._client_environment()
assert environment["DOCKER_HOST"] == "unix:///run/itworx-job-docker/docker.sock"
assert environment["DOCKER_CONFIG"] == "/job/config"
assert not any(name in environment for name in ("DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"))
with pytest.raises(job_docker.WrapperError, match="endpoint override"):
job_docker._reject_endpoint_override(["--host=tcp://shared-dind:2375", "build"])
with pytest.raises(job_docker.WrapperError, match="endpoint override"):
job_docker._reject_endpoint_override(["--context", "shared-context", "info"])
job_docker._reject_endpoint_override(["run", "alpine", "sh", "-c", "echo --context"])
assert job_docker._is_local_version_request(["compose", "version"])
assert job_docker._is_local_version_request(["compose", "--version"])
assert job_docker._is_local_version_request(["buildx", "version"])
assert job_docker._is_local_version_request(["buildx", "--version"])
assert not job_docker._is_local_version_request(["compose", "up"])
@pytest.mark.skipif(os.name == "nt", reason="rootless Docker wrapper is Linux-only")
def test_job_docker_daemon_command_uses_fixed_rootlesskit_and_unix_only_engine():
sys.path.insert(0, str(ROOT / "ops" / "runner-task-images" / "build"))
import job_docker
command = job_docker._daemon_command()
assert command[:8] == [
"/usr/local/bin/rootlesskit",
"--state-dir=/run/itworx-job-docker/rootlesskit",
"--net=slirp4netns",
"--mtu=1500",
"--disable-host-loopback",
"--port-driver=builtin",
"--copy-up=/etc",
"--copy-up=/run",
]
assert command[8:11] == ["/usr/local/bin/docker-init", "--", "/usr/local/bin/dockerd"]
daemon_args = command[11:]
assert "--host=unix:///run/itworx-job-docker/docker.sock" in daemon_args
assert not any(argument.startswith("--host=tcp:") for argument in daemon_args)
assert "--storage-driver=overlay2" in daemon_args
assert "--feature=containerd-snapshotter=false" in daemon_args
assert "--data-root=/home/itworx-builder/.local/share/docker" in daemon_args
backend/tests/test_container_restart.py
+5 -1
View File
@@ -344,7 +344,11 @@ def test_workmesh_first_deploy_script_accepts_the_pinned_release_descriptor(tmp_
for original, replacement in (
('Path("/opt/gitea-deploy")', f'Path({json.dumps(str(controller_root))})'),
(
'Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")',
'Path("/controller-config/projects.json")',
f'Path({json.dumps(str(registry_path))})',
),
(
'Path("/controller-hub-releases/controller-config/projects.json")',
f'Path({json.dumps(str(registry_path))})',
),
(
backend/tests/test_controller_artifact_bootstrap.py
+1106
View File
File diff suppressed because it is too large. Load diff
backend/tests/test_controller_artifact_promoter.py
+90 -9
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import re
import shlex
from pathlib import Path
import pytest
@@ -8,6 +9,22 @@ import pytest
ROOT = Path(__file__).resolve().parents[2]
def test_promoter_archive_covers_every_required_installer_source():
from backend.app.services.controller_artifact_promoter import helper_container_spec
spec = helper_container_spec(revision="a" * 40, image_id="sha256:" + "b" * 64)
command = spec["Cmd"][2].replace("\\\n", " ")
archive = re.search(r'archive --format=tar "\$revision"(.*?)> "\$work/controller.tar"', command, re.S)
assert archive is not None
packaged = shlex.split(archive.group(1))
installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8")
required_block = re.search(r"(?m)^source_paths=\(\n(.*?)\n\)", installer, re.S)
assert required_block is not None
required = shlex.split(required_block.group(1))
assert required
assert all(any(path == prefix or path.startswith(prefix + "/") for prefix in packaged) for path in required)
def test_controller_promoter_action_is_fixed_and_registered():
registry = (ROOT / "backend/app/services/action_registry.py").read_text(encoding="utf-8")
execution = (ROOT / "backend/app/services/action_execution.py").read_text(encoding="utf-8")
@@ -28,39 +45,103 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded():
image_id = "sha256:" + "b" * 64
spec = helper_container_spec(revision=revision, image_id=image_id)
assert HELPER_NAME == "itworx-controller-artifact-promoter"
assert CONTROLLER_VERSION == "2.1.13"
assert HELPER_NAME == "itworx-controller-artifact-promoter-modern"
assert CONTROLLER_VERSION == "2.1.14"
assert re.fullmatch(r"2\.1\.\d+", CONTROLLER_VERSION)
assert spec["Image"] == image_id
assert spec["HostConfig"]["Privileged"] is False
assert spec["HostConfig"]["NetworkMode"] == "host"
assert spec["HostConfig"]["SecurityOpt"] == ["no-new-privileges:true"]
assert spec["HostConfig"]["ReadonlyRootfs"] is True
assert spec["HostConfig"]["PidsLimit"] <= 128
assert spec["HostConfig"]["Memory"] <= 1024 * 1024 * 1024
assert spec["HostConfig"]["CpuQuota"] <= spec["HostConfig"]["CpuPeriod"]
assert set(spec["HostConfig"]["Tmpfs"]) == {"/tmp"}
assert set(spec["HostConfig"]["Binds"]) == {
"/mnt/cache:/mnt/cache",
"/mnt/user:/mnt/user",
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
"/var/lib/docker:/var/lib/docker:ro",
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
"/var/run/docker.sock:/var/run/docker.sock",
}
assert spec["Cmd"][:2] == ["/bin/bash", "-lc"]
command = spec["Cmd"][2]
assert "scripts/upgrade_deployment_controller.sh" in command
assert "UPGRADE_DEPLOYMENT_CONTROLLER_V2" in command
assert revision in command
assert CONTROLLER_VERSION in command
assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command
assert "mktemp -d -p /tmp itworx-controller-artifact-promoter-" in command
assert f"itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX" in command
assert "/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-" not in command
assert "work='/mnt/cache" not in command
assert "rm -rf -- \"$work\"" in command
assert "scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"" in command
assert "chmod 0600 \"$work/controller.tar\"" in command
assert "scripts/prepare_controller_hub_release_volume.py" in command
assert "scripts/run_repopulse_adoption.py" in command
assert "UPGRADE_DEPLOYMENT_CONTROLLER_V2" in command
assert revision in command
assert "--image-id" in command
assert CONTROLLER_VERSION in command
assert "version='2.1.14'" in command
assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command
archive_start = command.index('git --git-dir="$repo" archive')
archive_end = command.index('> "$work/controller.tar"', archive_start)
assert "scripts/vendor" in command[archive_start:archive_end]
assert "scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"" in command
assert f"--image-id '{image_id}'" in command
assert '--image-id "$image_id"' in command
def test_controller_promoter_bootstrap_child_spec_binds_legacy_target_identity():
from backend.app.services.controller_artifact_promoter import helper_container_spec
revision = "a" * 40
image_id = "sha256:" + "b" * 64
target_id = "c" * 64
target_revision = "d" * 40
target_tree = "e" * 40
spec = helper_container_spec(
revision=revision,
image_id=image_id,
current_controller_id=target_id,
current_controller_revision=target_revision,
current_controller_tree=target_tree,
current_controller_version="2.1.12",
)
assert spec["Image"] == image_id
assert spec["Labels"] == {
"itworx.controller.promoter": "true",
"itworx.source.repository": "jens/unraid-appops-gateway",
"itworx.source.revision": revision,
"itworx.controller.version": "2.1.14",
"itworx.controller.bootstrap.target.id": target_id,
"itworx.controller.bootstrap.target.image": image_id,
"itworx.controller.bootstrap.target.revision": target_revision,
"itworx.controller.bootstrap.target.tree": target_tree,
"itworx.controller.bootstrap.target.version": "2.1.12",
}
def test_controller_promoter_rejects_partial_or_mutable_bootstrap_binding():
from backend.app.services.controller_artifact_promoter import helper_container_spec
base = {"revision": "a" * 40, "image_id": "sha256:" + "b" * 64}
with pytest.raises(ValueError):
helper_container_spec(**base, current_controller_id="c" * 64)
with pytest.raises(ValueError):
helper_container_spec(
**base,
current_controller_id="c" * 64,
current_controller_revision="../controller",
current_controller_tree="e" * 40,
current_controller_version="2.1.12",
)
with pytest.raises(ValueError):
helper_container_spec(
**base,
current_controller_id="../controller",
current_controller_revision="d" * 40,
current_controller_tree="e" * 40,
current_controller_version="2.1.12",
)
@pytest.mark.parametrize(
"revision,image_id",
[
backend/tests/test_controller_bootstrap_import_host.py
+41
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import importlib.util
import json
import os
from pathlib import Path
import pytest
@@ -41,3 +42,43 @@ def test_bootstrap_passes_only_on_stdin(monkeypatch):
assert observed["input"] == b"sensitive-test-value"
assert "sensitive-test-value" not in " ".join(observed["argv"])
assert observed["capture_output"] is True
@pytest.mark.skipif(os.name == "nt", reason="POSIX metadata and symlinks")
def test_bootstrap_import_rejects_symlinks_and_multiple_links(tmp_path, monkeypatch):
monkeypatch.setattr(host_import.os, "geteuid", lambda: 0)
secret = tmp_path / "bootstrap"
secret.write_bytes(b"x" * 64)
secret.chmod(0o600)
link = tmp_path / "link"
link.symlink_to(secret)
with pytest.raises(ValueError):
host_import.read_bootstrap(link)
link.unlink()
os.link(secret, link)
with pytest.raises(ValueError):
host_import.read_bootstrap(secret)
def test_host_import_uses_protected_volume_configuration():
assert host_import.BOOTSTRAP_FILE.as_posix() == (
"/var/lib/docker/volumes/itworx-controller-hub-releases/_data/controller-config/"
"appops-enrollment-bootstrap"
)
@pytest.mark.skipif(os.name == "nt", reason="POSIX mode checks")
def test_bootstrap_import_rechecks_private_mode_after_open(tmp_path, monkeypatch):
secret = tmp_path / "bootstrap"
secret.write_bytes(b"x" * 64)
secret.chmod(0o600)
monkeypatch.setattr(host_import.os, "geteuid", lambda: 0)
original_open = host_import.os.open
def change_mode_before_open(path, flags):
secret.chmod(0o644)
return original_open(path, flags)
monkeypatch.setattr(host_import.os, "open", change_mode_before_open)
with pytest.raises(ValueError, match="changed while opening"):
host_import.read_bootstrap(secret)
backend/tests/test_controller_config_paths.py
+69
View File
@@ -1,7 +1,15 @@
from __future__ import annotations
import ast
import importlib.util
import json
import os
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
ROOT = Path(__file__).resolve().parents[2]
@@ -114,3 +122,64 @@ def test_status_reads_hub_state_through_the_fixed_pinned_adapter(monkeypatch, tm
monkeypatch.setattr(mcp_hub_release_store, "read_hub_state_bytes", lambda: b'\xff')
assert controller._deployment_state("itworx-mcp-hub") == {}
@pytest.mark.skipif(os.name == "nt", reason="the registration script uses POSIX flock")
@pytest.mark.parametrize("matching_aliases", [True, False])
def test_embedded_registry_writer_uses_protected_aliases_and_preserves_code(
monkeypatch, tmp_path, matching_aliases
):
from backend.app.services.deployment_target_registry import _REGISTER_SCRIPT
registry = tmp_path / "protected" / "projects.json"
registry.parent.mkdir(mode=0o755)
registry.write_text("{}\n")
registry.chmod(0o600)
readonly = registry if matching_aliases else tmp_path / "different-registry.json"
if not matching_aliases:
readonly.write_text("{}\n")
readonly.chmod(0o600)
code = tmp_path / "old-code"
code.mkdir()
old_registry = code / "projects.json"
old_registry.write_text('{"old":"must stay unchanged"}\n')
original = old_registry.read_bytes()
old_host_registry = tmp_path / "old-host-registry.json"
old_host_registry.write_text("{}\n")
repositories = tmp_path / "repositories"
bare = repositories / "jens" / "example.git"
bare.mkdir(parents=True)
paths = {
"/controller-config/projects.json": readonly,
"/controller-hub-releases/controller-config/projects.json": registry,
"/opt/gitea-deploy/projects.json": old_registry,
"/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json": old_host_registry,
"/mnt/cache/appdata/gitea-deploy-runners/deploy.lock": tmp_path / "deploy.lock",
"/gitea-repositories": repositories,
}
def fixture_path(value):
return paths.get(str(value), Path(value))
revision = "a" * 40
monkeypatch.setattr(
subprocess, "run", lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stdout=revision + "\n")
)
project = {"branch": "main", "slug": "example", "compose_files": ["docker-compose.yml"]}
monkeypatch.setattr(sys, "argv", ["script", "jens/example", revision, "main", json.dumps(project)])
tree = ast.parse(_REGISTER_SCRIPT)
# Redirect only filesystem paths into disposable fixtures. All production
# validation, locking, backup, atomic replace and readback code executes.
tree.body = [node for node in tree.body if not (isinstance(node, ast.ImportFrom) and node.module == "pathlib")]
if matching_aliases:
exec(compile(tree, "registry-writer", "exec"), {"Path": fixture_path})
assert json.loads(registry.read_text()) == {"jens/example": project}
assert registry.stat().st_mode & 0o777 == 0o600
assert list(registry.parent.glob("projects.json.appops-backup-*"))
else:
with pytest.raises(SystemExit) as error:
exec(compile(tree, "registry-writer", "exec"), {"Path": fixture_path})
assert error.value.code == 36
assert registry.read_text() == "{}\n"
assert old_registry.read_bytes() == original
assert old_host_registry.read_text() == "{}\n"
backend/tests/test_controller_hub_release_volume.py
+175 -4
View File
@@ -57,6 +57,109 @@ def test_only_the_exact_local_volume_is_accepted():
module.validate_volume(volume())
def writer_inventory(inherited_labels):
identifier = "c" * 64
spec = module._writer_spec(IMAGE_ID, REVISION, kind="config")
return {
"Id": identifier, "Name": f"/{module.WRITER_NAME}", "Image": IMAGE_ID,
"Config": {
"Image": IMAGE_ID, "User": "0:0", "Entrypoint": ["python3"], "Cmd": spec[-2:],
"Labels": {**inherited_labels, module.WRITER_LABEL: "true",
module.WRITER_TARGET_LABEL: "config", module.WRITER_REVISION_LABEL: REVISION},
},
"HostConfig": {
"NetworkMode": "none", "Binds": [], "Privileged": False, "ReadonlyRootfs": True,
"CpuQuota": module.WRITER_CPU_QUOTA, "CpuPeriod": module.WRITER_CPU_PERIOD,
"Memory": module.WRITER_MEMORY_BYTES, "PidsLimit": module.WRITER_PIDS_LIMIT,
"CapDrop": ["ALL"], "Tmpfs": {"/tmp": "rw,nosuid,nodev,size=16777216,mode=1777"},
"SecurityOpt": ["no-new-privileges:true"],
},
"Mounts": [{"Type": "volume", "Name": module.VOLUME,
"Source": module.MOUNTPOINT.as_posix(), "Destination": module.TARGET, "RW": True}],
}
@pytest.mark.parametrize("damage", [None, "changed_image_label", "extra_label", "reserved_image_label", "falsey_labels"])
def test_writer_labels_are_bound_to_exact_image_inheritance(monkeypatch, damage):
inherited = {"org.opencontainers.image.revision": "d" * 40, "io.buildah.version": "1.40.0"}
item = writer_inventory(inherited)
if damage == "changed_image_label":
item["Config"]["Labels"]["org.opencontainers.image.revision"] = "e" * 40
elif damage == "extra_label":
item["Config"]["Labels"]["unexpected"] = "value"
elif damage == "reserved_image_label":
inherited[module.WRITER_LABEL] = "inherited-marker"
elif damage == "falsey_labels":
inherited = []
def run(arguments, **kwargs):
if arguments == ["docker", "inspect", item["Id"]]:
return json.dumps([item]).encode()
assert arguments == ["docker", "image", "inspect", IMAGE_ID]
return json.dumps([{"Id": IMAGE_ID, "Config": {"Labels": inherited}}]).encode()
monkeypatch.setattr(module, "_run", run)
if damage:
with pytest.raises(module.VolumeTransitionError, match="writer_.*invalid"):
module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config")
else:
assert module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config") == item
@pytest.mark.parametrize("damage", [None, "tmpfs_projection", "extra_bind", "wrong_volume", "read_only"])
def test_writer_accepts_declared_tmpfs_without_inspect_mount_projection(monkeypatch, damage):
item = writer_inventory({})
if damage == "tmpfs_projection":
item["Mounts"].append({"Type": "tmpfs", "Destination": "/tmp"})
elif damage == "extra_bind":
item["Mounts"].append({"Type": "bind", "Destination": "/foreign"})
elif damage == "wrong_volume":
item["Mounts"][0]["Name"] = "foreign-volume"
elif damage == "read_only":
item["Mounts"][0]["RW"] = False
def run(arguments, **kwargs):
if arguments == ["docker", "inspect", item["Id"]]:
return json.dumps([item]).encode()
assert arguments == ["docker", "image", "inspect", IMAGE_ID]
return json.dumps([{"Id": IMAGE_ID, "Config": {"Labels": None}}]).encode()
monkeypatch.setattr(module, "_run", run)
if damage in {"extra_bind", "wrong_volume", "read_only"}:
with pytest.raises(module.VolumeTransitionError, match="writer_mount_invalid"):
module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config")
else:
module._inspect_writer(item["Id"], IMAGE_ID, REVISION, kind="config")
@pytest.mark.parametrize("name", ["foreign-project", "", None, "${COMPOSE_PROJECT_NAME}"])
def test_source_compose_name_cannot_be_overridden_by_fixed_project_flag(monkeypatch, name):
def never_compose(*args, **kwargs):
pytest.fail("Foreign source project reached Compose")
monkeypatch.setattr(module, "_run", never_compose)
source = f"name: {json.dumps(name)}\nservices:\n deploy-control:\n image: immutable-controller\n".encode()
with pytest.raises(module.VolumeTransitionError, match="project_invalid"):
module.rendered_model(source, REVISION, IMAGE_ID)
def test_compose_without_explicit_name_retains_canonical_project(monkeypatch):
def run(arguments, *, data=None, **kwargs):
if arguments[:3] == ["docker", "image", "inspect"]:
return IMAGE_ID.encode()
normalized = model()
normalized["name"] = (
arguments[arguments.index("--project-name") + 1]
if "--project-name" in arguments else "data"
)
return json.dumps(normalized).encode()
monkeypatch.setattr(module, "_run", run)
original = b"services:\n deploy-control:\n image: immutable-controller\n"
rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID))
assert rendered["name"] == "gitea-deploy-runners"
def model():
return {
"name": "gitea-deploy-runners",
@@ -146,12 +249,30 @@ def test_compose_uses_protected_code_and_two_config_views(compose):
]
assert compose_calls
assert all("--project-directory" in arguments for arguments in compose_calls)
assert all(module.MOUNTPOINT.as_posix() in arguments for arguments in compose_calls)
assert module.ROOT.as_posix() in compose_calls[0]
assert all(module.MOUNTPOINT.as_posix() in arguments for arguments in compose_calls[1:])
assert all("--env-file" in arguments and "/dev/null" in arguments for arguments in compose_calls)
assert all("-f" in arguments and "-" in arguments for arguments in compose_calls)
assert calls[0][1] == ORIGINAL_COMPOSE
def test_source_relative_controller_bind_uses_original_compose_directory(monkeypatch):
def run(arguments, *, data=None, **kwargs):
if arguments[:3] == ["docker", "image", "inspect"]:
return IMAGE_ID.encode()
normalized = model()
directory = arguments[arguments.index("--project-directory") + 1]
if arguments[-2:] == ["--format", "json"]:
normalized["services"][module.SERVICE]["volumes"][1]["source"] = directory + "/controller"
return json.dumps(normalized).encode()
monkeypatch.setattr(module, "_run", run)
original = ORIGINAL_COMPOSE + b" volumes:\n - ./controller:/opt/gitea-deploy:ro\n"
rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID))
mounts = rendered["services"][module.SERVICE]["volumes"]
assert next(m["source"] for m in mounts if m["target"] == module.EXECUTABLE_TARGET) == module._mount_source_for_revision(REVISION)
@pytest.mark.parametrize("damage", ["target_collision", "foreign_volume", "graph_rw", "relative_bind", "env_file"])
def test_compose_rejects_unsafe_or_unpinned_activation_inputs(compose, damage):
baseline, _ = compose
@@ -191,6 +312,55 @@ def test_compose_tag_replacement_cannot_change_controller_startup_image(compose,
)
@pytest.mark.parametrize("damage", [None, "foreign_reference", "wrong_project", "wrong_service", "unhealthy", "digest_reference", "wrong_image", "normalized_reference", "interpolation"])
def test_drifted_tag_requires_healthy_current_controller_provenance(compose, monkeypatch, damage):
baseline, calls = compose
reference = "registry.example/controller:stable"
if damage == "digest_reference":
reference = "registry.example/controller@sha256:" + "c" * 64
baseline["services"][module.SERVICE]["image"] = reference
current = {
"Image": IMAGE_ID,
"Config": {"Image": reference, "Labels": {
"com.docker.compose.project": "gitea-deploy-runners",
"com.docker.compose.service": module.SERVICE,
}},
"State": {"Health": {"Status": "healthy"}},
}
if damage == "foreign_reference":
current["Config"]["Image"] = "registry.example/foreign:stable"
elif damage == "wrong_project":
current["Config"]["Labels"]["com.docker.compose.project"] = "foreign"
elif damage == "wrong_service":
current["Config"]["Labels"]["com.docker.compose.service"] = "foreign"
elif damage == "unhealthy":
current["State"]["Health"]["Status"] = "unhealthy"
elif damage == "wrong_image":
current["Image"] = "sha256:" + "c" * 64
original_run = module._run
def drifted_tag(arguments, **kwargs):
if arguments[:3] == ["docker", "image", "inspect"]:
return ("sha256:" + "c" * 64).encode("ascii")
return original_run(arguments, **kwargs)
monkeypatch.setattr(module, "_run", drifted_tag)
source_reference = reference
if damage == "normalized_reference":
source_reference = "registry.example/foreign:stable"
elif damage == "interpolation":
source_reference = "${CONTROLLER_IMAGE}"
original = f"name: gitea-deploy-runners\nservices:\n deploy-control:\n image: {source_reference}\n".encode()
if damage:
with pytest.raises(module.VolumeTransitionError, match="image_reference_(changed|invalid)"):
module.rendered_model(original, REVISION, IMAGE_ID, current_container=current)
else:
rendered = json.loads(module.rendered_model(original, REVISION, IMAGE_ID, current_container=current))
assert rendered["services"][module.SERVICE]["image"] == IMAGE_ID
assert rendered["services"][module.SERVICE]["pull_policy"] == "never"
assert not any("up" in arguments or "pull" in arguments for arguments, _ in calls)
def test_config_snapshot_has_exact_allowlist_and_broker_token():
entries = {
"projects.json": (b"{}", 0o600),
@@ -482,7 +652,7 @@ def test_installer_release_order_matches_renderer_contract():
renderer_spec.loader.exec_module(renderer)
assert tuple(module.RELEASE_FILES) == tuple(renderer._ARTIFACT_FILES)
assert module.RELEASE_FILES[:3] == (
"controller_paths.py", "mcp_hub_release_store.py", "deploy.py"
"controller_paths.py", "mcp_hub_release_store.py", "hub_permission_preparation.py"
)
@@ -566,11 +736,12 @@ def test_private_stage_rejects_symlinked_source_directory():
shutil.rmtree(work)
def test_source_compose_preflight_rejects_host_file_reads_before_compose(compose):
@pytest.mark.parametrize("directive", ["env_file", "label_file"])
def test_source_compose_preflight_rejects_host_file_reads_before_compose(compose, directive):
_baseline, calls = compose
unsafe = (
b"name: gitea-deploy-runners\nservices:\n deploy-control:\n"
b" image: x\n env_file: /tmp/secret\n"
+ f" image: x\n {directive}: /tmp/secret\n".encode()
)
with pytest.raises(module.VolumeTransitionError, match="external_source_invalid"):
module.rendered_model(unsafe, REVISION, IMAGE_ID)
backend/tests/test_controller_state_reconciliation.py
+12 -2
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import importlib.util
import os
import re
import shlex
import stat
from pathlib import Path
@@ -211,7 +213,9 @@ def test_upgrade_prepares_and_validates_captured_optional_private_tls_before_sto
def test_upgrade_rolls_back_state_before_restoring_or_starting_the_old_writer():
script = UPGRADE_PATH.read_text(encoding="utf-8")
restore = script[script.index("restore_previous() {") : script.index("cleanup() {")]
restore_start = script.index("restore_previous() {")
cleanup_start = script.index("cleanup() {", restore_start)
restore = script[restore_start:cleanup_start]
stop = 'if ! python3 "${work}/${PREPARER_NAME}" rollback --backup "$backup" --stop-only; then'
rollback = '--rollback'
@@ -245,6 +249,12 @@ def test_wrapper_archives_the_fixed_operator_handoff_required_by_root_upgrade():
wrapper = WRAPPER_PATH.read_text(encoding="utf-8")
upgrade = UPGRADE_PATH.read_text(encoding="utf-8")
archive_line = next(line for line in wrapper.splitlines() if "git archive" in line)
source_paths_match = re.search(r"(?ms)^source_paths=\((.*?)^\)", upgrade)
assert "scripts/run_repopulse_adoption.py" in archive_line
assert '"scripts/run_repopulse_adoption.py"' in upgrade
assert source_paths_match is not None
assert "scripts/run_repopulse_adoption.py" in shlex.split(source_paths_match.group(1))
assert (
'install -o root -g root -m 0700 "${script_dir}/run_repopulse_adoption.py" '
'"${work}/source/controller/run_repopulse_adoption.py"'
) in upgrade
backend/tests/test_deploy_broker.py
+1438 -4
View File
File diff suppressed because it is too large. Load diff
backend/tests/test_deployment_target_registry.py
+1 -1
View File
@@ -87,7 +87,7 @@ def test_derives_bounded_single_container_topology_and_uses_fixed_exec():
assert '"source_dir":"/mnt/cache/appdata/gitea-deploy-runners/sources/certscope"' in command[6]
assert '"containers":["certscope"]' in command[6]
script = command[2]
assert '/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json' in script
assert '/controller-hub-releases/controller-config/projects.json' in script
assert '/srv/example/appdata/gitea-deploy-runners/deploy.lock' not in script
assert '/mnt/cache/appdata/gitea-deploy-runners/deploy.lock' in script
assert 'os.path.samefile(path, host_path)' in script
backend/tests/test_gitea_actions.py
+57
View File
@@ -95,6 +95,63 @@ def test_runner_snapshot_recommends_only_approval_gated_restart_for_stalled_queu
assert "fixture-secret" not in repr(result)
def test_pending_jobs_do_not_trigger_runnable_queue_diagnostics(monkeypatch):
_configure(monkeypatch)
monkeypatch.setitem(
settings.apis,
"GITEA_RUNNER_TARGETS_JSON",
'{"itworx-ci-build":"itworx-security-runner"}',
)
connector = GiteaActionsConnector()
created_at = (datetime.now(UTC) - timedelta(minutes=15)).isoformat()
requested_statuses = []
def fake_request(path, params=None):
if path.endswith("/runners"):
return {
"total_count": 1,
"runners": [
{
"id": 1,
"name": "itworx-security-runner",
"status": "online",
"busy": False,
"disabled": False,
"labels": ["linux-validation"],
}
],
}
status = params["status"]
requested_statuses.append(status)
if status == "pending":
return {
"total_count": 1,
"jobs": [
{
"id": 171,
"name": "conditional release full",
"status": "requested",
"created_at": created_at,
"labels": ["itworx-ci-build"],
}
],
}
return {"total_count": 0, "jobs": []}
monkeypatch.setattr(connector, "_request", fake_request)
result = connector.snapshot(refresh=True)
assert requested_statuses == ["queued", "pending", "in_progress"]
assert result["severity"] == "healthy"
assert result["queue"]["queued"] == 0
assert result["queue"]["oldestQueuedAgeSeconds"] is None
assert result["queue"]["sampleTruncated"] is False
assert result["queue"]["blockedByRunnerLabels"] == []
assert result["queue"]["staleAssignmentRunnerLabels"] == []
assert result["queue"]["items"] == []
assert result["recommendedAction"] is None
def test_runner_snapshot_is_cached_and_healthy_when_queue_is_clear(monkeypatch):
_configure(monkeypatch)
connector = GiteaActionsConnector()
backend/tests/test_host_guard_memory_threshold_installer.py
+228
View File
@@ -0,0 +1,228 @@
from __future__ import annotations
import contextlib
import hashlib
import importlib.util
import os
import re
import shutil
import subprocess
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
INSTALLER_PATH = (
ROOT
/ "ops"
/ "gitea-deploy-broker"
/ "tune_host_guard_memory_thresholds.py"
)
SPEC = importlib.util.spec_from_file_location(
"host_guard_memory_threshold_installer_test", INSTALLER_PATH
)
assert SPEC and SPEC.loader
installer = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(installer)
@pytest.fixture
def guard_script() -> bytes:
return b"""#!/usr/bin/env bash
set -euo pipefail
readonly MEM_PAUSE_KB=4194304
readonly MEM_RESUME_KB=6291456
readonly SWAP_CRITICAL_MEMORY_KB=8388608
readonly SWAP_PAUSE_KBPS=1024
readonly SWAP_RESUME_KBPS=256
readonly SWAP_CRITICAL_SAMPLES=3
if [ \"$mem_kb\" -lt \"$MEM_PAUSE_KB\" ]; then
runner_critical=1
fi
"""
def _baseline(monkeypatch, payload: bytes) -> None:
monkeypatch.setattr(installer, "BASELINE_SHA256", hashlib.sha256(payload).hexdigest())
def _assignment_values(payload: bytes) -> dict[str, int]:
rows = re.findall(rb"(?m)^readonly ([A-Z0-9_]+)=([0-9]+)$", payload)
return {name.decode("ascii"): int(value) for name, value in rows}
def _fixture_install(monkeypatch, tmp_path, guard_script, *, fsync_fails=False):
target = tmp_path / "script"
backup = tmp_path / ".script.backup"
lock = tmp_path / ".lock"
target.write_bytes(guard_script)
_baseline(monkeypatch, guard_script)
monkeypatch.setattr(installer, "TARGET_PATH", target)
monkeypatch.setattr(installer, "BACKUP_PATH", backup)
monkeypatch.setattr(installer, "LOCK_PATH", lock)
monkeypatch.setattr(installer, "_assert_fixed_environment", lambda: target.stat())
monkeypatch.setattr(installer, "_exclusive_lock", contextlib.nullcontext)
monkeypatch.setattr(
installer,
"_read_private_backup",
lambda: backup.read_bytes() if backup.exists() else None,
)
events: list[str] = []
def assert_private_file(path, *, allow_missing=False):
if path == backup:
events.append("backup_identity")
elif path == target:
events.append("target_identity")
return path.lstat() if path.exists() else None
def write_backup(path, payload):
events.append("backup_create")
with path.open("xb") as handle:
handle.write(payload)
handle.flush()
os.fsync(handle.fileno())
def write_stage(payload):
stage = tmp_path / f".stage-{len(events)}"
stage.write_bytes(payload)
return stage
def fsync_directory(_path):
events.append("directory_fsync")
if fsync_fails:
raise OSError("fixture directory fsync failure")
def atomic_replace(stage, destination):
events.append("replace")
os.replace(stage, destination)
events.append("post_replace_fsync")
original_read_bytes = Path.read_bytes
def traced_read_bytes(path):
payload = original_read_bytes(path)
if path == backup:
events.append("backup_readback")
return payload
monkeypatch.setattr(installer, "_assert_private_regular_file", assert_private_file)
monkeypatch.setattr(installer, "_write_private_file_exclusive", write_backup)
monkeypatch.setattr(installer, "_write_stage", write_stage)
monkeypatch.setattr(installer, "_bash_syntax_check", lambda _path: None)
monkeypatch.setattr(installer, "_fsync_directory", fsync_directory)
monkeypatch.setattr(installer, "_atomic_replace", atomic_replace)
monkeypatch.setattr(Path, "read_bytes", traced_read_bytes)
return target, backup, events
def test_fixture_patch_changes_only_reviewed_memory_literals(guard_script, monkeypatch):
_baseline(monkeypatch, guard_script)
candidate = installer._render_candidate(guard_script)
before = _assignment_values(guard_script)
after = _assignment_values(candidate)
assert after == {
**before,
"MEM_PAUSE_KB": 2_097_152,
"MEM_RESUME_KB": 4_194_304,
"SWAP_CRITICAL_MEMORY_KB": 4_194_304,
}
changed = [
(old.decode("ascii"), new.decode("ascii"))
for old, new in zip(guard_script.splitlines(), candidate.splitlines())
if old != new
]
assert changed == [
("readonly MEM_PAUSE_KB=4194304", "readonly MEM_PAUSE_KB=2097152"),
("readonly MEM_RESUME_KB=6291456", "readonly MEM_RESUME_KB=4194304"),
(
"readonly SWAP_CRITICAL_MEMORY_KB=8388608",
"readonly SWAP_CRITICAL_MEMORY_KB=4194304",
),
]
def test_candidate_passes_real_bash_syntax(guard_script, monkeypatch, tmp_path):
bash = shutil.which("bash")
if not bash:
pytest.skip("bash is required for this fixture syntax check")
_baseline(monkeypatch, guard_script)
candidate = installer._render_candidate(guard_script)
fixture = tmp_path / "Host_Resource_Guard.script"
fixture.write_bytes(candidate)
fixture_argument = str(fixture)
if os.name == "nt":
# Windows bash.exe here is the WSL launcher; translate C:\... to /mnt/c/...
drive = fixture.drive.rstrip(":").lower()
fixture_argument = f"/mnt/{drive}/{fixture.as_posix()[3:]}"
completed = subprocess.run(
[bash, "-n", fixture_argument], check=False, capture_output=True, timeout=15
)
assert completed.returncode == 0, completed.stderr.decode("utf-8", errors="replace")
def test_baseline_fingerprint_mismatch_refuses_candidate(guard_script, monkeypatch):
_baseline(monkeypatch, guard_script)
changed = guard_script + b"# unreviewed source change\n"
with pytest.raises(installer.InstallError, match="baseline_fingerprint_mismatch"):
installer._render_candidate(changed)
def test_duplicate_or_changed_memory_literal_refuses_candidate(guard_script, monkeypatch):
duplicate = guard_script + b"readonly MEM_PAUSE_KB=4194304\n"
_baseline(monkeypatch, duplicate)
with pytest.raises(installer.InstallError, match="threshold_shape_invalid:MEM_PAUSE_KB"):
installer._render_candidate(duplicate)
wrong_rate = guard_script.replace(
b"readonly SWAP_RESUME_KBPS=256", b"readonly SWAP_RESUME_KBPS=512"
)
_baseline(monkeypatch, wrong_rate)
with pytest.raises(installer.InstallError, match="threshold_shape_invalid:SWAP_RESUME_KBPS"):
installer._render_candidate(wrong_rate)
def test_backup_is_readback_verified_and_directory_synced_before_replace(
guard_script, monkeypatch, tmp_path
):
target, backup, events = _fixture_install(
monkeypatch, tmp_path, guard_script
)
status, after_sha = installer.install()
assert status == "applied"
assert hashlib.sha256(target.read_bytes()).hexdigest() == after_sha
assert backup.read_bytes() == guard_script
replace_at = events.index("replace")
assert events.index("backup_create") < events.index("backup_readback")
assert events.index("backup_readback") < events.index("directory_fsync")
assert events.index("directory_fsync") < max(
index
for index, event in enumerate(events[:replace_at])
if event == "target_identity"
) < replace_at
assert events.index("post_replace_fsync") > replace_at
def test_backup_directory_sync_failure_leaves_target_unmodified(
guard_script, monkeypatch, tmp_path
):
target, backup, events = _fixture_install(
monkeypatch, tmp_path, guard_script, fsync_fails=True
)
with pytest.raises(installer.InstallError, match="backup_directory_sync_failed"):
installer.install()
assert target.read_bytes() == guard_script
assert backup.read_bytes() == guard_script
assert "replace" not in events
backend/tests/test_hub_candidate_delivery.py
+194
View File
@@ -0,0 +1,194 @@
from __future__ import annotations
import importlib.util
import json
import sys
import types
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
@pytest.fixture
def controller(monkeypatch):
path = ROOT / "ops/gitea-deploy-broker/status_server.py"
spec = importlib.util.spec_from_file_location("hub_candidate_delivery_test", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
monkeypatch.setattr(module, "_gitea_validation_settings",
lambda: ("https://gitea.example.com/api/v1", b"x" * 40))
return module
def test_candidate_api_reads_only_fixed_hub_paths(controller, monkeypatch):
opened = []
def run(argv, **kwargs):
opened.append((argv, kwargs))
assert argv == [sys.executable, str(controller.ROOT / "hub_candidate_transport.py"), "json"]
assert kwargs["timeout"] == 20
assert kwargs["pass_fds"] == ()
assert kwargs["stderr"] == controller.subprocess.DEVNULL
assert set(kwargs["env"]) == {"PATH", "PYTHONDONTWRITEBYTECODE", "PYTHONIOENCODING"}
return types.SimpleNamespace(returncode=0, stdout=b'{"workflows":[]}')
monkeypatch.setattr(controller.subprocess, "run", run)
prefix = "/repos/jens/itworx_mcp_hub"
assert controller._hub_candidate_api_json(prefix + "/actions/workflows", query={}) == {"workflows": []}
payload = json.loads(opened[0][1]["input"])
assert payload["url"] == "https://gitea.example.com/api/v1" + prefix + "/actions/workflows"
assert payload["outputFd"] is None
assert payload["headers"]["Authorization"] == "token " + "x" * 40
assert all("x" * 40 not in item for item in opened[0][0])
for path, query in (
("https://other.invalid/api/v1", {}),
(prefix + "/../other/actions/workflows", {}),
(prefix + "/actions/runs", {"event": "pull_request"}),
(prefix + "/actions/runs/1/jobs", {"page": "11", "limit": "100"}),
(prefix + "/actions/runs/1/artifacts", {"name": "other-artifact"}),
):
with pytest.raises(controller._HubCandidateTransportError):
controller._hub_candidate_api_json(path, query=query)
assert len(opened) == 1
@pytest.mark.parametrize("raw", [b'{"id":1,"id":2}', b"x" * (2 * 1024 * 1024 + 1)])
def test_candidate_metadata_is_bounded_and_rejects_duplicate_keys(controller, monkeypatch, raw):
monkeypatch.setattr(controller.subprocess, "run", lambda *_args, **_kwargs:
types.SimpleNamespace(returncode=0, stdout=raw))
with pytest.raises(controller._HubCandidateTransportError):
controller._hub_candidate_api_json("/repos/jens/itworx_mcp_hub/branches/main", query={})
def test_candidate_transport_deadline_is_parent_enforced(controller, monkeypatch):
def blocked(argv, **kwargs):
assert kwargs["timeout"] == 20
raise controller.subprocess.TimeoutExpired(argv, kwargs["timeout"], output=b"private")
monkeypatch.setattr(controller.subprocess, "run", blocked)
with pytest.raises(controller._HubCandidateTransportError, match="read_unavailable") as error:
controller._hub_candidate_api_json("/repos/jens/itworx_mcp_hub/branches/main", query={})
assert "private" not in str(error.value)
def test_artifact_transport_writes_only_parent_owned_fd(controller, monkeypatch, tmp_path):
import os
controller.STATE_ROOT = tmp_path
parent = tmp_path / "mcp-hub-release-imports"
parent.mkdir(mode=0o700)
destination = parent / "candidate.zip"
def run(argv, **kwargs):
assert argv[-1] == "zip"
assert kwargs["timeout"] == 1800
payload = json.loads(kwargs["input"])
assert set(payload) == {"url", "headers", "maxBytes", "outputFd"}
assert str(destination) not in kwargs["input"].decode()
assert kwargs["pass_fds"] == (payload["outputFd"],)
os.write(payload["outputFd"], b"zip data")
return types.SimpleNamespace(returncode=0, stdout=b'{"bytes":8}\n')
monkeypatch.setattr(controller.subprocess, "run", run)
controller._hub_candidate_download_artifact(123, destination, 1024)
assert destination.read_bytes() == b"zip data"
assert destination.stat().st_mode & 0o777 == 0o600
@pytest.mark.parametrize("receipt", [b'{"bytes":9}', b'{"bytes":true}', b'{"bytes":0}'])
def test_artifact_receipt_must_match_actual_file(controller, monkeypatch, tmp_path, receipt):
controller.STATE_ROOT = tmp_path
parent = tmp_path / "mcp-hub-release-imports"
parent.mkdir(mode=0o700)
monkeypatch.setattr(controller.subprocess, "run", lambda *_args, **_kwargs:
types.SimpleNamespace(returncode=0, stdout=receipt))
with pytest.raises(controller._HubCandidateTransportError):
controller._hub_candidate_download_artifact(123, parent / "candidate.zip", 1024)
def _delivery(controller, monkeypatch, tmp_path, outcome):
record = {"repository": controller.CANONICAL_HUB_REPOSITORY, "revision": "a" * 40,
"status": "validation_passed", "audit": []}
marker = tmp_path / "inbox.json"
monkeypatch.setattr(controller, "_atomic_inbox_update",
lambda _marker, item: marker.write_text(json.dumps(item)))
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True)
calls = []
def prepare(*args, **kwargs):
calls.append((args, kwargs))
return {"revision": record["revision"], "status": outcome,
"candidateSha256": "b" * 64}
monkeypatch.setitem(sys.modules, "hub_candidate_import", types.SimpleNamespace(
prepare_candidate=prepare, CandidateRejected=type("CandidateRejected", (ValueError,), {}),
CandidateDeferred=type("CandidateDeferred", (RuntimeError,), {})))
return marker, record, calls
@pytest.mark.parametrize("outcome", ["awaiting_candidate", "pending_review"])
def test_candidate_wait_never_marks_deployment_started_and_throttles(controller, monkeypatch, tmp_path, outcome):
marker, record, calls = _delivery(controller, monkeypatch, tmp_path, outcome)
assert not controller._prepare_hub_candidate_delivery(marker, record, {})
assert record["status"] == "validation_passed"
assert "deployment_started" not in [entry["stage"] for entry in record["audit"]]
assert not controller._prepare_hub_candidate_delivery(marker, record, {})
assert len(calls) == 1
def test_candidate_ready_is_hash_bound_before_deployment(controller, monkeypatch, tmp_path):
marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready")
assert controller._prepare_hub_candidate_delivery(marker, record, {})
assert record["status"] == "validation_passed"
assert record["candidate_preparation"]["candidate_sha256"] == "b" * 64
assert len(calls) == 1
def test_candidate_wait_expires_without_production_failure(controller, monkeypatch, tmp_path):
marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "awaiting_candidate")
monkeypatch.setattr(controller.time, "time", lambda: 12000)
record["candidate_preparation"] = {"started_at": 1, "deadline_at": 10801,
"next_check_at": 2, "status": "awaiting_candidate"}
assert not controller._prepare_hub_candidate_delivery(marker, record, {})
assert record["status"] == "validation_timeout"
assert record["reason"] == "hub_candidate_timeout"
assert calls == []
def test_candidate_preparation_does_not_import_superseded_head(controller, monkeypatch, tmp_path):
marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready")
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: False)
assert not controller._prepare_hub_candidate_delivery(marker, record, {})
assert record["status"] == "superseded"
assert calls == []
def test_candidate_does_not_deploy_when_head_advances_during_import(controller, monkeypatch, tmp_path):
marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready")
heads = iter((True, False))
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: next(heads))
assert not controller._prepare_hub_candidate_delivery(marker, record, {})
assert len(calls) == 1
assert record["status"] == "superseded"
assert "candidate_sha256" not in record["candidate_preparation"]
assert "deployment_started" not in [entry["stage"] for entry in record["audit"]]
def test_candidate_import_cannot_overrun_the_delivery_deadline(controller, monkeypatch, tmp_path):
marker, record, calls = _delivery(controller, monkeypatch, tmp_path, "ready")
clock = [1]
monkeypatch.setattr(controller.time, "time", lambda: clock[0])
prepare = sys.modules["hub_candidate_import"].prepare_candidate
def slow(*args, **kwargs):
result = prepare(*args, **kwargs)
clock[0] = 10802
return result
monkeypatch.setattr(sys.modules["hub_candidate_import"], "prepare_candidate", slow)
assert not controller._prepare_hub_candidate_delivery(marker, record, {})
assert len(calls) == 1
assert record["status"] == "validation_timeout"
assert record["reason"] == "hub_candidate_timeout"
assert "candidate_sha256" not in record["candidate_preparation"]
backend/tests/test_hub_candidate_import.py
+837
View File
@@ -0,0 +1,837 @@
from __future__ import annotations
import hashlib
import importlib
import io
import json
import contextlib
import stat
import struct
import sys
import tarfile
import types
import zipfile
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
BROKER_DIR = ROOT / "ops" / "gitea-deploy-broker"
@pytest.fixture
def modules(monkeypatch):
if "fcntl" not in sys.modules:
shim = types.ModuleType("fcntl")
shim.LOCK_EX, shim.LOCK_UN, shim.LOCK_NB = 2, 8, 4
shim.flock = lambda *_args: None
monkeypatch.setitem(sys.modules, "fcntl", shim)
monkeypatch.syspath_prepend(str(BROKER_DIR))
for name in ("deploy", "prepare_mcp_hub_candidate", "hub_candidate_import"):
sys.modules.pop(name, None)
controller = importlib.import_module("deploy")
prepare = importlib.import_module("prepare_mcp_hub_candidate")
importer = importlib.import_module("hub_candidate_import")
monkeypatch.setattr(prepare, "_is_host_root", lambda: True)
yield types.SimpleNamespace(
controller=controller, prepare=prepare, importer=importer
)
for name in ("hub_candidate_import", "prepare_mcp_hub_candidate", "deploy"):
sys.modules.pop(name, None)
def _zip_stream_member(name: str, payload: bytes, offset: int) -> tuple[bytes, bytes]:
encoded_name = name.encode("ascii")
crc = zipfile.crc32(payload) & 0xFFFFFFFF
local = struct.pack(
"<IHHHHHIIIHH",
0x04034B50,
20,
0x8,
0,
0,
0,
0,
0,
0,
len(encoded_name),
0,
) + encoded_name + payload + struct.pack(
"<IIII", 0x08074B50, crc, len(payload), len(payload)
)
central = struct.pack(
"<IHHHHHHIIIHHHHHII",
0x02014B50,
20,
20,
0x8,
0,
0,
0,
crc,
len(payload),
len(payload),
len(encoded_name),
0,
0,
0,
0,
(stat.S_IFREG | 0o600) << 16,
offset,
) + encoded_name
return local, central
def _streamed_zip_bytes(importer, *, extra_member: bool = False) -> bytearray:
names = sorted(importer._expected_zip_members())
if extra_member:
names.append("unexpected.txt")
members = [(name, b"{}" if name != "images.tar" else b"tar-data") for name in names]
output = bytearray()
centrals: list[bytes] = []
for name, payload in members:
local, central = _zip_stream_member(name, payload, len(output))
output.extend(local)
centrals.append(central)
central_offset = len(output)
for central in centrals:
output.extend(central)
central_size = len(output) - central_offset
output.extend(
struct.pack(
"<IHHHHIIH",
0x06054B50,
0,
0,
len(members),
len(members),
central_size,
central_offset,
0,
)
)
return output
def _private_file(path: Path, data: bytes) -> None:
path.write_bytes(data)
path.chmod(0o600)
def _synthetic_scan_reports(
services, images, *, created_at: str, finding_service: str | None = None
) -> dict[str, bytes]:
report_data: dict[str, bytes] = {}
for service in services:
image_id = images[service]["imageId"]
vulnerabilities = (
[
{
"Severity": "HIGH",
"PkgName": "package",
"VulnerabilityID": "CVE-2099-0001",
"InstalledVersion": "1.0",
}
]
if service == finding_service
else []
)
vulnerability = {
"SchemaVersion": 2,
"Metadata": {
"ImageID": image_id,
"ImageConfig": {"os": "linux", "architecture": "amd64"},
},
"CreatedAt": created_at,
"Trivy": {"Version": "0.65.0"},
"Results": [
{
"Packages": [{"Name": "package", "Version": "1.0"}],
"Vulnerabilities": vulnerabilities,
}
],
}
secret = {
"ImageID": image_id,
"CreatedAt": created_at,
"TrivyVersion": "0.65.0",
"FindingCount": 0,
"SensitiveFindingDetailsPersisted": False,
"Results": [{"FindingCount": 0}],
}
report_data[f"vulnerability-{service}.json"] = json.dumps(
vulnerability
).encode()
report_data[f"secret-{service}.json"] = json.dumps(secret).encode()
report_data["trivy-db-metadata.json"] = json.dumps(
{"UpdatedAt": created_at, "DownloadedAt": created_at}
).encode()
return report_data
def test_run_identity_requires_exact_protected_push_and_no_pull_request(modules):
importer = modules.importer
revision = "a" * 40
run = {
"path": importer.RUN_PATH,
"event": "push",
"head_branch": "main",
"head_sha": revision,
"repository": {"full_name": "Jens/ITWorx_MCP_Hub"},
"head_repository": None,
"pull_requests": [],
}
assert importer._run_matches(run, revision)
for change in (
{"event": "workflow_dispatch"},
{"path": "other.yml@refs/heads/main"},
{"head_repository": {"full_name": "attacker/fork"}},
{"pull_requests": [{"number": 12}]},
{"head_sha": "b" * 40},
{"repository": {"full_name": "attacker/repository"}},
):
assert not importer._run_matches({**run, **change}, revision)
def test_selected_run_and_jobs_accept_the_live_gitea_job_shape(modules):
importer = modules.importer
revision = "a" * 40
now = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace(
"+00:00", "Z"
)
run = {
"id": 21531,
"path": importer.RUN_PATH,
"event": "push",
"head_branch": "main",
"head_sha": revision,
"repository": {"full_name": "Jens/ITWorx_MCP_Hub"},
"head_repository": None,
"pull_requests": [],
"created_at": now,
"run_attempt": 1,
"status": "completed",
"conclusion": "success",
}
job_rows = [
{
"id": 37145 + index,
"name": name,
"run_id": 21531,
"status": "completed",
"conclusion": "success",
"runner_id": 22,
"started_at": now,
"completed_at": now,
}
for index, name in enumerate(("full", "release-images"))
]
calls = []
def api(path, *, query):
calls.append((path, query))
if path.endswith("/actions/runs"):
return {"total_count": 1, "workflow_runs": [run]}
if path.endswith("/actions/runs/21531/jobs"):
return {"total_count": 2, "jobs": job_rows}
raise AssertionError(f"unexpected fixed API path: {path}")
selected, built_at = importer._selected_run(api, revision)
assert selected == run
assert built_at.endswith("Z")
assert importer._jobs_ready(api, revision, selected)
assert calls[0][1] == {
"event": "push",
"branch": "main",
"head_sha": revision,
"page": "1",
"limit": "100",
}
assert calls[1][1] == {"page": "1", "limit": "100"}
def test_required_candidate_jobs_must_both_succeed_and_bind_run(modules):
importer = modules.importer
run = {
"id": 5,
"status": "completed",
"conclusion": "success",
"run_attempt": 1,
}
rows = [
{"name": name, "run_id": 5, "status": "completed", "conclusion": "success"}
for name in ("full", "release-images")
]
def api(_path, *, query):
assert query == {"page": "1", "limit": "100"}
return {"total_count": len(rows), "jobs": rows}
assert importer._jobs_ready(api, "a" * 40, run)
rows[1] = {**rows[1], "conclusion": "failure"}
with pytest.raises(importer.CandidateRejected):
importer._jobs_ready(api, "a" * 40, run)
def test_recheck_rejects_a_new_attempt_for_the_same_workflow_run(modules, monkeypatch):
importer = modules.importer
revision = "a" * 40
original = {"id": 21531, "run_attempt": 1}
restarted = {"id": 21531, "run_attempt": 2}
artifact = {"id": 730}
monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: "b" * 40)
monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None)
monkeypatch.setattr(
importer,
"_selected_run",
lambda *_args: (restarted, "2026-10-03T10:00:00.000Z"),
)
monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True)
with pytest.raises(importer.CandidateRejected, match="run changed"):
importer._recheck_protected_artifact(
lambda *_args, **_kwargs: None,
revision=revision,
tree="b" * 40,
initial_run=original,
initial_built_at="2026-10-03T10:00:00.000Z",
initial_artifact=artifact,
)
def test_unfixable_vulnerability_finding_is_bound_to_fixed_image_reference(modules):
importer = modules.importer
controller = modules.controller
created_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace(
"+00:00", "Z"
)
images = {
service: {
"imageId": f"sha256:{index:064x}",
"imageRef": f"registry.example/{service}:fixed",
}
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1)
}
report_data = _synthetic_scan_reports(
controller.MCP_HUB_SERVICES,
images,
created_at=created_at,
finding_service=controller.MCP_HUB_SERVICES[0],
)
_reports, findings = importer._validate_reports(images, report_data)
assert len(findings) == 1
assert findings[0]["imageRef"] == images[controller.MCP_HUB_SERVICES[0]]["imageRef"]
summary = importer._candidate_findings_summary(findings, images)
assert summary["unresolvedCriticalOrHigh"][0]["affectedImages"] == [
findings[0]["imageRef"]
]
@pytest.mark.parametrize("finding_present", [False, True])
@pytest.mark.parametrize("rerun_during_load", [False, True])
def test_fake_local_import_publishes_candidate_and_requires_review_for_findings(
modules, monkeypatch, tmp_path, finding_present, rerun_during_load
):
importer = modules.importer
prepare = modules.prepare
controller = modules.controller
revision, tree, epoch = "a" * 40, "b" * 40, "1790985600"
built_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace(
"+00:00", "Z"
)
build = {
"id": revision[:12],
"repository": importer.REPOSITORY,
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": built_at,
}
compose_sha = "c" * 64
model_services = {}
images = {}
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1):
external = importer.EXTERNAL_IMAGES.get(service)
image_ref = (
external[0]
if external is not None
else f"registry.example/itworx/{service}:{revision[:12]}"
)
archive_ref = external[1] if external is not None else image_ref
model_services[service] = {"image": image_ref}
images[service] = {
"imageRef": image_ref,
"archiveRef": archive_ref,
"imageId": f"sha256:{index:064x}",
}
model = {"services": model_services}
created_at = built_at
finding_service = controller.MCP_HUB_SERVICES[0] if finding_present else None
report_data = _synthetic_scan_reports(
controller.MCP_HUB_SERVICES,
images,
created_at=created_at,
finding_service=finding_service,
)
_reports, findings = importer._validate_reports(images, report_data)
summary = importer._candidate_findings_summary(findings, images)
digests = {
"images": {
service: {
"imageId": images[service]["imageId"],
"tag": images[service]["imageRef"],
}
for service in controller.MCP_HUB_SERVICES
}
}
exception_rows = [
{
"package": finding["package"],
"cve": finding["cve"],
"installedVersion": finding["installedVersion"],
"severity": finding["severity"],
"affectedImageDigests": {
finding["imageRef"]: images_by_ref["imageId"]
for images_by_ref in [
next(
row
for row in images.values()
if row["imageRef"] == finding["imageRef"]
)
]
},
}
for finding in findings
]
draft = {"schemaVersion": "1.0", "exceptions": exception_rows}
state_root = tmp_path / "state"
state_root.mkdir(mode=0o700)
state_root.chmod(0o700)
releases_root = tmp_path / "releases"
source = releases_root / revision
source.mkdir(parents=True)
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases_root)
monkeypatch.setattr(controller, "GLOBAL_DEPLOY_LOCK", state_root / "deploy.lock")
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda *_args: None)
monkeypatch.setattr(controller, "_root_owned", lambda *_args: True)
def create_private_directory(path, *, create=False, root=None):
del root
if create:
path.mkdir(mode=0o700, parents=True, exist_ok=True)
path.chmod(0o700)
if path.is_symlink() or not path.is_dir():
raise controller.DeployError("test private directory missing")
def candidate_directory(candidate_revision):
directory = state_root / "mcp-hub-release-candidates" / candidate_revision
for path in (directory.parent, directory):
if path.is_symlink() or not path.is_dir():
raise controller.DeployError("test candidate directory missing")
return directory
monkeypatch.setattr(controller, "_mcp_hub_private_directory", create_private_directory)
monkeypatch.setattr(controller, "_mcp_hub_candidate_directory", candidate_directory)
project, repo = {"branch": "main"}, tmp_path / "repo"
monkeypatch.setattr(prepare, "_fixed_target", lambda *_args: (project, repo))
monkeypatch.setattr(controller, "deployment_metadata", lambda *_args, **_kwargs: build)
monkeypatch.setattr(controller, "mcp_hub_source_date_epoch", lambda *_args: epoch)
monkeypatch.setattr(controller, "mcp_hub_stage_release_source", lambda *_args: source)
monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True)
monkeypatch.setattr(controller, "_mcp_hub_verify_release_tree_metadata", lambda *_args: None)
monkeypatch.setattr(
prepare,
"_render",
lambda *_args, **_kwargs: (model, compose_sha),
)
monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: tree)
monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None)
run = {"id": 21531, "run_attempt": 1}
run_state = {"attempt": 1}
monkeypatch.setattr(
importer,
"_selected_run",
lambda *_args: ({**run, "run_attempt": run_state["attempt"]}, built_at),
)
monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True)
artifact_bytes = b"bounded fake Gitea artifact"
artifact = {
"id": 730,
"size_in_bytes": len(artifact_bytes),
"name": importer.ARTIFACT_PREFIX + revision,
"workflow_run": {"id": run["id"]},
}
monkeypatch.setattr(importer, "_candidate_artifact", lambda *_args: artifact)
monkeypatch.setattr(importer, "_docker_root", lambda: tmp_path)
monkeypatch.setattr(importer, "_check_disk_headroom", lambda **_kwargs: None)
monkeypatch.setattr(importer, "_parse_artifact_zip", lambda *_args, **_kwargs: (
{},
report_data,
hashlib.sha256(artifact_bytes).hexdigest(),
"d" * 64,
"e" * 64,
123,
))
monkeypatch.setattr(
importer,
"_validate_manifest",
lambda *_args, **_kwargs: (images, {}),
)
monkeypatch.setattr(importer, "_validate_docker_save", lambda *_args, **_kwargs: None)
monkeypatch.setattr(importer, "_bounded_global_deploy_lock", contextlib.nullcontext)
monkeypatch.setattr(importer, "_image_preflight", lambda *_args: None)
def fake_fixed_docker(*args, **_kwargs):
if rerun_during_load and args and args[0] == "load":
run_state["attempt"] = 2
return object()
monkeypatch.setattr(importer, "_fixed_docker", fake_fixed_docker)
monkeypatch.setattr(importer, "_verify_loaded_images", lambda *_args, **_kwargs: {})
monkeypatch.setattr(
controller,
"_mcp_hub_check_candidate_exceptions",
lambda *_args: None,
)
monkeypatch.setattr(
prepare,
"_draft_pending_exceptions",
lambda *_args: draft,
)
def fake_plan(_repository, _revision, *, built_at, staging_directory, **_kwargs):
Path(staging_directory).mkdir(mode=0o700)
frozen = {
"schema": prepare.BUILD_SCHEMA,
"repository": importer.REPOSITORY,
"revision": revision,
"build": build,
"sourceDateEpoch": epoch,
"composeSha256": compose_sha,
}
controller._mcp_hub_write_private_new(
Path(staging_directory) / "build.json", prepare._json_bytes(frozen)
)
return {"status": "planned", "revision": revision, "composeSha256": compose_sha}
monkeypatch.setattr(prepare, "plan_candidate", fake_plan)
def fake_capture(_repository, _revision, *, staging_directory, **_kwargs):
image_rows = {
service: {
"imageRef": images[service]["imageRef"],
"imageId": images[service]["imageId"],
"vulnerabilitySha256": hashlib.sha256(
report_data[f"vulnerability-{service}.json"]
).hexdigest(),
"secretSha256": hashlib.sha256(
report_data[f"secret-{service}.json"]
).hexdigest(),
}
for service in controller.MCP_HUB_SERVICES
}
candidate = {
"schema": prepare.CANDIDATE_SCHEMA,
"repository": importer.REPOSITORY,
"revision": revision,
"build": build,
"sourceDateEpoch": epoch,
"composeSha256": compose_sha,
"images": image_rows,
"databaseMetadataSha256": hashlib.sha256(
report_data["trivy-db-metadata.json"]
).hexdigest(),
"summarySha256": hashlib.sha256(prepare._json_bytes(summary)).hexdigest(),
"digestsSha256": hashlib.sha256(prepare._json_bytes(digests)).hexdigest(),
}
outputs = {
"candidate.json": candidate,
"scan-summary.json": summary,
"image-digests.json": digests,
"exceptions.json": draft,
}
for filename, value in outputs.items():
controller._mcp_hub_write_private_new(
Path(staging_directory) / filename, prepare._json_bytes(value)
)
candidate_sha = hashlib.sha256(prepare._json_bytes(candidate)).hexdigest()
exceptions_sha = hashlib.sha256(prepare._json_bytes(draft)).hexdigest()
return {
"candidateSha256": candidate_sha,
"exceptionsSha256": exceptions_sha,
}
monkeypatch.setattr(prepare, "capture_candidate", fake_capture)
def download_artifact(_artifact_id, destination, _maximum):
_private_file(destination, artifact_bytes)
return None
candidate_directory = state_root / "mcp-hub-release-candidates" / revision
if rerun_during_load:
with pytest.raises(importer.CandidateRejected, match="run changed"):
importer.prepare_candidate(
importer.REPOSITORY,
revision,
api_json=lambda *_args, **_kwargs: None,
download_artifact=download_artifact,
)
assert not candidate_directory.exists()
return
first = importer.prepare_candidate(
importer.REPOSITORY,
revision,
api_json=lambda *_args, **_kwargs: None,
download_artifact=download_artifact,
)
assert candidate_directory.is_dir()
assert stat.S_IMODE(candidate_directory.stat().st_mode) == 0o700
candidate_before_review = (candidate_directory / "candidate.json").read_bytes()
receipt_before_review = (candidate_directory / "import-receipt.json").read_bytes()
collision_stage = tmp_path / "second-candidate"
collision_stage.mkdir(mode=0o700)
with pytest.raises(importer.CandidateRejected, match="never overwrites"):
importer._publish_candidate(collision_stage, candidate_directory)
assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review
if finding_present:
assert first["status"] == "pending_review"
reviewed = {
**draft,
"reviewer": "release-operator",
"exceptions": [
{**row, "justification": "Reviewed unfixable finding"}
for row in draft["exceptions"]
],
}
exceptions_path = candidate_directory / "exceptions.json"
exceptions_path.unlink()
controller._mcp_hub_write_private_new(
exceptions_path, prepare._json_bytes(reviewed)
)
approval = {
"schema": importer.APPROVAL_SCHEMA,
"candidateSha256": first["candidateSha256"],
"exceptionsSha256": hashlib.sha256(
prepare._json_bytes(reviewed)
).hexdigest(),
}
controller._mcp_hub_write_private_new(
candidate_directory / "approval.json", prepare._json_bytes(approval)
)
final = importer.prepare_candidate(
importer.REPOSITORY,
revision,
api_json=lambda *_args, **_kwargs: None,
download_artifact=download_artifact,
)
assert final["status"] == "ready"
else:
assert first["status"] == "ready"
final = first
assert final["candidateSha256"] == hashlib.sha256(candidate_before_review).hexdigest()
assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review
assert (candidate_directory / "import-receipt.json").read_bytes() == receipt_before_review
def test_candidate_artifact_is_exact_bounded_and_unexpired(modules):
importer = modules.importer
revision = "c" * 40
run_id = 11
artifact = {
"id": 22,
"name": importer.ARTIFACT_PREFIX + revision,
"size_in_bytes": 4096,
"expired": False,
"expires_at": (datetime.now(timezone.utc) + timedelta(hours=1))
.isoformat()
.replace("+00:00", "Z"),
"workflow_run": {"id": run_id},
}
result = importer._candidate_artifact(
lambda _path, *, query: {"artifacts": [artifact]}, run_id, revision
)
assert result == artifact
for changed in (
{"expired": True},
{"name": "hub-release-candidate-" + "d" * 40},
{"size_in_bytes": importer.MAX_ARTIFACT_ZIP_BYTES + 1},
{"workflow_run": {"id": run_id + 1}},
):
with pytest.raises(importer.CandidateRejected):
importer._candidate_artifact(
lambda _path, *, query: {"artifacts": [{**artifact, **changed}]},
run_id,
revision,
)
def test_streamed_uploader_zip_descriptors_are_verified(modules, tmp_path):
importer = modules.importer
payload = _streamed_zip_bytes(importer)
zip_path = tmp_path / "candidate.zip"
tar_path = tmp_path / "images.tar"
_private_file(zip_path, bytes(payload))
(
manifest,
reports,
zip_sha,
manifest_sha,
tar_sha,
tar_bytes,
) = importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(payload))
assert manifest == {}
assert len(reports) == 35
assert zip_sha == hashlib.sha256(payload).hexdigest()
assert manifest_sha == hashlib.sha256(b"{}").hexdigest()
assert tar_sha == hashlib.sha256(b"tar-data").hexdigest()
assert tar_bytes == len(b"tar-data")
assert tar_path.read_bytes() == b"tar-data"
tampered = bytearray(payload)
name_size = len("candidate-manifest.json")
descriptor_offset = 30 + name_size + 2
tampered[descriptor_offset + 4] ^= 0x01
_private_file(zip_path, bytes(tampered))
with pytest.raises(importer.CandidateRejected, match="descriptor"):
importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(tampered))
def test_zip_central_directory_is_bounded_before_zipfile_parse(modules, tmp_path, monkeypatch):
importer = modules.importer
payload = _streamed_zip_bytes(importer, extra_member=True)
path = tmp_path / "too-many.zip"
_private_file(path, bytes(payload))
monkeypatch.setattr(
importer.zipfile,
"ZipFile",
lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized member inventory"),
)
with pytest.raises(importer.CandidateRejected, match="central directory"):
importer._zip_central_directory_preflight(path, len(payload))
def test_zip_oversized_central_directory_is_rejected_without_allocating_it(
modules, tmp_path, monkeypatch
):
importer = modules.importer
payload = struct.pack(
"<IHHHHIIH",
0x06054B50,
0,
0,
37,
37,
65 * 1024,
0,
0,
)
path = tmp_path / "oversized-directory.zip"
_private_file(path, payload)
monkeypatch.setattr(
importer.zipfile,
"ZipFile",
lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized central directory"),
)
with pytest.raises(importer.CandidateRejected, match="central directory"):
importer._zip_central_directory_preflight(path, len(payload))
def test_tar_member_count_is_stopped_during_streamed_header_walk(
modules, tmp_path, monkeypatch
):
importer = modules.importer
path = tmp_path / "many-members.tar"
with tarfile.open(path, "w", format=tarfile.USTAR_FORMAT) as archive:
for name in ("one/", "two/"):
info = tarfile.TarInfo(name)
info.type = tarfile.DIRTYPE
archive.addfile(info)
monkeypatch.setattr(importer, "MAX_TAR_MEMBERS", 1)
with pytest.raises(importer.CandidateRejected, match="member-count"):
importer._preflight_uncompressed_tar(path)
def test_tar_preflight_rejects_extension_headers_before_tarfile(modules, tmp_path):
importer = modules.importer
path = tmp_path / "pax.tar"
with tarfile.open(path, "w", format=tarfile.PAX_FORMAT) as archive:
info = tarfile.TarInfo("x" * 120)
archive.addfile(info, io.BytesIO(b"x"))
with pytest.raises(importer.CandidateRejected, match="unsupported member type"):
importer._preflight_uncompressed_tar(path)
def test_remote_head_without_completed_candidate_stays_awaiting(modules, tmp_path, monkeypatch):
controller = modules.controller
prepare = modules.prepare
importer = modules.importer
revision = "a" * 40
tree = "b" * 40
monkeypatch.setattr(
prepare, "_fixed_target", lambda _repository, _revision: ({}, tmp_path)
)
responses = {
"/repos/jens/itworx_mcp_hub/branches/main": {"commit": {"id": revision}},
f"/repos/jens/itworx_mcp_hub/git/commits/{revision}": {
"sha": revision,
"commit": {"tree": {"sha": tree}},
},
importer.WORKFLOW_LIST_PATH: {
"total_count": 1,
"workflows": [
{
"id": importer.WORKFLOW_ID,
"path": importer.WORKFLOW_PATH,
"state": "active",
}
],
},
"/repos/jens/itworx_mcp_hub/actions/runs": {
"total_count": 0,
"workflow_runs": [],
},
}
calls = []
def api(path, *, query):
calls.append((path, query))
return responses[path]
result = importer.prepare_candidate(
controller.MCP_HUB_REPOSITORY,
revision,
api_json=api,
download_artifact=lambda *_args: pytest.fail("an absent run cannot download artifacts"),
)
assert result == {"status": "awaiting_candidate", "revision": revision}
assert len(calls) == 4
def test_fixed_compose_environment_discards_ambient_docker_and_compose_context(
modules, monkeypatch
):
controller = modules.controller
importer = modules.importer
monkeypatch.setattr(
controller,
"deployment_environment",
lambda *_args: {
"PATH": "/usr/bin",
"DOCKER_HOST": "tcp://peer:2375",
"DOCKER_CONTEXT": "attacker",
"DOCKER_TLS_VERIFY": "1",
"COMPOSE_FILE": "/tmp/untrusted.yml",
"COMPOSE_PROFILES": "untrusted",
},
)
env = importer._fixed_compose_environment({"compose_project": "fixed"}, "a" * 40, "jens/itworx_mcp_hub")
assert env == {
"PATH": "/usr/bin",
"DOCKER_HOST": importer.DOCKER_HOST,
"COMPOSE_PROJECT_NAME": "fixed",
"COMPOSE_PROFILES": ",".join(controller.MCP_HUB_COMPOSE_PROFILES),
}
backend/tests/test_hub_candidate_transport.py
+236
View File
@@ -0,0 +1,236 @@
from __future__ import annotations
import importlib.util
import io
import json
import os
from pathlib import Path
import sys
import pytest
ROOT = Path(__file__).resolve().parents[2]
PATH = ROOT / "ops/gitea-deploy-broker/hub_candidate_transport.py"
SPEC = importlib.util.spec_from_file_location("hub_candidate_transport_test", PATH)
transport = importlib.util.module_from_spec(SPEC)
assert SPEC.loader is not None
sys.modules[SPEC.name] = transport
SPEC.loader.exec_module(transport)
def _request(url: str, *, mode="json", maximum=None, output_fd=None):
return {
"url": url,
"headers": {
"Accept": "application/json" if mode == "json" else "application/zip",
"Authorization": "token " + "x" * 40,
"User-Agent": transport.USER_AGENT,
},
"maxBytes": maximum if maximum is not None else (
transport.MAX_METADATA_BYTES if mode == "json" else transport.MAX_ARTIFACT_BYTES
),
"outputFd": output_fd,
}
class Response:
status = 200
def __init__(self, chunks):
self.chunks = list(chunks)
self.reads = []
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def read1(self, limit):
self.reads.append(limit)
if not self.chunks:
return b""
chunk = self.chunks.pop(0)
return chunk[:limit]
def read(self, _limit):
pytest.fail("transport must use deadline-aware read1")
class Opener:
def __init__(self, response):
self.response = response
self.requests = []
self.timeouts = []
def open(self, request, timeout):
self.requests.append(request)
self.timeouts.append(timeout)
return self.response
@pytest.mark.parametrize("url", [
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/branches/main",
"https://gitea.example.com:443/api/v1/repos/jens/itworx_mcp_hub/actions/workflows",
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/git/commits/" + "a" * 40,
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs?event=push&branch=main&head_sha=" + "a" * 40 + "&page=1&limit=100",
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs/123/jobs?page=2&limit=100",
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs/123/artifacts?name=hub-release-candidate-" + "a" * 40,
])
def test_fixed_json_routes_are_allowed(url):
assert transport._url_is_allowed(url, "json")
@pytest.mark.parametrize("url", [
"https://other.invalid/api/v1/repos/jens/itworx_mcp_hub/actions/workflows",
"http://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/workflows",
"https://gitea.example.com:444/api/v1/repos/jens/itworx_mcp_hub/actions/workflows",
"https://user@gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/workflows",
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs?event=push&event=push&branch=main&head_sha=" + "a" * 40 + "&page=1&limit=100",
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs?event=pull_request&branch=main&head_sha=" + "a" * 40 + "&page=1&limit=100",
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs/99999999999999999999/jobs?page=1&limit=100",
"https://gitea.example.com/api/v1/repos/jens/other/actions/workflows",
])
def test_nonfixed_json_routes_are_rejected(url):
assert not transport._url_is_allowed(url, "json")
def test_zip_route_is_fixed_and_has_bounded_artifact_id():
assert transport._url_is_allowed(
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/artifacts/12/zip", "zip"
)
assert not transport._url_is_allowed(
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/artifacts/9223372036854775808/zip", "zip"
)
assert not transport._url_is_allowed(
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/artifacts/12/zip?other=x", "zip"
)
def test_ipc_requires_exact_fields_unique_json_and_16kib_bound():
valid = json.dumps(_request("https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/branches/main")).encode()
assert transport._read_request(io.BytesIO(valid))["maxBytes"] == transport.MAX_METADATA_BYTES
for invalid in (
b'{"url":"x","url":"y"}',
valid[:-1] + b',"unexpected":true}',
b'{"a":NaN}',
b"x" * (transport.MAX_INPUT_BYTES + 1),
):
with pytest.raises(Exception):
transport._read_request(io.BytesIO(invalid))
@pytest.mark.parametrize("patch", [
{"headers": {"Accept": "application/json", "Authorization": "token x\r\nInjected: yes", "User-Agent": transport.USER_AGENT}},
{"headers": {"Accept": "application/json", "Authorization": "token " + "x" * 40, "User-Agent": transport.USER_AGENT, "Cookie": "x"}},
{"maxBytes": True},
{"outputFd": 3},
])
def test_json_request_rejects_header_injection_extra_header_and_bad_limits(patch):
value = _request("https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/branches/main")
value.update(patch)
with pytest.raises(transport.TransportError):
transport._validate_request(value, "json")
def test_json_mode_emits_only_raw_bounded_response_bytes():
body = b'{"workflows":[]}'
opener = Opener(Response([body]))
value = _request("https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/workflows")
result = transport.execute(value, "json", opener_factory=lambda mode: opener)
assert result == body
assert opener.timeouts == [10]
headers = {key.lower(): val for key, val in opener.requests[0].header_items()}
assert headers == {
"accept": "application/json",
"authorization": "token " + "x" * 40,
"user-agent": transport.USER_AGENT,
}
def test_zip_mode_streams_only_to_inherited_fd_and_reports_byte_count():
read_fd, write_fd = os.pipe()
try:
response = Response([b"zip-", b"bytes"])
opener = Opener(response)
value = _request(
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/artifacts/12/zip",
mode="zip", output_fd=write_fd,
)
assert transport.execute(value, "zip", opener_factory=lambda mode: opener) == 9
os.close(write_fd)
write_fd = -1
assert os.read(read_fd, 32) == b"zip-bytes"
assert opener.timeouts == [60]
finally:
if write_fd >= 0:
os.close(write_fd)
os.close(read_fd)
def test_redirect_strips_authorization_and_rejects_other_hosts():
request = transport.Request(
"https://gitea.example.com/api/v1/artifact",
headers={"Accept": "application/zip", "Authorization": "token secret", "User-Agent": transport.USER_AGENT},
)
handler = transport._SameHostArtifactRedirect()
redirected = handler.redirect_request(
request, None, 302, "", {}, "https://gitea.example.com/attachments/opaque?signature=opaque"
)
assert redirected is not None
assert {key.lower(): value for key, value in redirected.header_items()} == {
"accept": "application/zip", "user-agent": transport.USER_AGENT,
}
for url in (
"http://gitea.example.com/attachments/file",
"https://other.invalid/attachments/file",
"https://gitea.example.com:444/attachments/file",
"https://user@gitea.example.com/attachments/file",
"https://gitea.example.com/attachments/file#fragment",
):
assert handler.redirect_request(request, None, 302, "", {}, url) is None
def test_deadline_stops_a_continuous_read1_trickle():
clock = [0.0]
class Trickle(Response):
def read1(self, _limit):
clock[0] += 2
return b" "
value = _request("https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/branches/main")
with pytest.raises(transport.TransportError, match="deadline"):
transport.execute(value, "json", opener_factory=lambda _mode: Opener(Trickle([])),
monotonic=lambda: clock[0])
def test_cli_errors_are_generic_and_zip_marker_is_compact():
errors = io.BytesIO()
stdout = io.BytesIO()
assert transport.main(["wrong"], stdin=io.BytesIO(b""), stdout=stdout, stderr=errors) == 1
assert errors.getvalue() == transport._GENERIC_ERROR
assert b"wrong" not in errors.getvalue()
read_fd, write_fd = os.pipe()
try:
value = _request(
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/artifacts/12/zip",
mode="zip", output_fd=write_fd,
)
response = Response([b"zip"])
payload = json.dumps(value, separators=(",", ":")).encode()
stdout = io.BytesIO()
errors = io.BytesIO()
result = transport.main(["zip"], stdin=io.BytesIO(payload), stdout=stdout,
stderr=errors, opener_factory=lambda _mode: Opener(response))
assert result == 0
assert stdout.getvalue() == b'{"bytes":3}\n'
assert errors.getvalue() == b""
os.close(write_fd)
write_fd = -1
assert os.read(read_fd, 32) == b"zip"
finally:
if write_fd >= 0:
os.close(write_fd)
os.close(read_fd)
backend/tests/test_hub_permission_preparation.py
+775
View File
@@ -0,0 +1,775 @@
from __future__ import annotations
import fcntl
import hashlib
import importlib.util
import io
import json
import os
import stat
import sys
import time
from email.message import Message
from pathlib import Path
import pytest
from sqlmodel import Session, SQLModel, create_engine, select
from backend.app.db.models import AuditEvent, JobRun
from backend.app.services import action_execution, deployment_broker
from backend.app.services.action_execution import (
approval_execution_supported,
mcp_execution_supported,
)
from backend.app.services.action_registry import ACTION_RISK, ACTIONS, action_default_target
ROOT = Path(__file__).resolve().parents[2]
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
ACTION = "hub.migration.prepare_permissions"
TARGET = "tower/itworx-mcp-hub-permissions"
STATE_PATH = "/mnt/cache/appdata/gitea-deploy-runners/state/itworx-mcp-hub.json"
LEGACY_BACKUP_PATH = "/srv/example/appdata/itworx-mcp-hub/backups"
@pytest.fixture
def hub():
path = CONTROLLER / "hub_permission_preparation.py"
assert path.is_file(), "the protected controller permission helper is missing"
spec = importlib.util.spec_from_file_location("hub_permission_preparation_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@pytest.fixture
def controller(monkeypatch):
monkeypatch.syspath_prepend(str(CONTROLLER))
spec = importlib.util.spec_from_file_location(
"hub_permission_controller_test", CONTROLLER / "status_server.py"
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def permission_fixture(hub, tmp_path, monkeypatch):
if os.name != "posix" or not hasattr(os, "geteuid") or os.geteuid() != 0:
pytest.skip("permission ownership fixture requires Linux root")
state_dir = tmp_path / "cache" / "appdata" / "gitea-deploy-runners" / "state"
state = state_dir / "itworx-mcp-hub.json"
lock = state_dir.parent / "deploy.lock"
state_dir.mkdir(parents=True)
state_bytes = b'{"fixture":"hub state"}\n'.ljust(4096, b" ")
state.write_bytes(state_bytes)
lock.write_bytes(b"")
os.chown(state, 0, 0)
os.chmod(state, 0o644)
os.chown(lock, 0, 0)
os.chmod(lock, 0o644)
monkeypatch.setattr(hub, "STATE_FILE", str(state))
monkeypatch.setattr(hub, "LOCK_FILE", str(lock))
monkeypatch.setattr(hub, "LOCK_TIMEOUT_SECONDS", 0.25)
monkeypatch.setattr(hub, "LOCK_POLL_INTERVAL_SECONDS", 0.01)
digest = hashlib.sha256(state_bytes).hexdigest()
return state, lock, state_bytes, digest
def state_target(state: Path, *, mode: str = "0600", content: bytes | None = None) -> dict:
payload = content or b'{"fixture":"hub state"}\n'.ljust(4096, b" ")
return {
"name": "state_file",
"path": str(state),
"uid": 0,
"gid": 0,
"mode": mode,
"desired_mode": "0600",
"would_change": mode != "0600",
"acl_present": False,
"size": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
}
def test_permission_action_is_high_risk_and_has_one_fixed_target():
assert ACTION in ACTIONS
assert ACTION_RISK[ACTION] == "high"
assert action_default_target(ACTION) == TARGET
assert approval_execution_supported(ACTION)
assert mcp_execution_supported(ACTION)
def test_broker_posts_only_the_bounded_reason_to_fixed_signed_path(monkeypatch):
calls = []
broker = deployment_broker.DeploymentBrokerClient()
monkeypatch.setattr(broker, "_request", lambda *args, **kwargs: calls.append((args, kwargs)) or {})
broker.hub_migration_prepare_permissions(reason="prepare fixed Hub permissions" * 40)
assert calls[0][0] == ("POST", "/v1/appops/hub-migration/prepare-permissions")
assert calls[0][1]["payload"] == {"reason": ("prepare fixed Hub permissions" * 40)[:1000]}
def test_governed_execution_blocks_wrong_target_before_job_or_broker(tmp_path, monkeypatch):
engine = create_engine(f"sqlite:///{tmp_path / 'blocked.db'}")
SQLModel.metadata.create_all(engine)
monkeypatch.setattr(
deployment_broker.DeploymentBrokerClient,
"hub_migration_prepare_permissions",
lambda *args, **kwargs: pytest.fail("wrong target must not call the controller"),
)
with Session(engine) as session:
result = action_execution.execute_governed_action(
session,
action=ACTION,
target="tower/other",
source="chatgpt-mcp",
risk="high",
reason="prepare fixed Hub permissions",
original_policy_decision="allowed",
)
assert result["policyDecision"] == "blocked"
assert session.exec(select(JobRun)).all() == []
@pytest.mark.parametrize(
"status,success",
[("prepared", True), ("already_compliant", True), ("partial", False), ("blocked", False)],
)
def test_governed_execution_audits_single_state_file_outcome(tmp_path, monkeypatch, status, success):
engine = create_engine(f"sqlite:///{tmp_path / f'{status}.db'}")
SQLModel.metadata.create_all(engine)
result_payload = {
"status": status,
"revision": "a" * 40,
"changedTargets": [STATE_PATH] if status in {"prepared", "partial"} else [],
"changedTargetDetails": [],
"uncertainTargets": [STATE_PATH] if status == "partial" else [],
}
if status == "partial":
result_payload["changedTargetDetails"] = [
{
"name": "state_file",
"path": STATE_PATH,
"observedMode": "0600",
"sameIdentityAtPath": True,
}
]
if status in {"prepared", "already_compliant"}:
result_payload["verifiedTargets"] = [
{
"name": "state_file",
"path": STATE_PATH,
"uid": 0,
"gid": 0,
"mode": "0600",
"aclPresent": False,
}
]
monkeypatch.setattr(
deployment_broker.DeploymentBrokerClient,
"hub_migration_prepare_permissions",
lambda self, **kwargs: result_payload,
)
with Session(engine) as session:
result = action_execution.execute_governed_action(
session,
action=ACTION,
target=TARGET,
source="chatgpt-mcp",
risk="high",
reason="prepare fixed Hub permissions",
original_policy_decision="allowed",
)
job = session.get(JobRun, result["jobId"])
audit = session.get(AuditEvent, result["auditId"])
assert result["success"] is success
assert job.status == ("succeeded" if success else "failed")
assert audit.result == ("success" if success else "failed")
assert audit.target == TARGET
assert audit.after_state["status"] == status
assert audit.after_state["changedTargets"] == result_payload["changedTargets"]
assert audit.after_state["changedTargetDetails"] == result_payload["changedTargetDetails"]
@pytest.mark.parametrize("legacy_status", ["prepared", "already_compliant", "partial"])
def test_governed_execution_rejects_legacy_two_target_reports_as_unknown(
tmp_path, monkeypatch, legacy_status
):
engine = create_engine(f"sqlite:///{tmp_path / f'legacy-{legacy_status}.db'}")
SQLModel.metadata.create_all(engine)
result_payload = {
"status": legacy_status,
"revision": "a" * 40,
"changedTargets": [LEGACY_BACKUP_PATH] if legacy_status != "already_compliant" else [],
"changedTargetDetails": [],
"uncertainTargets": [],
"verifiedTargets": [
{
"name": "backup_directory",
"path": LEGACY_BACKUP_PATH,
"uid": 99,
"gid": 100,
"mode": "0700",
"aclPresent": False,
},
{
"name": "state_file",
"path": STATE_PATH,
"uid": 0,
"gid": 0,
"mode": "0600",
"aclPresent": False,
},
],
}
monkeypatch.setattr(
deployment_broker.DeploymentBrokerClient,
"hub_migration_prepare_permissions",
lambda self, **kwargs: result_payload,
)
with Session(engine) as session:
result = action_execution.execute_governed_action(
session,
action=ACTION,
target=TARGET,
source="chatgpt-mcp",
risk="high",
reason="prepare fixed Hub permissions",
original_policy_decision="allowed",
)
audit = session.get(AuditEvent, result["auditId"])
assert result["success"] is False
assert result["status"] == "outcome_unknown"
assert result["result"] == "outcome_unknown"
assert result["uncertainTargets"] == [STATE_PATH]
assert audit.result == "outcome_unknown"
assert audit.after_state["uncertainTargets"] == [STATE_PATH]
@pytest.mark.parametrize(
"failure", ["broker_exception", "unreachable", "malformed_success", "malformed_partial"]
)
def test_governed_execution_marks_unresolved_controller_results_unknown(
tmp_path, monkeypatch, failure
):
engine = create_engine(f"sqlite:///{tmp_path / f'{failure}.db'}")
SQLModel.metadata.create_all(engine)
calls = []
def broker_call(self, **_kwargs):
calls.append("called")
if failure == "broker_exception":
raise RuntimeError("request ended without a controller result")
if failure == "unreachable":
return {"status": "unreachable"}
if failure == "malformed_partial":
return {
"status": "partial",
"revision": "a" * 40,
"changedTargets": [LEGACY_BACKUP_PATH],
"changedTargetDetails": [],
"uncertainTargets": [],
}
return {
"status": "prepared",
"revision": "a" * 40,
"changedTargets": [STATE_PATH],
"changedTargetDetails": [],
"uncertainTargets": [],
}
monkeypatch.setattr(
deployment_broker.DeploymentBrokerClient,
"hub_migration_prepare_permissions",
broker_call,
)
with Session(engine) as session:
result = action_execution.execute_governed_action(
session,
action=ACTION,
target=TARGET,
source="chatgpt-mcp",
risk="high",
reason="prepare fixed Hub permissions",
original_policy_decision="allowed",
)
job = session.get(JobRun, result["jobId"])
audit = session.get(AuditEvent, result["auditId"])
assert calls == ["called"]
assert result["success"] is False
assert result["status"] == "outcome_unknown"
assert result["result"] == "outcome_unknown"
assert result["uncertainTargets"] == [STATE_PATH]
assert job.status == "failed"
assert "outcome is unknown" in job.result_summary
assert audit.result == "outcome_unknown"
assert audit.after_state["status"] == "outcome_unknown"
assert audit.after_state["changedTargets"] == []
def test_preparation_changes_only_fixed_state_file_and_is_idempotent(hub, tmp_path, monkeypatch):
state, lock, state_bytes, digest = permission_fixture(hub, tmp_path, monkeypatch)
state_identity = (state.stat().st_dev, state.stat().st_ino)
original_open = hub._open_pinned_object
opened = []
changed_objects = []
# Keep a legacy backup path with an intentionally irrelevant mode. The
# helper must neither open nor chmod it.
legacy_backup = tmp_path / "user" / "appdata" / "itworx-mcp-hub" / "backups"
legacy_backup.mkdir(parents=True)
os.chmod(legacy_backup, 0o750)
monkeypatch.setattr(hub, "BACKUP_DIRECTORY", str(legacy_backup), raising=False)
def record_open(name, path):
opened.append((name, path))
return original_open(name, path)
real_set_mode = hub._set_mode
def record_change(fd, mode):
st = os.fstat(fd)
changed_objects.append((st.st_dev, st.st_ino))
real_set_mode(fd, mode)
monkeypatch.setattr(hub, "_open_pinned_object", record_open)
monkeypatch.setattr(hub, "_set_mode", record_change)
code, report = hub.prepare_fixed_permissions()
assert code == 0
assert report["status"] == "applied"
assert report["changed_targets"] == [str(state)]
assert [target["path"] for target in report["targets"]] == [str(state)]
assert stat.S_IMODE(state.stat().st_mode) == 0o600
assert stat.S_IMODE(lock.stat().st_mode) == 0o644
assert stat.S_IMODE(legacy_backup.stat().st_mode) == 0o750
assert state.read_bytes() == state_bytes
assert hashlib.sha256(state.read_bytes()).hexdigest() == digest
assert (state.stat().st_dev, state.stat().st_ino) == state_identity
assert (state.stat().st_uid, state.stat().st_gid) == (0, 0)
assert report["lock"]["acquired"] is True
assert opened == [("deploy_lock", str(lock)), ("state_file", str(state))]
assert changed_objects == [state_identity]
code, report = hub.prepare_fixed_permissions()
assert code == 0
assert report["status"] == "already_compliant"
assert report["changed_targets"] == []
assert [target["path"] for target in report["targets"]] == [str(state)]
assert stat.S_IMODE(legacy_backup.stat().st_mode) == 0o750
def test_dry_run_does_not_change_state(hub, tmp_path, monkeypatch):
state, lock, before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
code, report = hub.execute()
assert code == 0
assert report["status"] == "ready"
assert [item["path"] for item in report["targets"]] == [str(state)]
assert stat.S_IMODE(state.stat().st_mode) == 0o644
assert stat.S_IMODE(lock.stat().st_mode) == 0o644
assert state.read_bytes() == before
def test_preparation_rechecks_fresh_state_digest_before_changing_mode(hub, tmp_path, monkeypatch):
state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
real_apply = hub._apply_changes
def change_before_apply(lock, pinned_state, mode, expected_digest):
state.write_bytes(b"changed after inspection".ljust(4096, b" "))
return real_apply(lock, pinned_state, mode, expected_digest)
monkeypatch.setattr(hub, "_apply_changes", change_before_apply)
code, report = hub.prepare_fixed_permissions()
assert code == 2
assert report["status"] == "failed"
assert "digest mismatch" in report["error"]
assert stat.S_IMODE(state.stat().st_mode) == 0o644
def test_symlinks_wrong_owner_mode_hardlinks_and_acl_fail_closed(hub, tmp_path, monkeypatch):
state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
real_state = state.with_name("state-real.json")
state.rename(real_state)
state.symlink_to(real_state)
code, report = hub.execute()
assert code == 2 and report["status"] == "blocked"
state.unlink()
real_state.rename(state)
os.chown(state, 99, 100)
code, report = hub.execute()
assert code == 2 and "owner mismatch" in report["error"]
os.chown(state, 0, 0)
os.link(state, state.with_name("unexpected-hardlink"))
code, report = hub.execute()
assert code == 2 and report["status"] == "blocked"
assert "link-count mismatch" in report["error"]
state.with_name("unexpected-hardlink").unlink()
os.chmod(state, 0o640)
code, report = hub.execute()
assert code == 2 and "mode mismatch" in report["error"]
os.chmod(state, 0o644)
state_identity = (state.stat().st_dev, state.stat().st_ino)
actual_has_acl = hub._has_acl
monkeypatch.setattr(
hub,
"_has_acl",
lambda fd: (os.fstat(fd).st_dev, os.fstat(fd).st_ino) == state_identity
or actual_has_acl(fd),
)
code, report = hub.execute()
assert code == 2 and "POSIX ACL is present" in report["error"]
assert stat.S_IMODE(state.stat().st_mode) == 0o644
def test_unsafe_deploy_lock_metadata_blocks_without_state_mutation(hub, tmp_path, monkeypatch):
state, lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
os.chmod(lock, 0o600)
code, report = hub.prepare_fixed_permissions()
assert code == 2 and report["status"] == "blocked"
assert "mode mismatch" in report["error"]
assert stat.S_IMODE(state.stat().st_mode) == 0o644
assert stat.S_IMODE(lock.stat().st_mode) == 0o600
def test_replaced_state_path_reports_uncertain_outcome(hub, tmp_path, monkeypatch):
state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
moved_state = state.with_name("state-pinned.json")
real_set_mode = hub._set_mode
replaced = False
def replace_after_mode_change(fd, mode):
nonlocal replaced
real_set_mode(fd, mode)
if not replaced:
replaced = True
state.rename(moved_state)
state.write_bytes(b"replacement")
os.chown(state, 0, 0)
os.chmod(state, 0o644)
monkeypatch.setattr(hub, "_set_mode", replace_after_mode_change)
code, report = hub.prepare_fixed_permissions()
assert code == 2
assert report["status"] == "partial"
assert report["changed_targets"] == [
{
"name": "state_file",
"path": str(state),
"observed_mode": "0600",
"same_identity_at_path": False,
}
]
assert report["uncertain_targets"] == [str(state)]
assert stat.S_IMODE(moved_state.stat().st_mode) == 0o600
assert stat.S_IMODE(state.stat().st_mode) == 0o644
def test_fsync_failure_reports_changed_and_uncertain_state(hub, tmp_path, monkeypatch):
state, _lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
monkeypatch.setattr(hub, "_fsync", lambda _fd: (_ for _ in ()).throw(OSError("fsync failed")))
code, report = hub.prepare_fixed_permissions()
assert code == 2
assert report["status"] == "partial"
assert report["changed_targets"] == [
{
"name": "state_file",
"path": str(state),
"observed_mode": "0600",
"same_identity_at_path": True,
}
]
assert report["uncertain_targets"] == [str(state)]
def test_bounded_deploy_lock_contention_fails_without_mutation(hub, tmp_path, monkeypatch):
state, lock, _before, _digest = permission_fixture(hub, tmp_path, monkeypatch)
fd = os.open(lock, os.O_RDONLY)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
started = time.monotonic()
code, report = hub.prepare_fixed_permissions()
elapsed = time.monotonic() - started
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
assert code == 2
assert report["status"] == "blocked"
assert "bounded wait" in report["error"]
assert elapsed < 1.0
assert stat.S_IMODE(state.stat().st_mode) == 0o644
def controller_handler(controller, monkeypatch, payload=None):
instance = object.__new__(controller.Handler)
body = json.dumps(payload or {"reason": "Prepare fixed Hub permissions"}).encode()
instance.headers = Message()
instance.headers["Content-Length"] = str(len(body))
instance.headers["Content-Type"] = "application/json"
instance.rfile = io.BytesIO(body)
instance.path = controller.HUB_PERMISSION_PREPARATION_PATH
responses = []
signatures = []
monkeypatch.setattr(instance, "_trusted_client", lambda: True)
monkeypatch.setattr(instance, "_send_json", lambda code, data: responses.append((code, json.loads(data))))
monkeypatch.setattr(
controller,
"_self_upgrade_signature_is_valid",
lambda *args, **kwargs: signatures.append(kwargs) or True,
)
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {"revision": "a" * 40})
monkeypatch.setattr(
controller,
"_require_controller_release",
lambda: {
"sourceRevision": "a" * 40,
"sourceRepository": controller.SELF_UPGRADE_REPOSITORY,
},
)
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda revision: {"status": "already_deployed"})
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}})
monkeypatch.setattr(controller, "_repository_path_for_delivery", lambda *args: Path("fixed-repo.git"))
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: True)
return instance, responses, signatures
def controller_success_report(state_path: str, *, already: bool = False) -> dict:
return {
"status": "already_compliant" if already else "applied",
"changed_targets": [] if already else [state_path],
"uncertain_targets": [],
"targets": [state_target(Path(state_path))],
}
def test_controller_route_uses_only_signed_protected_validated_current_main(
controller, monkeypatch
):
instance, responses, signatures = controller_handler(controller, monkeypatch)
calls = []
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: calls.append("prepare")
or (0, controller_success_report(STATE_PATH)),
)
instance.do_POST()
assert signatures == [{"method": "POST", "path": controller.HUB_PERMISSION_PREPARATION_PATH}]
assert calls == ["prepare"]
assert responses == [
(
200,
{
"status": "prepared",
"revision": "a" * 40,
"changedTargets": [STATE_PATH],
"changedTargetDetails": [],
"uncertainTargets": [],
"verifiedTargets": [
{
"name": "state_file",
"path": STATE_PATH,
"uid": 0,
"gid": 0,
"mode": "0600",
"aclPresent": False,
}
],
},
)
]
def test_controller_route_rejects_legacy_two_target_success_report(controller, monkeypatch):
instance, responses, _signatures = controller_handler(controller, monkeypatch)
legacy = controller_success_report(STATE_PATH)
legacy["changed_targets"] = [LEGACY_BACKUP_PATH, STATE_PATH]
legacy["targets"].insert(
0,
{
"name": "backup_directory",
"path": LEGACY_BACKUP_PATH,
"uid": 99,
"gid": 100,
"mode": "0700",
"desired_mode": "0700",
"would_change": False,
"acl_present": False,
},
)
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: (0, legacy),
)
instance.do_POST()
assert responses == [
(
200,
{
"status": "outcome_unknown",
"revision": "a" * 40,
"changedTargets": [],
"changedTargetDetails": [],
"uncertainTargets": [STATE_PATH],
},
)
]
def test_controller_route_rejects_success_without_full_helper_projection(controller, monkeypatch):
instance, responses, _signatures = controller_handler(controller, monkeypatch)
malformed = controller_success_report(STATE_PATH, already=True)
malformed.pop("changed_targets")
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: (0, malformed),
)
instance.do_POST()
assert responses == [
(
200,
{
"status": "outcome_unknown",
"revision": "a" * 40,
"changedTargets": [],
"changedTargetDetails": [],
"uncertainTargets": [STATE_PATH],
},
)
]
@pytest.mark.parametrize(
"payload",
[
{"reason": "Prepare fixed Hub permissions", "path": "/"},
{"reason": "Prepare fixed Hub permissions", "mode": "0777"},
{"reason": "Prepare fixed Hub permissions", "command": "chmod -R 700 /"},
],
)
def test_controller_route_rejects_caller_selected_effects(controller, monkeypatch, payload):
instance, responses, _signatures = controller_handler(controller, monkeypatch, payload)
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: pytest.fail("invalid request must not inspect or mutate permissions"),
)
instance.do_POST()
assert responses == [(400, {"status": "invalid_request"})]
def test_controller_route_does_not_mutate_before_full_push_validation(controller, monkeypatch):
instance, responses, _signatures = controller_handler(controller, monkeypatch)
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: {"status": "preflight_rejected"})
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: pytest.fail("pending validation must block permission changes"),
)
instance.do_POST()
assert responses == [(409, {"status": "preflight_rejected", "revision": "a" * 40})]
@pytest.mark.parametrize("failure", ["helper_exception", "malformed_report", "malformed_partial"])
def test_controller_route_reports_unknown_state_after_unresolved_helper_call(
controller, monkeypatch, failure
):
instance, responses, _signatures = controller_handler(controller, monkeypatch)
if failure == "helper_exception":
def helper_failure():
raise RuntimeError("helper returned no result")
monkeypatch.setattr(
controller.hub_permission_preparation, "prepare_fixed_permissions", helper_failure
)
elif failure == "malformed_report":
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: (0, {"status": "applied", "changed_targets": "unexpected"}),
)
else:
monkeypatch.setattr(
controller.hub_permission_preparation,
"prepare_fixed_permissions",
lambda: (
2,
{
"status": "partial",
"changed_targets": [LEGACY_BACKUP_PATH],
"uncertain_targets": [],
},
),
)
instance.do_POST()
assert responses == [
(
200,
{
"status": "outcome_unknown",
"revision": "a" * 40,
"changedTargets": [],
"changedTargetDetails": [],
"uncertainTargets": [STATE_PATH],
},
)
]
def test_controller_artifact_manifests_include_new_fixed_helper(controller):
renderer_spec = importlib.util.spec_from_file_location(
"hub_permission_artifact_renderer", ROOT / "scripts/prepare_controller_artifact.py"
)
assert renderer_spec is not None and renderer_spec.loader is not None
renderer = importlib.util.module_from_spec(renderer_spec)
renderer_spec.loader.exec_module(renderer)
volume_spec = importlib.util.spec_from_file_location(
"hub_permission_volume_preparer", ROOT / "scripts/prepare_controller_hub_release_volume.py"
)
assert volume_spec is not None and volume_spec.loader is not None
volume = importlib.util.module_from_spec(volume_spec)
volume_spec.loader.exec_module(volume)
installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8")
assert "hub_permission_preparation.py" in controller._CONTROLLER_ARTIFACT_FILES
assert "hub_permission_preparation.py" in renderer._ARTIFACT_FILES
assert tuple(volume.RELEASE_FILES) == tuple(renderer._ARTIFACT_FILES)
assert "ops/gitea-deploy-broker/hub_permission_preparation.py" in installer
assert "hub_permission_preparation.py" in installer
backend/tests/test_mcp_hub_candidate_preparation.py
+147
View File
@@ -272,6 +272,122 @@ def test_plan_freezes_protected_metadata_without_candidate_or_approval(harness):
assert h.state.mutations == []
def test_plan_candidate_freezes_the_selected_workflow_creation_timestamp(
harness, monkeypatch
):
h = harness
revision = h.revision
built_at = "2026-10-03T12:34:56.789Z"
observed = []
metadata = {
"id": revision[:12],
"repository": h.controller.MCP_HUB_REPOSITORY,
"revision": revision,
"tree": "d" * 40,
"treeDigest": hashlib.sha256(("git-tree:" + "d" * 40).encode()).hexdigest(),
"builtAt": built_at,
}
monkeypatch.setattr(h.controller, "output", lambda _args: revision)
monkeypatch.setattr(h.controller, "validate_commit", lambda _repo, _branch, commit: commit)
def deployment_metadata(_repo, supplied_revision, repository, *, built_at=None):
observed.append(built_at)
assert supplied_revision == revision
assert repository == h.controller.MCP_HUB_REPOSITORY
return dict(metadata)
monkeypatch.setattr(h.controller, "deployment_metadata", deployment_metadata)
monkeypatch.setattr(h.controller, "mcp_hub_stage_release_source", lambda *_args: h.state.source)
monkeypatch.setattr(h.controller, "exact_source_matches_revision", lambda *_args: True)
compose_calls = []
def fixed_compose_modeler(project, revision, repository):
compose_calls.append((project, revision, repository))
return {
"services": {
service: {"image": f"fixture/{service}:candidate"}
for service in h.controller.MCP_HUB_SERVICES
}
}
monkeypatch.setattr(
h.controller,
"mcp_hub_compose_model",
lambda *_args: pytest.fail("plan used the ambient Compose renderer"),
)
staging = (
h.controller.STATE_ROOT
/ "mcp-hub-release-imports"
/ revision
/ ("1" * 32)
/ revision
)
h.utility.plan_candidate(
h.controller.MCP_HUB_REPOSITORY,
revision,
built_at=built_at,
staging_directory=staging,
compose_modeler=fixed_compose_modeler,
)
plan = json.loads(
(staging / "build.json").read_text()
)
assert observed == [built_at]
assert plan["build"]["builtAt"] == built_at
assert len(compose_calls) == 1
def test_candidate_preparation_rejects_staging_path_outside_private_import_anchor(
harness,
):
h = harness
revision = h.revision
staging = h.controller.STATE_ROOT / "outside-imports" / revision
with pytest.raises(h.controller.DeployError, match="private anchor|staging path"):
h.utility.plan_candidate(
h.controller.MCP_HUB_REPOSITORY,
revision,
staging_directory=staging,
)
assert not staging.exists()
def test_candidate_preparation_rejects_symlinked_staging_destination(harness):
h = harness
revision = h.revision
anchor = h.controller.STATE_ROOT / "mcp-hub-release-imports" / revision
token = anchor / ("2" * 32)
token.mkdir(parents=True, mode=0o700)
target = h.controller.STATE_ROOT / "untrusted-target"
target.mkdir(mode=0o700)
staging = token / revision
staging.symlink_to(target, target_is_directory=True)
with pytest.raises(h.controller.DeployError):
h.utility.plan_candidate(
h.controller.MCP_HUB_REPOSITORY,
revision,
staging_directory=staging,
)
def test_candidate_preparation_rejects_nonprivate_staging_parent(harness):
h = harness
revision = h.revision
anchor = h.controller.STATE_ROOT / "mcp-hub-release-imports" / revision
token = anchor / ("3" * 32)
token.mkdir(parents=True, mode=0o700)
token.chmod(0o755)
staging = token / revision
with pytest.raises(h.controller.DeployError, match="private|safe|canonical"):
h.utility.plan_candidate(
h.controller.MCP_HUB_REPOSITORY,
revision,
staging_directory=staging,
)
def test_wrong_repository_and_nonroot_fail_before_controller_access(
modules, monkeypatch
):
@@ -354,6 +470,37 @@ def test_capture_emits_candidate_and_pending_only_envelope_without_apply_calls(
}
def test_capture_can_use_the_importers_fixed_image_inspector(harness, monkeypatch):
h = harness
inspected = []
def fixed_inspector(reference):
inspected.append(reference)
return h.state.images.get(reference)
monkeypatch.setattr(
h.controller,
"inspect_image",
lambda _reference: pytest.fail("capture used the ambient Docker context"),
)
monkeypatch.setattr(
h.controller,
"mcp_hub_compose_model",
lambda *_args: pytest.fail("capture used the ambient Compose renderer"),
)
result = h.utility.capture_candidate(
h.controller.MCP_HUB_REPOSITORY,
h.revision,
image_inspector=fixed_inspector,
compose_modeler=lambda *_args: h.model,
)
assert result["status"] == "pending_review"
assert len(inspected) == 17
assert set(inspected) == {
row["image"] for row in h.model["services"].values()
}
@pytest.mark.parametrize("mutation", ["label", "root"])
def test_capture_rejects_contextforge_provenance_drift_without_candidate(
harness, mutation
backend/tests/test_mcp_hub_governed_topology.py
+156 -6
View File
@@ -48,6 +48,138 @@ def _legacy_target(controller):
"source_dir": (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix(),
}
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="backup ownership contract requires root-owned POSIX fixtures",
)
def test_hub_backup_child_is_created_with_controller_owner_and_private_mode(
controller, tmp_path, monkeypatch
):
state_root = tmp_path / "controller-state"
backup_root = state_root / "mcp-hub-backups"
state_root.mkdir(mode=0o700)
backup_root.mkdir(mode=0o700)
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root)
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
attempt = backup_root / "attempt"
controller._mcp_hub_private_directory(attempt, create=True, root=backup_root)
metadata = attempt.stat()
assert (metadata.st_uid, metadata.st_gid) == (0, 0)
assert stat.S_IMODE(metadata.st_mode) == 0o700
controller._mcp_hub_private_directory(attempt, root=backup_root)
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="backup root contract requires fixed-owner POSIX fixtures",
)
def test_hub_backup_root_rejects_group_readable_root(controller, tmp_path, monkeypatch):
state_root = tmp_path / "controller-state"
backup_root = state_root / "mcp-hub-backups"
state_root.mkdir(mode=0o700)
backup_root.mkdir(mode=0o750)
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root)
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
with pytest.raises(controller.DeployError, match="not private and canonical"):
controller._mcp_hub_private_directory(backup_root, root=backup_root)
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="fixed Hub backup owner contract requires root-owned POSIX fixtures",
)
def test_hub_backup_root_rejects_runtime_owner_even_when_it_matches_parent(
controller, tmp_path, monkeypatch
):
state_root = tmp_path / "controller-state"
backup_root = state_root / "mcp-hub-backups"
state_root.mkdir(mode=0o700)
backup_root.mkdir(mode=0o700)
os.chown(state_root, 99, 100)
os.chown(backup_root, 99, 100)
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root)
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
with pytest.raises(controller.DeployError, match="owner differs"):
controller._mcp_hub_private_directory(backup_root, root=backup_root)
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="descriptor-relative private storage requires root-owned POSIX fixtures",
)
def test_hub_backup_anchor_is_created_under_the_pinned_controller_state(
controller, tmp_path, monkeypatch
):
state_root = tmp_path / "controller-state"
state_root.mkdir(mode=0o755)
backup_root = state_root / "mcp-hub-backups"
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root)
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
controller._mcp_hub_private_directory(backup_root, create=True)
metadata = backup_root.stat()
assert (metadata.st_uid, metadata.st_gid) == (0, 0)
assert stat.S_IMODE(metadata.st_mode) == 0o700
assert stat.S_IMODE(state_root.stat().st_mode) == 0o755
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="descriptor-relative path replacement requires root-owned POSIX fixtures",
)
def test_hub_backup_child_rejects_parent_replacement_during_creation(
controller, tmp_path, monkeypatch
):
state_root = tmp_path / "controller-state"
state_root.mkdir(mode=0o700)
backup_root = state_root / "mcp-hub-backups"
backup_root.mkdir(mode=0o700)
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root)
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
real_mkdir = controller.os.mkdir
def replace_parent(path, mode=0o777, *, dir_fd=None):
real_mkdir(path, mode, dir_fd=dir_fd)
if path == "attempt" and dir_fd is not None:
backup_root.rename(state_root / "replaced-backups")
real_mkdir(backup_root, 0o700)
monkeypatch.setattr(controller.os, "mkdir", replace_parent)
with pytest.raises(controller.DeployError, match="could not be created safely"):
controller._mcp_hub_private_directory(backup_root / "attempt", create=True)
assert not (backup_root / "attempt").exists()
@pytest.mark.skipif(os.name == "nt", reason="release receipt storage requires POSIX mode and ACL metadata")
def test_hub_backup_receipt_directory_is_guarded_by_its_release_volume_parent(
controller, tmp_path, monkeypatch
):
volume_root = tmp_path / "controller-hub-releases"
state_root = volume_root / "state"
receipt_root = state_root / "backup-receipts"
receipt_root.mkdir(mode=0o700, parents=True)
volume_root.chmod(0o700)
state_root.chmod(0o700)
monkeypatch.setattr(controller, "MCP_HUB_RELEASE_STORE_STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_RECEIPT_ROOT", receipt_root)
controller._mcp_hub_private_directory(receipt_root, root=state_root)
with pytest.raises(controller.DeployError, match="escapes its fixed parent"):
controller._mcp_hub_private_directory(receipt_root, root=controller.STATE_ROOT)
def _valid_model(controller):
services = {}
for service_name in controller.MCP_HUB_SERVICES:
@@ -159,14 +291,11 @@ def test_installed_legacy_registry_shape_is_adapted_only_in_memory(controller):
"docker-compose.prod.yml",
"docker-compose.autodeploy.yml",
],
"exact_source": None,
"verify_oci_revision": None,
"manual": False,
"deploy_script": "",
"archive_deploy_script": "",
"live_probe": None,
"managed_plan_version": "",
"clean_untracked_paths": None,
"env_source": "/srv/example/appdata/itworx-mcp-hub/deploy.env",
}
original = copy.deepcopy(target)
@@ -182,8 +311,15 @@ def test_installed_legacy_registry_shape_is_adapted_only_in_memory(controller):
for field, value in (
("source_dir", "/srv/example/appdata/other/repository"),
("compose_files", ["docker-compose.yml", "unexpected.yml"]),
("exact_source", None),
("exact_source", False),
("exact_source", True),
("verify_oci_revision", None),
("verify_oci_revision", False),
("verify_oci_revision", True),
("clean_untracked_paths", None),
("clean_untracked_paths", False),
("clean_untracked_paths", []),
("env_source", "/tmp/untrusted.env"),
):
changed = {**original, field: value}
@@ -921,14 +1057,28 @@ def test_database_size_query_is_limited_to_two_fixed_database_names(controller,
assert "pg_database_size" in query
assert captured["timeout"] == 30
@pytest.mark.skipif(
os.name != "nt" and (not hasattr(os, "geteuid") or os.geteuid() != 0),
reason="backup cleanup ownership fixture requires root on POSIX",
)
def test_failed_backup_attempt_removes_only_its_unique_private_directory(
controller, monkeypatch, tmp_path
):
backup_root = tmp_path / "backups"
appdata_root = tmp_path / "hub-appdata"
backup_root = appdata_root / "backups"
appdata_root.mkdir(mode=0o700)
backup_root.mkdir(mode=0o700)
monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root)
monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", tmp_path)
monkeypatch.setattr(controller, "_mcp_hub_private_directory", lambda *_args, **_kwargs: None)
monkeypatch.setattr(controller, "STATE_ROOT", appdata_root)
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata_root)
private_directory = controller._mcp_hub_private_directory
monkeypatch.setattr(
controller,
"_mcp_hub_private_directory",
lambda path, **kwargs: private_directory(path, **{"root": backup_root, **kwargs}),
)
monkeypatch.setattr(controller, "mcp_hub_validate_secret_files", lambda _model: 21)
monkeypatch.setattr(
controller,
backend/tests/test_mcp_hub_legacy_registry_compatibility.py
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
import pytest
from backend.tests.test_mcp_hub_governed_topology import _legacy_target, _load_controller
LEGACY_SOURCE = "/srv/example/appdata/itworx-mcp-hub/deployment/repository"
def observed_legacy_target(controller):
project = _legacy_target(controller)
project.pop("exact_source")
project.pop("verify_oci_revision")
project["source_dir"] = LEGACY_SOURCE
project["compose_dir"] = "infrastructure/compose"
project["compose_files"] = list(controller.MCP_HUB_COMPOSE_FILES[:3])
project["env_source"] = "/srv/example/appdata/itworx-mcp-hub/deploy.env"
return project
@pytest.mark.parametrize("legacy", [False, True])
def test_observed_hub_legacy_source_is_only_normalized_in_memory(legacy):
controller = _load_controller()
project = observed_legacy_target(controller) if legacy else _legacy_target(controller)
canonical = (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix()
original = LEGACY_SOURCE if legacy else canonical
project["source_dir"] = original
effective = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project)
assert effective["source_dir"] == canonical
assert project["source_dir"] == original
assert effective["containers"] == list(controller.MCP_HUB_CONTAINERS)
assert effective["compose_files"] == list(controller.MCP_HUB_COMPOSE_FILES)
assert effective["_mcp_hub_contract"] == controller.MCP_HUB_TOPOLOGY_CONTRACT
def test_observed_legacy_registry_omissions_are_upgraded_only_in_memory():
controller = _load_controller()
project = observed_legacy_target(controller)
effective = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project)
assert effective["exact_source"] is True
assert effective["verify_oci_revision"] is True
assert "exact_source" not in project
assert "verify_oci_revision" not in project
@pytest.mark.parametrize(
"field,value",
[
("source_dir", "/mnt/cache/appdata/gitea-deploy-runners/sources/itworx-mcp-hub"),
("compose_dir", "."),
("compose_files", ["other.yml"]),
("env_source", "/tmp/deploy.env"),
("exact_source", None),
("exact_source", False),
("exact_source", True),
("verify_oci_revision", None),
("verify_oci_revision", False),
("verify_oci_revision", True),
("clean_untracked_paths", None),
("clean_untracked_paths", False),
("clean_untracked_paths", []),
("clean_untracked_paths", ["local.env"]),
],
)
def test_legacy_registry_omissions_do_not_widen_other_target_shapes(field, value):
controller = _load_controller()
project = observed_legacy_target(controller)
project[field] = value
with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"):
controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project)
@pytest.mark.parametrize("field", ["exact_source", "verify_oci_revision"])
def test_fully_declared_legacy_target_rejects_explicit_false_verification(field):
controller = _load_controller()
project = observed_legacy_target(controller)
project[field] = False
with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"):
controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project)
@pytest.mark.parametrize(
"source",
[
None,
True,
{},
LEGACY_SOURCE + "/",
LEGACY_SOURCE + "/../repository",
"/mnt/cache/appdata/itworx-mcp-hub/deployment/repository",
"/srv/example/appdata/another-hub/deployment/repository",
"/tmp/repository",
],
)
def test_hub_legacy_source_compatibility_rejects_other_spellings_and_types(source):
controller = _load_controller()
project = _legacy_target(controller)
project["source_dir"] = source
with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"):
controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, project)
backend/tests/test_mcp_hub_release_store.py
+86
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import copy
import hashlib
import importlib.util
import json
import os
@@ -106,6 +107,91 @@ def test_fixed_hub_state_reader_is_bounded_and_rejects_symlink_or_missing_state(
assert attacker_state.read_bytes() == b'{"status":"attacker"}'
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="Hub state reader verifies root-owned fixed storage",
)
@pytest.mark.parametrize("state_file_mode", [0o640, 0o644, 0o660, 0o664, 0o666])
def test_fixed_hub_state_reader_rejects_other_file_modes(
store, tmp_path, monkeypatch, state_file_mode
):
controller_root = tmp_path / "controller"
state_root = controller_root / "state"
state_root.mkdir(parents=True, mode=0o700)
controller_root.chmod(0o700)
state_file = state_root / store.HUB_STATE_FILENAME
state_file.write_bytes(b'{"status":"success"}')
state_file.chmod(state_file_mode)
monkeypatch.setattr(store, "CONTROLLER_STATE_PARENT", controller_root)
monkeypatch.setattr(store, "CONTROLLER_STATE_DIRECTORY", state_root)
with pytest.raises(store.ReleaseStoreError, match="unavailable or untrusted"):
store.read_hub_state_bytes(missing_ok=False)
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="external backup owner checks require root-owned POSIX fixtures",
)
def test_descriptor_helpers_verify_an_exact_external_owner(store, tmp_path):
expected_owner = (99, 100)
backup_root = tmp_path / "backups"
backup_root.mkdir(mode=0o700)
backup_root.chmod(0o700)
os.chown(backup_root, *expected_owner)
root_fd = os.open(backup_root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
attempt_fd = None
try:
attempt_fd = store.open_child_directory(
root_fd,
"attempt",
create=True,
mode=0o700,
expected_owner=expected_owner,
)
artifact_fd = os.open(
"manifest.json",
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
0o600,
dir_fd=attempt_fd,
)
try:
os.write(artifact_fd, b"private manifest")
os.fchmod(artifact_fd, 0o600)
os.fchown(artifact_fd, *expected_owner)
finally:
os.close(artifact_fd)
assert store.read_at(
attempt_fd,
"manifest.json",
max_bytes=64,
expected_mode=0o600,
expected_owner=expected_owner,
) == b"private manifest"
digest, size = store.sha256_at(
attempt_fd,
"manifest.json",
max_bytes=64,
expected_mode=0o600,
expected_owner=expected_owner,
)
assert digest == hashlib.sha256(b"private manifest").hexdigest()
assert size == len(b"private manifest")
with pytest.raises(store.ReleaseStoreError, match="metadata is unsafe"):
store.read_at(
attempt_fd,
"manifest.json",
max_bytes=64,
expected_mode=0o600,
expected_owner=(0, 0),
)
finally:
if attempt_fd is not None:
os.close(attempt_fd)
os.close(root_fd)
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="Hub state reader verifies root-owned fixed storage",
backend/tests/test_media_codec_dashboard_deployment_reconciliation.py
+2 -3
View File
@@ -69,9 +69,8 @@ def test_reconciliation_derives_the_verified_live_revision_from_runtime_evidence
"Media-Codec-Dashboard",
]
script = command[2]
# The controller registry must be the canonical physical cache bind, not
# the user-share alias that can resolve to a different inode.
assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script
# The mutable registry must identify the protected volume's fixed config.
assert "/controller-hub-releases/controller-config/projects.json" in script
assert "/srv/example/appdata/gitea-deploy-runners/controller/projects.json" not in script
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
backend/tests/test_portainer_restart_timeout.py
+103 -3
View File
@@ -1,8 +1,47 @@
from types import SimpleNamespace
import pytest
from backend.app.connectors.portainer import PortainerConnector
from backend.app.core.config import settings
CONTROLLER_OWNED_CI_NAMES = (
"itworx-ci-checks",
"itworx-ci-backend",
"itworx-ci-frontend",
"itworx-ci-security",
"itworx-ci-browser",
"itworx-ci-build",
"itworx-security-runner",
"itworx-validation-runner-2",
"itworx-validation-runner-3",
"itworx-docker-runner",
"itworx-browser-runner",
"itworx-validation-runner-burst-1",
"gitea-runner-dind",
)
def _connector_with_containers(monkeypatch, containers):
monkeypatch.setitem(settings.apis, "PORTAINER_URL", "https://portainer.example.test")
monkeypatch.setitem(settings.apis, "PORTAINER_API_KEY", "test-key")
monkeypatch.setitem(settings.apis, "PORTAINER_ENDPOINT_ID", "9")
connector = PortainerConnector()
monkeypatch.setattr(
connector,
"list_containers",
lambda: {"status": "ok", "containers": containers},
)
calls = []
def request(method: str, path: str, **kwargs):
calls.append((method, path, kwargs))
return SimpleNamespace(status_code=204)
monkeypatch.setattr(connector, "_request", request)
return connector, calls
def test_restart_request_allows_docker_grace_period_without_shortening_container_timeout(monkeypatch):
previous = dict(settings.apis)
@@ -17,7 +56,7 @@ def test_restart_request_allows_docker_grace_period_without_shortening_container
"list_containers",
lambda: {
"status": "ok",
"containers": [{"id": "runner-id", "name": "itworx-validation-runner-2", "appId": "itworx-validation-runner-2"}],
"containers": [{"id": "user-container-id", "name": "jellyfin", "appId": "jellyfin"}],
},
)
calls: list[tuple[str, str, dict]] = []
@@ -28,7 +67,7 @@ def test_restart_request_allows_docker_grace_period_without_shortening_container
monkeypatch.setattr(connector, "_request", request)
try:
result = connector.restart_container("itworx-validation-runner-2")
result = connector.restart_container("jellyfin")
finally:
settings.apis.clear()
settings.apis.update(previous)
@@ -36,6 +75,67 @@ def test_restart_request_allows_docker_grace_period_without_shortening_container
assert result["status"] == "ok"
assert calls == [(
"POST",
"/api/endpoints/9/docker/containers/runner-id/restart",
"/api/endpoints/9/docker/containers/user-container-id/restart",
{"timeout": 30.0},
)]
@pytest.mark.parametrize("container_name", CONTROLLER_OWNED_CI_NAMES)
def test_restart_blocks_fixed_controller_owned_ci_containers_by_name(monkeypatch, container_name):
connector, calls = _connector_with_containers(
monkeypatch,
[{"id": "0123456789abcdef0123456789abcdef", "name": container_name, "appId": "ci-stack"}],
)
result = connector.restart_container(container_name)
assert result == {"status": "blocked", "reason": "controller_owned_ci_lifecycle"}
assert calls == []
@pytest.mark.parametrize(
"target",
(
"itworx-ci-backend",
"0123456789abcdef0123456789abcdef",
"01234567",
"backend-ci-stack",
),
)
def test_restart_blocks_controller_owned_container_selected_by_alias(monkeypatch, target):
connector, calls = _connector_with_containers(
monkeypatch,
[{
"id": "0123456789abcdef0123456789abcdef",
"name": "itworx-ci-backend",
"appId": "backend-ci-stack",
}],
)
result = connector.restart_container(target)
assert result == {"status": "blocked", "reason": "controller_owned_ci_lifecycle"}
assert calls == []
@pytest.mark.parametrize("container_name", ("itworx-unraid-deploy", "jellyfin"))
def test_restart_still_allows_unrelated_deploy_and_user_containers(monkeypatch, container_name):
container_id = f"id-{container_name}"
connector, calls = _connector_with_containers(
monkeypatch,
[{"id": container_id, "name": container_name, "appId": container_name}],
)
result = connector.restart_container(container_name)
assert result == {
"status": "ok",
"container": container_id,
"target": container_name,
"action": "container.restart",
}
assert calls == [(
"POST",
f"/api/endpoints/9/docker/containers/{container_id}/restart",
{"timeout": 30.0},
)]
backend/tests/test_portfolio_deployment_enrollment.py
+2 -2
View File
@@ -55,9 +55,9 @@ def test_portfolio_enrollment_contract_is_fixed_and_paired():
def test_portfolio_enrollment_registry_alias_matches_controller_cache_bind_source():
assert 'registry_path = Path("/opt/gitea-deploy/projects.json")' in _ENROLL_SCRIPT
assert 'registry_path = Path("/controller-config/projects.json")' in _ENROLL_SCRIPT
assert (
'host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")'
'host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")'
in _ENROLL_SCRIPT
)
assert "os.path.samefile(registry_path, host_registry_path)" in _ENROLL_SCRIPT
backend/tests/test_privatemuse_deployment_enrollment.py
+1 -1
View File
@@ -75,7 +75,7 @@ def test_enrollment_uses_fixed_private_muse_contract_and_controller_exec():
assert "appops_managed_target" not in canonical_payload
script = command[2]
assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script
assert "/controller-hub-releases/controller-config/projects.json" in script
assert "/srv/example/appdata/gitea-deploy-runners/controller/projects.json" not in script
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
backend/tests/test_privatemuse_webhook_repair.py
+1 -1
View File
@@ -81,7 +81,7 @@ def test_privatemuse_webhook_repair_executes_only_fixed_contract():
def test_privatemuse_webhook_repair_script_never_returns_credentials():
assert "runner_guard_admin_token" in _REPAIR_SCRIPT
assert 'secret_root = Path("/opt/gitea-deploy/webhook-secrets")' in _REPAIR_SCRIPT
assert 'secret_root = Path("/controller-config/webhook-secrets")' in _REPAIR_SCRIPT
assert 'secret_path = secret_root / "jens__privatemuse"' in _REPAIR_SCRIPT
assert "secretMaterialReturned" not in _REPAIR_SCRIPT
assert 'print(f"privatemuse_runnerless_webhook_ready hook_id={hook_id}"' in _REPAIR_SCRIPT
backend/tests/test_repopulse_target_attestation.py
+50
View File
@@ -5,11 +5,15 @@ from __future__ import annotations
import hashlib
import hmac
import importlib
import importlib.util
import json
import sqlite3
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
def _module(monkeypatch):
root = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"
@@ -17,6 +21,52 @@ def _module(monkeypatch):
return importlib.import_module("repopulse_target_attestation")
@pytest.mark.parametrize("code_root", [Path("/opt/gitea-deploy"), Path("/private/controller-stage")])
def test_bootstrap_producer_and_verifier_share_the_config_path(monkeypatch, code_root):
root = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
monkeypatch.syspath_prepend(str(root))
spec = importlib.util.spec_from_file_location("config_path_attestation_test", root / "repopulse_target_attestation.py")
verifier = importlib.util.module_from_spec(spec)
verifier.__file__ = str(code_root / "repopulse_target_attestation.py")
exec(compile((root / "repopulse_target_attestation.py").read_bytes(), str(spec.origin), "exec"), verifier.__dict__)
expected = (
Path("/controller-hub-releases/controller-config")
if code_root == Path("/opt/gitea-deploy") else code_root
) / "appops-enrollment-bootstrap"
assert verifier.BOOTSTRAP_PATH == expected
monkeypatch.setitem(sys.modules, "repopulse_target_attestation", verifier)
producer_spec = importlib.util.spec_from_file_location("config_path_producer_test", root / "stage_repopulse_target_attestation.py")
producer = importlib.util.module_from_spec(producer_spec)
producer_spec.loader.exec_module(producer)
observed = []
def unavailable_secret(path, _maximum):
observed.append(path)
raise OSError("fixture stops before any external operation")
monkeypatch.setattr(producer.os, "geteuid", lambda: 0, raising=False)
monkeypatch.setattr(producer.sys, "argv", ["stage_repopulse_target_attestation.py"])
monkeypatch.setattr(producer, "_private_bytes", unavailable_secret)
assert producer.main() == 2
assert observed == [expected]
observed.clear()
def verifier_file(path, *, maximum):
if path == verifier.ATTESTATION_PATH:
return json.dumps({"schema": verifier.ENVELOPE_SCHEMA, "claims": {}, "signatureSha256": "0" * 64}).encode()
return unavailable_secret(path, maximum)
monkeypatch.setattr(verifier, "_private_file", verifier_file)
required = {
"repository", "applicationId", "profileId", "profileVersion",
"planSha256", "intentId", "targetSecretRef", "controllerTlsCertificateSha256",
}
assert verifier.FixedTargetAttestationVerifier().verify_target_attestation(
expected={key: "fixture" for key in required}
) is None
assert observed == [expected]
def _sample(now: datetime) -> dict:
return {
"schema": "itworx.repopulse-target-attestation/v1",
backend/tests/test_runner_admission_controller.py
+1640
View File
File diff suppressed because it is too large. Load diff
backend/tests/test_runner_admission_policy.py
+179
View File
@@ -0,0 +1,179 @@
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
POLICY_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_admission_policy.py"
PROFILES_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py"
def _load(name: str, path: Path):
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@pytest.fixture
def policy():
profiles = _load("appops_runner_profiles_for_admission_test", PROFILES_PATH)
sys.modules["runner_profiles"] = profiles
return _load("appops_runner_admission_policy_test", POLICY_PATH)
def _job(job_id: str, labels: list[str], queued_at: float):
return {"id": job_id, "labels": labels, "queued_at": queued_at}
def test_oldest_compatible_job_wins_and_profile_choice_rotates(policy):
jobs = [
_job("newer-security", ["itworx-ci-security"], 200.0),
_job("older-linux", ["self-hosted", "linux-validation"], 100.0),
]
first = policy.select_admission(
jobs,
available_profiles=["backend", "frontend", "security"],
active_claims=[],
cursor=0,
)
second = policy.select_admission(
jobs,
available_profiles=["backend", "frontend", "security"],
active_claims=[],
cursor=first.next_cursor,
)
assert (first.job_id, first.profile_id) == ("older-linux", "backend")
assert (second.job_id, second.profile_id) == ("older-linux", "frontend")
def test_enabled_legacy_slots_count_as_claims_and_can_consume_all_capacity(policy):
decision = policy.select_admission(
[_job("checks", ["itworx-ci-checks"], 1.0)],
available_profiles=["checks"],
# Two old broad Tower registrations are charged as two backend-envelope slots.
active_claims=["backend", "backend"],
)
assert decision.profile_id is None
assert decision.reason == "runner_admission_claim_limit"
def test_duplicate_legacy_claims_are_charged_against_dind_and_family_budgets(policy):
dind_limited = policy.validate_claims(["frontend", "backend"])
family_excess = policy.validate_claims(["build", "backend"])
assert dind_limited.ok
assert family_excess.reason == "runner_admission_dind_budget_exceeded"
def test_two_heavy_profiles_cannot_be_admitted_together(policy):
result = policy.validate_claims(["browser", "build"])
assert not result.ok
assert result.reason == "runner_admission_heavy_claim_limit"
def test_unmapped_enabled_registration_fails_closed(policy):
result = policy.validate_claims(["unknown-legacy-runner"])
assert not result.ok
assert result.reason == "runner_admission_profile_unmapped"
def test_candidate_is_reserved_before_it_can_be_selected(policy):
result = policy.select_admission(
[_job("build", ["itworx-ci-build"], 1.0)],
available_profiles=["build"],
active_claims=["browser"],
)
assert result.profile_id is None
assert result.reason == "runner_admission_heavy_claim_limit"
def test_unknown_required_job_label_does_not_fall_back_to_a_broad_profile(policy):
result = policy.select_admission(
[_job("mystery", ["linux-validation", "unraid-deploy"], 1.0)],
available_profiles=["backend", "frontend"],
active_claims=[],
)
assert result.profile_id is None
assert result.reason == "runner_admission_no_compatible_queue"
def test_legacy_docker_alias_keeps_heavy_reservation_and_budget_gates(policy):
selected = policy.select_admission(
[_job("docker", ["docker-validation"], 1.0)],
available_profiles=["build"], active_claims=[],
)
assert selected.profile_id == "build"
blocked = policy.select_admission(
[_job("docker", ["docker-validation"], 1.0)],
available_profiles=["build"], active_claims=["browser"],
)
assert blocked.profile_id is None
assert blocked.reason == "runner_admission_heavy_claim_limit"
def test_malformed_queue_row_fails_closed(policy):
with pytest.raises(policy.RunnerAdmissionPolicyError):
policy.select_admission(
[{"id": "bad", "labels": ["itworx-ci-checks"], "queued_at": "now"}],
available_profiles=["checks"],
active_claims=[],
)
def test_lease_yields_to_the_oldest_eligible_other_profile_without_stealing_same_profile_work(
policy,
):
queue = [
_job("older-build", ["itworx-ci-build"], 100.0),
_job("newer-backend", ["linux-validation"], 200.0),
]
decision = policy.select_lease_yield(
queue,
leased_profile="backend",
available_profiles=["checks", "security", "browser", "build"],
active_claims=["backend", "frontend"],
cursor=0,
)
assert (decision.profile_id, decision.job_id) == ("build", "older-build")
same_profile_is_oldest = policy.select_lease_yield(
[
_job("older-backend", ["linux-validation"], 100.0),
_job("newer-build", ["itworx-ci-build"], 200.0),
],
leased_profile="backend",
available_profiles=["checks", "security", "browser", "build"],
active_claims=["backend", "frontend"],
cursor=0,
)
assert same_profile_is_oldest.profile_id is None
assert same_profile_is_oldest.reason == "runner_admission_lease_retained"
def test_lease_does_not_yield_when_current_claims_can_admit_queued_work(policy):
decision = policy.select_lease_yield(
[_job("checks", ["itworx-ci-checks"], 100.0)],
leased_profile="backend",
available_profiles=["checks"],
active_claims=["backend"],
cursor=0,
)
assert decision.profile_id is None
assert decision.reason == "runner_admission_lease_not_needed"
backend/tests/test_runner_capacity_controller.py
+275
View File
@@ -1,7 +1,9 @@
from __future__ import annotations
import importlib.util
import io
import sys
import types
from pathlib import Path
import pytest
@@ -14,6 +16,9 @@ CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py"
@pytest.fixture
def capacity_module(monkeypatch):
if sys.platform == "win32":
# The daemon lock is not exercised by these in-process cycle tests.
monkeypatch.setitem(sys.modules, "fcntl", types.ModuleType("fcntl"))
monkeypatch.syspath_prepend(str(CONTROLLER_ROOT))
spec = importlib.util.spec_from_file_location(
"appops_runner_capacity_controller_test", CONTROLLER_PATH
@@ -25,6 +30,137 @@ def capacity_module(monkeypatch):
return module
def _wire_read_only_host(service, module, monkeypatch, samples):
counters = iter(samples)
monkeypatch.setattr(service, "_strict_swap_pages", lambda: next(counters))
monkeypatch.setattr(module.os, "cpu_count", lambda: 20)
monkeypatch.setattr(module.os, "getloadavg", lambda: (4.0, 3.0, 2.0), raising=False)
monkeypatch.setattr(module.os, "sysconf", lambda _name: 4096, raising=False)
monkeypatch.setattr(service, "_mem_available_bytes", lambda: 16 * 1024**3)
monkeypatch.setattr(service, "_cache_free_bytes", lambda: 32 * 1024**3)
def forbidden(*_args, **_kwargs):
pytest.fail("read-only host sampling touched burst inventory or a mutation")
for name in ("_burst_container", "_runner_registration", "_gitea_request",
"_run", "_read_state", "_write_state", "_record"):
monkeypatch.setattr(service, name, forbidden)
def test_host_snapshot_measures_swap_without_optional_burst(capacity_module, monkeypatch):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [(50, 70)])
prior = {"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}
preserved = dict(prior)
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("baseline already available"))
result = service.host_snapshot(prior, now=100.0)
assert result["status"] == "observed"
assert result["host"]["loadRatio"] == 0.2
assert result["host"]["swapInKbps"] == 200.0
assert result["host"]["swapOutKbps"] == 280.0
assert result["host"]["swapCritical"] is False
assert prior == preserved
@pytest.mark.parametrize("prior", [None, {"swapSampleAt": 0}, {
"swapSampleAt": 1.0, "swapInPages": 5, "swapOutPages": 5,
}, {"swapSampleAt": float("nan"), "swapInPages": 5, "swapOutPages": 5}])
def test_host_snapshot_takes_two_real_samples_for_missing_or_stale_baseline(
capacity_module, monkeypatch, prior
):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (266, 532)])
elapsed = iter([4.0, 5.0])
sleeps = []
monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed))
monkeypatch.setattr(capacity_module.time, "sleep", sleeps.append)
result = service.host_snapshot(prior, now=100.0)
assert result["status"] == "observed"
assert result["host"]["swapInKbps"] == 1024.0
assert result["host"]["swapOutKbps"] == 2048.0
assert result["host"]["swapSampleAt"] == 101.0
assert sleeps == [1.0]
def test_host_snapshot_reports_unavailable_if_swap_counter_resets(capacity_module, monkeypatch):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (9, 20)])
elapsed = iter([4.0, 5.0])
monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed))
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: None)
result = service.host_snapshot(now=100.0)
assert result["status"] == "unavailable"
assert result["reason"] == "runner_capacity_swap_baseline_unavailable"
assert result["host"] is None
def test_host_snapshot_rejects_reset_against_recent_protected_baseline(capacity_module, monkeypatch):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [(9, 20), (10, 21)])
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("recent counter reset must fail closed"))
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 10, "swapOutPages": 20}, now=100.0)
assert result["status"] == "unavailable"
assert result["reason"] == "runner_capacity_swap_baseline_unavailable"
assert result["host"] is None
def test_host_snapshot_handles_absent_load_tuple(capacity_module, monkeypatch):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)])
monkeypatch.setattr(capacity_module.os, "getloadavg", lambda: ())
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0)
assert result["status"] == "unavailable"
assert result["host"] is None
@pytest.mark.parametrize("raw", ["", "pswpin 1\n", "pswpin -1\npswpout 2\n",
"pswpin 1\npswpin 2\npswpout 3\n", "pswpin invalid\npswpout 2\n", "x" * 65537])
def test_strict_swap_reader_rejects_missing_or_invalid_counters(capacity_module, monkeypatch, raw):
monkeypatch.setattr(capacity_module, "Path", lambda _path: types.SimpleNamespace(
open=lambda **_kwargs: io.StringIO(raw)
))
with pytest.raises(capacity_module.RunnerCapacityError, match="swap_telemetry_unavailable"):
capacity_module.RunnerCapacityController._strict_swap_pages()
def test_host_snapshot_does_not_turn_missing_swap_into_zero(capacity_module, monkeypatch):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [])
def unavailable():
raise capacity_module.RunnerCapacityError("runner_capacity_swap_telemetry_unavailable")
monkeypatch.setattr(service, "_strict_swap_pages", unavailable)
result = service.host_snapshot(now=100.0)
assert result["status"] == "unavailable"
assert result["host"] is None
@pytest.mark.parametrize("metric", ["cpu_count", "getloadavg", "sysconf", "memory", "cache"])
def test_host_snapshot_rejects_invalid_global_metrics(capacity_module, monkeypatch, metric):
service = capacity_module.RunnerCapacityController()
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)])
if metric == "getloadavg":
monkeypatch.setattr(capacity_module.os, metric, lambda: (float("nan"), 0, 0))
elif metric in {"cpu_count", "sysconf"}:
monkeypatch.setattr(capacity_module.os, metric, lambda *_args: 0)
else:
monkeypatch.setattr(service, "_mem_available_bytes" if metric == "memory" else "_cache_free_bytes", lambda: True)
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0)
assert result["status"] == "unavailable"
assert result["host"] is None
def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True):
monkeypatch.setattr(
service,
@@ -50,6 +186,7 @@ def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=Tr
monkeypatch.setattr(service, "_read_state", lambda: {})
monkeypatch.setattr(service, "_write_state", lambda _state: None)
monkeypatch.setattr(service, "_record", lambda _event: None)
monkeypatch.setattr(service, "_host_guard_state", lambda: ("clear", ""))
def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch):
@@ -218,3 +355,141 @@ def test_cycle_never_stops_when_gitea_still_has_active_assignment(
assert result["decision"]["result"] == "stop_deferred_not_proven_idle"
assert commands == []
def test_paused_host_guard_blocks_burst_start_before_docker_or_gitea_write(
capacity_module, monkeypatch
):
service = capacity_module.RunnerCapacityController()
_wire_common(service, monkeypatch, running=False, disabled=True)
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
monkeypatch.setattr(
service,
"_host",
lambda _previous, now: {
"cpuCount": 20,
"loadOne": 2.0,
"loadRatio": 0.1,
"availableMemoryBytes": 20 * 1024**3,
"cacheFreeBytes": 100 * 1024**3,
"swapInKbps": 0.0,
"swapOutKbps": 0.0,
"swapPressureSamples": 0,
"swapCritical": False,
"swapSampleAt": now,
"swapInPages": 0,
"swapOutPages": 0,
},
)
monkeypatch.setattr(service, "_host_guard_state", lambda: ("paused", "guard"))
commands = []
disabled = []
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
monkeypatch.setattr(
service,
"_set_disabled",
lambda runner_id, value: disabled.append((runner_id, value)),
)
result = service.cycle()
assert result["decision"]["action"] == "start"
assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused"
assert commands == []
assert disabled == []
def test_guard_is_rechecked_after_docker_start_before_gitea_enable(
capacity_module, monkeypatch
):
service = capacity_module.RunnerCapacityController()
_wire_common(service, monkeypatch, running=False, disabled=True)
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
monkeypatch.setattr(
service,
"_host",
lambda _previous, now: {
"cpuCount": 20,
"loadOne": 2.0,
"loadRatio": 0.1,
"availableMemoryBytes": 20 * 1024**3,
"cacheFreeBytes": 100 * 1024**3,
"swapInKbps": 0.0,
"swapOutKbps": 0.0,
"swapPressureSamples": 0,
"swapCritical": False,
"swapSampleAt": now,
"swapInPages": 0,
"swapOutPages": 0,
},
)
guard_states = iter([("clear", ""), ("paused", "guard")])
monkeypatch.setattr(service, "_host_guard_state", lambda: next(guard_states))
commands = []
disabled = []
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
monkeypatch.setattr(
service,
"_set_disabled",
lambda runner_id, value: disabled.append((runner_id, value)),
)
result = service.cycle()
assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused"
assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]]
assert disabled == []
def test_fresh_pressure_is_rechecked_before_burst_start(
capacity_module, monkeypatch
):
service = capacity_module.RunnerCapacityController()
_wire_common(service, monkeypatch, running=False, disabled=True)
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
samples = iter(
[
{
"loadRatio": 0.1,
"availableMemoryBytes": 20 * 1024**3,
"cacheFreeBytes": 100 * 1024**3,
"swapCritical": False,
},
{
"loadRatio": 0.95,
"availableMemoryBytes": 20 * 1024**3,
"cacheFreeBytes": 100 * 1024**3,
"swapCritical": False,
},
]
)
def sample(_previous, now):
values = next(samples)
return {
"cpuCount": 20,
"loadOne": values["loadRatio"] * 20,
**values,
"swapInKbps": 0.0,
"swapOutKbps": 0.0,
"swapPressureSamples": 0,
"swapSampleAt": now,
"swapInPages": 0,
"swapOutPages": 0,
}
monkeypatch.setattr(service, "_host", sample)
commands = []
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
disabled = []
monkeypatch.setattr(
service,
"_set_disabled",
lambda runner_id, value: disabled.append((runner_id, value)),
)
result = service.cycle()
assert result["decision"]["result"] == "failed:runner_capacity_pressure_critical"
assert commands == []
assert disabled == []
backend/tests/test_runner_controller_artifact.py
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import importlib.util
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
def load(name, path):
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.fixture
def artifact(tmp_path, monkeypatch):
renderer = load('runner_artifact_renderer', ROOT / 'scripts/prepare_controller_artifact.py')
controller_dir = ROOT / 'ops/gitea-deploy-broker'
monkeypatch.syspath_prepend(str(controller_dir))
controller = load('runner_artifact_controller', controller_dir / 'status_server.py')
stage = tmp_path / 'controller'
stage.mkdir()
for name in renderer._ARTIFACT_FILES:
source = ROOT / 'scripts/run_repopulse_adoption.py' if name == 'run_repopulse_adoption.py' else controller_dir / name
shutil.copyfile(source, stage / name)
result = subprocess.run([
sys.executable, str(ROOT / 'scripts/prepare_controller_artifact.py'),
'--artifact-dir', str(stage), '--source-repository', 'jens/unraid-appops-gateway',
'--source-revision', 'a' * 40, '--source-tree', 'b' * 40,
'--controller-version', '2.1.13', '--built-at', '2026-10-01T12:00:00Z',
], capture_output=True, text=True, check=True)
assert 'artifactSha256' in result.stdout
monkeypatch.setattr(controller, 'ROOT', stage)
monkeypatch.setattr(controller, 'CONTROLLER_RELEASE_PATH', stage / 'controller-release.json')
return controller, stage
def test_renderer_and_runtime_accept_the_same_complete_specialized_artifact(artifact):
controller, _stage = artifact
release = controller._controller_release()
assert release['controllerVersion'] == '2.1.13'
assert release['sourceRevision'] == 'a' * 40
@pytest.mark.parametrize('damage', ['missing_binding', 'missing_hub_store', 'changed_admission', 'symlink_envelope'])
def test_incomplete_or_changed_specialized_artifact_cannot_be_promoted(artifact, damage):
controller, stage = artifact
if damage == 'missing_binding':
(stage / 'runner_statistics_binding.py').unlink()
elif damage == 'missing_hub_store':
(stage / 'mcp_hub_release_store.py').unlink()
elif damage == 'changed_admission':
with (stage / 'runner_admission_policy.py').open('ab') as stream:
stream.write(b'\n# changed after manifest rendering\n')
else:
target = stage / 'runner_resource_envelope.py'
original = stage / 'moved_envelope.py'
target.rename(original)
try:
target.symlink_to(original)
except OSError:
pytest.skip('Host cannot create symlink fixture')
assert controller._controller_release() is None
backend/tests/test_runner_fleet.py
+81 -1
View File
@@ -1,6 +1,9 @@
from __future__ import annotations
import importlib.util
import sys
import types
from datetime import UTC, datetime
from pathlib import Path
import pytest
@@ -11,14 +14,71 @@ FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
@pytest.fixture
def fleet_module():
def fleet_module(monkeypatch):
if sys.platform == "win32" and "fcntl" not in sys.modules:
fcntl = types.ModuleType("fcntl")
fcntl.LOCK_EX = 2
fcntl.LOCK_UN = 8
fcntl.LOCK_NB = 4
fcntl.flock = lambda *_args, **_kwargs: None
monkeypatch.setitem(sys.modules, "fcntl", fcntl)
spec = importlib.util.spec_from_file_location("appops_runner_fleet_test", FLEET_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
monkeypatch.setitem(
sys.modules,
"runner_fleet_specialize",
types.SimpleNamespace(specialization_state=lambda: "legacy"),
)
monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", ""))
monkeypatch.setattr(
module.RunnerCapacityController,
"snapshot",
lambda _self: _healthy_capacity_snapshot(),
)
return module
def _healthy_capacity_snapshot():
return {
"status": "ok",
"decision": {"pressure": "headroom"},
"host": {
"swapCritical": False,
"loadRatio": 0.1,
"availableMemoryBytes": 12 * 1024**3,
"cacheFreeBytes": 100 * 1024**3,
"swapSampleAt": datetime.now(UTC).timestamp(),
},
}
def test_dind_uses_private_unix_socket_only_for_trusted_active_specialization(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
commands = []
monkeypatch.setattr(
service,
"_run",
lambda args, **_kwargs: commands.append(args) or "",
)
specialization = sys.modules["runner_fleet_specialize"]
service._dind(["info"])
assert "DOCKER_HOST=tcp://127.0.0.1:2375" in commands[-1]
monkeypatch.setattr(specialization, "specialization_state", lambda: "active")
service._dind(["info"])
assert f"DOCKER_HOST={fleet_module.DIND_DOCKER_HOST}" in commands[-1]
for state in ("legacy", "blocked", "in_progress"):
monkeypatch.setattr(specialization, "specialization_state", lambda value=state: value)
service._dind(["info"])
assert "DOCKER_HOST=tcp://127.0.0.1:2375" in commands[-1]
def test_cleanup_candidates_exclude_unknown_attached_and_labeled_networks(
fleet_module, monkeypatch
):
@@ -80,6 +140,7 @@ def test_automatic_cycle_only_deletes_exact_preview_candidates(fleet_module, mon
}
snapshot = {
"status": "healthy",
"capacity": _healthy_capacity_snapshot(),
"networks": {"cleanupCandidates": [candidate]},
"storage": {"status": "healthy"},
"topology": {
@@ -143,6 +204,7 @@ def test_automatic_cycle_starts_only_an_exited_known_runner(fleet_module, monkey
service = fleet_module.RunnerFleetService()
snapshot = {
"status": "healthy",
"capacity": _healthy_capacity_snapshot(),
"networks": {"cleanupCandidates": []},
"storage": {"status": "healthy"},
"topology": {
@@ -178,3 +240,21 @@ def test_automatic_cycle_starts_only_an_exited_known_runner(fleet_module, monkey
assert commands == [["docker", "start", "itworx-validation-runner-2"]]
assert {event["action"] for event in events} == {"start", "restart.deferred"}
@pytest.mark.parametrize("specialization", ["active", "blocked"])
def test_legacy_automatic_cycle_never_observes_or_mutates_after_specialization(
fleet_module, monkeypatch, specialization
):
service = fleet_module.RunnerFleetService()
monkeypatch.setitem(
sys.modules,
"runner_fleet_specialize",
types.SimpleNamespace(specialization_state=lambda: specialization),
)
monkeypatch.setattr(service, "snapshot", lambda: pytest.fail("legacy snapshot ran"))
monkeypatch.setattr(service, "_run", lambda *_args, **_kwargs: pytest.fail("legacy mutation ran"))
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == "runner_fleet_specialization_owns_recovery"
backend/tests/test_runner_fleet_scheduler_recovery.py
+498 -29
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import importlib.util
import sys
import types
from datetime import UTC, datetime
from pathlib import Path
@@ -12,41 +14,93 @@ FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
@pytest.fixture
def fleet_module():
def fleet_module(monkeypatch):
if sys.platform == "win32" and "fcntl" not in sys.modules:
fcntl = types.ModuleType("fcntl")
fcntl.LOCK_EX = 2
fcntl.LOCK_UN = 8
fcntl.LOCK_NB = 4
fcntl.flock = lambda *_args, **_kwargs: None
monkeypatch.setitem(sys.modules, "fcntl", fcntl)
spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module._host_guard_state_unpatched = module._host_guard_state
monkeypatch.setitem(
sys.modules,
"runner_fleet_specialize",
types.SimpleNamespace(specialization_state=lambda: "legacy"),
)
monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", ""))
monkeypatch.setattr(
module.RunnerCapacityController,
"snapshot",
lambda _self: _healthy_capacity_snapshot(),
)
return module
def _healthy_capacity_snapshot():
return {
"status": "ok",
"decision": {"pressure": "headroom"},
"host": {
"swapCritical": False,
"loadRatio": 0.1,
"availableMemoryBytes": 12 * 1024**3,
"cacheFreeBytes": 100 * 1024**3,
"swapSampleAt": datetime.now(UTC).timestamp(),
},
}
def _healthy_snapshot():
runners = [
{
"name": name,
"running": True,
"health": "healthy",
"restarts": 0,
"startedAt": "2026-09-16T18:00:00Z",
"oomKilled": False,
"restartPolicy": "unless-stopped",
}
for name in (
"itworx-security-runner",
"itworx-validation-runner-2",
)
]
return {
"status": "healthy",
"networks": {"cleanupCandidates": []},
"storage": {"status": "healthy"},
"generatedAt": "2026-10-01T10:00:00Z",
"capacity": _healthy_capacity_snapshot(),
"networks": {"count": 0, "capacity": None, "cleanupCandidates": []},
"storage": {
"status": "healthy",
"freeBytes": 30 * 1024**3,
"totalBytes": 40 * 1024**3,
},
"scheduler": {
"status": "healthy",
"candidate": None,
"stallThresholdSeconds": 600,
},
"topology": {
"dind": {
"name": "gitea-runner-dind",
"running": True,
"health": "healthy",
"restarts": 0,
"startedAt": "2026-09-16T18:00:00Z",
"oomKilled": False,
"restartPolicy": "unless-stopped",
},
"runners": [
{
"name": "itworx-validation-runner-2",
"running": True,
"health": "healthy",
"startedAt": "2026-09-16T18:00:00Z",
},
{
"name": "itworx-security-runner",
"running": True,
"health": "healthy",
"startedAt": "2026-09-16T18:00:00Z",
},
]
"runners": runners,
"generalSlots": 2,
"runnerCount": len(runners),
"specializedRunners": [],
"burstRunner": None,
},
}
@@ -220,19 +274,20 @@ def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting(
assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"}
def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind(
def test_automatic_cycle_recovers_critical_exited_dind_with_safe_host_capacity(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["status"] = "critical"
snapshot["topology"]["dind"] = {
"name": "gitea-runner-dind",
snapshot["topology"]["dind"].update(
{
"running": False,
"health": "none",
"startedAt": "",
"restartPolicy": "no",
}
)
commands: list[tuple[list[str], int]] = []
events: list[dict[str, object]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
@@ -250,14 +305,45 @@ def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind(
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30),
(["docker", "start", "gitea-runner-dind"], 30),
]
assert result["dindRecovery"]["status"] == "requested"
assert result["schedulerRecovery"]["status"] == "deferred_for_dind_recovery"
assert any(
event.get("action") == "start"
and event.get("target") == "gitea-runner-dind"
and event.get("reason") == "known_ci_dind_exited"
for event in events
assert [event["action"] for event in events] == [
"restart-policy.update",
"start",
]
assert result["status"] == "ok"
assert result["dindRecovery"] == {
"status": "requested",
"restartPolicy": "requested",
"target": "gitea-runner-dind",
}
def test_automatic_cycle_repairs_fixed_dind_policy_when_healthy_and_guard_clear(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["topology"]["dind"]["restartPolicy"] = "no"
commands: list[tuple[list[str], int]] = []
events: list[dict[str, object]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(
service,
"_run",
lambda args, timeout=20: commands.append((args, timeout)) or "",
)
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
result = service.automatic_cycle()
assert commands == [
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30)
]
assert result["dindRecovery"] == {
"status": "not_needed",
"restartPolicy": "requested",
}
assert len(events) == 1
assert events[0]["action"] == "restart-policy.update"
def test_automatic_cycle_never_restarts_running_unhealthy_dind(
@@ -265,7 +351,7 @@ def test_automatic_cycle_never_restarts_running_unhealthy_dind(
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["status"] = "critical"
snapshot["status"] = "healthy"
snapshot["topology"]["dind"] = {
"name": "gitea-runner-dind",
"running": True,
@@ -293,6 +379,385 @@ def test_automatic_cycle_never_restarts_running_unhealthy_dind(
)
@pytest.mark.parametrize(
("guard_state", "guard_reason"),
[
(("paused", "host_resource_guard_paused"), "host_resource_guard_paused"),
(("unavailable", "host_guard_state_untrusted"), "host_guard_state_untrusted"),
],
)
def test_automatic_cycle_never_mutates_when_host_guard_is_paused_or_untrusted(
fleet_module, monkeypatch, guard_state, guard_reason
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["topology"]["dind"].update(
{"running": False, "health": "none", "restartPolicy": "no"}
)
commands: list[list[str]] = []
events: list[dict[str, object]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: guard_state)
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == guard_reason
assert commands == []
assert events == []
@pytest.mark.parametrize(
"action",
[
"restart-policy-update",
"dind-start",
"scheduler-restart",
"network-remove",
"cache-prune",
"runner-start",
],
)
def test_guard_pause_appearing_after_cycle_entry_blocks_each_mutation(
fleet_module, monkeypatch, action
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
dind = snapshot["topology"]["dind"]
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
if action == "restart-policy-update":
dind["restartPolicy"] = "no"
elif action == "dind-start":
dind.update({"running": False, "health": "none"})
elif action == "scheduler-restart":
snapshot["topology"]["runners"] = [
{
"name": "itworx-validation-runner-2",
"running": True,
"health": "healthy",
}
]
monkeypatch.setattr(
service,
"_scheduler_recovery_candidate",
lambda: {
"target": "itworx-validation-runner-2",
"jobId": 22,
"ageSeconds": 900,
"reason": "stale_scheduler_assignment",
},
)
monkeypatch.setattr(
service,
"_container",
lambda name: {
"name": name,
"running": True,
"health": "healthy",
},
)
elif action == "network-remove":
snapshot["networks"]["cleanupCandidates"] = [
{
"id": "a" * 64,
"name": "GITEA-ACTIONS-TASK-22-WORKFLOW-ci-JOB-test-network",
"ageSeconds": 3600,
"reason": "empty_gitea_actions_task_network_in_dedicated_dind",
}
]
elif action == "cache-prune":
snapshot["storage"]["status"] = "critical"
elif action == "runner-start":
snapshot["topology"]["runners"] = [
{
"name": "itworx-validation-runner-2",
"running": False,
"health": "unhealthy",
}
]
guard_states = iter(
[
("clear", ""),
("paused", "host_resource_guard_paused"),
]
)
commands: list[list[str]] = []
events: list[dict[str, object]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: next(guard_states))
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == "host_resource_guard_paused"
assert commands == []
assert events == []
def test_critical_current_pressure_after_cycle_entry_blocks_mutation(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["topology"]["dind"]["restartPolicy"] = "no"
current_samples = iter(
[
{
"status": "ok",
"decision": {"pressure": "critical"},
"host": {"swapCritical": True},
}
]
)
commands: list[list[str]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
monkeypatch.setattr(
fleet_module.RunnerCapacityController,
"snapshot",
lambda _self: next(current_samples),
)
monkeypatch.setattr(
service,
"_run",
lambda args, timeout=20: commands.append(args) or "",
)
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == "runner_capacity_pressure_critical"
assert commands == []
def test_stale_capacity_sample_blocks_automatic_mutation(fleet_module, monkeypatch):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["topology"]["dind"]["restartPolicy"] = "no"
stale = _healthy_capacity_snapshot()
stale["host"]["swapSampleAt"] = (
datetime.now(UTC).timestamp()
- fleet_module.CAPACITY_SAMPLE_MAX_AGE_SECONDS
- 1
)
commands: list[list[str]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
monkeypatch.setattr(
fleet_module.RunnerCapacityController,
"snapshot",
lambda _self: stale,
)
monkeypatch.setattr(
service,
"_run",
lambda args, timeout=20: commands.append(args) or "",
)
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == "runner_capacity_telemetry_unavailable"
assert commands == []
@pytest.mark.parametrize("capacity", [{"status": "unavailable"}, {"status": "not_configured"}])
def test_unavailable_or_missing_current_pressure_blocks_mutation(
fleet_module, monkeypatch, capacity
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["topology"]["dind"]["restartPolicy"] = "no"
commands: list[list[str]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
monkeypatch.setattr(
fleet_module.RunnerCapacityController,
"snapshot",
lambda _self: capacity,
)
monkeypatch.setattr(
service,
"_run",
lambda args, timeout=20: commands.append(args) or "",
)
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == "runner_capacity_telemetry_unavailable"
assert commands == []
def test_automatic_cycle_rejects_incomplete_critical_snapshot(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["status"] = "critical"
snapshot["topology"]["dind"].update(
{"running": False, "health": "none", "restartPolicy": "no"}
)
snapshot["topology"].pop("runnerCount")
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(
service, "_run", lambda *_args, **_kwargs: pytest.fail("critical fleet must not mutate")
)
monkeypatch.setattr(
service, "_record", lambda **_event: pytest.fail("critical fleet must not audit a mutation")
)
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == "runner_fleet_snapshot_untrusted"
assert result["dindRecovery"]["status"] == "deferred"
def test_automatic_cycle_recovers_stalled_scheduler_in_critical_snapshot(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["status"] = "critical"
snapshot["scheduler"] = {
"status": "stalled",
"candidate": {
"target": "itworx-validation-runner-2",
"jobId": 22931,
"label": "linux-validation",
"ageSeconds": 900,
"reason": "stale_scheduler_assignment",
},
"stallThresholdSeconds": 600,
}
commands: list[tuple[list[str], int]] = []
events: list[dict[str, object]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(
service,
"_scheduler_recovery_candidate",
lambda: snapshot["scheduler"]["candidate"],
)
monkeypatch.setattr(
service,
"_container",
lambda name: {
"name": name,
"running": True,
"health": "healthy",
"startedAt": "2026-09-16T18:00:00Z",
"restarts": 0,
"oomKilled": False,
"restartPolicy": "unless-stopped",
},
)
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
monkeypatch.setattr(
service,
"_run",
lambda args, timeout=20: commands.append((args, timeout)) or "",
)
monkeypatch.setattr(service, "_record", lambda **event: events.append(event) or True)
result = service.automatic_cycle()
assert commands == [
(["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45)
]
assert result["status"] == "ok"
assert result["schedulerRecovery"] == {
"status": "requested",
"target": "itworx-validation-runner-2",
"jobId": 22931,
}
assert [event["action"] for event in events] == ["restart", "restart.result"]
def test_automatic_cycle_allows_critical_storage_prune_with_safe_host_capacity(
fleet_module, monkeypatch
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["status"] = "critical"
snapshot["storage"] = {
"status": "critical",
"freeBytes": 8 * 1024**3,
"totalBytes": 40 * 1024**3,
}
commands: list[list[str]] = []
events: list[dict[str, object]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
result = service.automatic_cycle()
assert commands == [
["image", "prune", "--force"],
["builder", "prune", "--force", "--filter", "until=168h"],
]
assert result["status"] == "ok"
assert events == [
{
"action": "cache.prune",
"target": "gitea-runner-dind",
"reason": "ci_storage_critical",
"result": "completed",
}
]
@pytest.mark.parametrize(
("capacity", "reason"),
[
({"status": "ok", "decision": {"pressure": "critical"}, "host": {"swapCritical": False}},
"runner_capacity_pressure_critical"),
({"status": "unavailable"}, "runner_capacity_telemetry_unavailable"),
({"status": "ok", "decision": {}, "host": {}}, "runner_capacity_telemetry_untrusted"),
],
)
def test_automatic_cycle_blocks_critical_or_untrusted_capacity_snapshot(
fleet_module, monkeypatch, capacity, reason
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["status"] = "critical"
snapshot["capacity"] = capacity
snapshot["topology"]["dind"].update(
{"running": False, "health": "none", "restartPolicy": "no"}
)
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(
service, "_run", lambda *_args, **_kwargs: pytest.fail("unsafe capacity must not mutate")
)
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
result = service.automatic_cycle()
assert result["status"] == "deferred"
assert result["reason"] == reason
def test_host_guard_probe_fails_closed_when_path_is_not_posix_root(monkeypatch, fleet_module):
monkeypatch.setattr(fleet_module.os, "name", "nt")
assert fleet_module._host_guard_state_unpatched() == (
"unavailable",
"host_guard_state_untrusted",
)
def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch):
service = fleet_module.RunnerFleetService()
now = 1_800_000_000
@@ -952,7 +1417,11 @@ def test_scheduler_recovery_rejects_runner_without_healthy_docker_state(
):
service = fleet_module.RunnerFleetService()
snapshot = _healthy_snapshot()
snapshot["topology"]["runners"][0]["health"] = "unhealthy"
next(
runner
for runner in snapshot["topology"]["runners"]
if runner["name"] == "itworx-validation-runner-2"
)["health"] = "unhealthy"
commands: list[list[str]] = []
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
monkeypatch.setattr(
backend/tests/test_runner_fleet_specialize.py
+2719
View File
File diff suppressed because it is too large. Load diff
backend/tests/test_runner_profiles.py
+90
View File
@@ -0,0 +1,90 @@
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PROFILES_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py"
def _profiles_module():
spec = importlib.util.spec_from_file_location("appops_runner_profiles_test", PROFILES_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_fixed_profiles_expose_stable_names_labels_and_resource_envelopes():
profiles = _profiles_module()
assert tuple(profiles.RUNNER_PROFILES) == (
"checks",
"backend",
"frontend",
"security",
"browser",
"build",
)
assert [profile.display_name for profile in profiles.RUNNER_PROFILES.values()] == [
"ITWorx CI · Checks",
"ITWorx CI · Backend",
"ITWorx CI · Frontend",
"ITWorx CI · Security",
"ITWorx CI · Browser",
"ITWorx CI · Build",
]
assert [profile.container_name for profile in profiles.RUNNER_PROFILES.values()] == [
"itworx-ci-checks",
"itworx-ci-backend",
"itworx-ci-frontend",
"itworx-ci-security",
"itworx-ci-browser",
"itworx-ci-build",
]
assert [
(profile.cpu_milli, profile.memory_mib, profile.heavy)
for profile in profiles.RUNNER_PROFILES.values()
] == [
(1000, 1536, False),
(3000, 5120, False),
(3000, 5120, False),
(1000, 2048, False),
(3000, 6144, True),
(4000, 6144, True),
]
assert (profiles.BASE_OVERHEAD_CPU_MILLI, profiles.BASE_OVERHEAD_MEMORY_MIB) == (
1000,
2048,
)
assert (profiles.HOST_CPU_BUDGET_MILLI, profiles.HOST_MEMORY_BUDGET_MIB) == (
8000,
16384,
)
assert (profiles.MAX_ACTIVE_CLAIMS, profiles.MAX_HEAVY_ACTIVE) == (2, 1)
def test_shared_legacy_labels_only_admit_backend_and_frontend_profiles():
profiles = _profiles_module()
assert [
profile.id for profile in profiles.profiles_for_labels(["linux-validation"])
] == ["backend", "frontend"]
assert [
profile.id for profile in profiles.profiles_for_labels(["ubuntu-latest"])
] == ["backend", "frontend"]
assert profiles.profiles_for_labels(["itworx-ci-security"])[0].id == "security"
assert profiles.profiles_for_labels(["linux-validation", "itworx-ci-security"]) == ()
assert profiles.profile_for_container("itworx-ci-browser").id == "browser"
assert profiles.profile_for_container("itworx-ci-unknown") is None
def test_legacy_docker_jobs_use_only_the_fixed_heavy_build_profile():
profiles = _profiles_module()
assert [profile.id for profile in profiles.profiles_for_labels(["docker-validation"])] == ["build"]
assert profiles.RUNNER_PROFILES['build'].heavy
assert profiles.profiles_for_labels(["docker-validation", "linux-validation"]) == ()
assert profiles.profiles_for_labels(["docker-validation", "unraid-deploy"]) == ()
backend/tests/test_runner_resource_envelope.py
+1106
View File
File diff suppressed because it is too large. Load diff
backend/tests/test_runner_specialization_controller_surface.py
+158
View File
@@ -0,0 +1,158 @@
from __future__ import annotations
import importlib.util
import io
import json
import sys
from email.message import Message
from pathlib import Path
from types import SimpleNamespace
import pytest
CONTROLLER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
@pytest.fixture
def controller(monkeypatch):
monkeypatch.syspath_prepend(str(CONTROLLER))
spec = importlib.util.spec_from_file_location("specialization_surface_test", CONTROLLER / "status_server.py")
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def handler(controller, monkeypatch, payload=None):
instance = object.__new__(controller.Handler)
body = json.dumps(payload or {"reason": "Specialize CI profiles"}).encode()
instance.headers = Message()
instance.headers["Content-Length"] = str(len(body))
instance.rfile = io.BytesIO(body)
instance.path = controller.RUNNER_FLEET_SPECIALIZE_PATH
responses = []
monkeypatch.setattr(instance, "_trusted_client", lambda: True)
monkeypatch.setattr(instance, "_send_json", lambda code, data: responses.append((code, json.loads(data))))
signatures = []
monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *args, **kwargs: signatures.append(kwargs) or True)
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {"revision": "a" * 40})
monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "a" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY})
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda revision: {"status": "already_deployed"})
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}})
monkeypatch.setattr(controller, "_repository_path_for_delivery", lambda *args: Path("fixed-repo.git"))
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: True)
monkeypatch.setattr(controller, "runner_fleet_specialized_topology", lambda: {"ready": False})
return instance, responses, signatures
@pytest.mark.parametrize("state", ["blocked", "active", "unknown"])
def test_legacy_upgrade_cannot_recreate_runners_after_cutover(controller, monkeypatch, state):
instance, responses, _ = handler(controller, monkeypatch)
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: state)
monkeypatch.setattr(controller, "execute_runner_fleet_v2", lambda _: pytest.fail("legacy executor must be fenced"))
instance._handle_runner_fleet_upgrade_v2()
assert responses == [(409, {"status": "legacy_fleet_upgrade_blocked"})]
def test_specialization_requires_matching_installed_controller_source(controller, monkeypatch):
instance, responses, _ = handler(controller, monkeypatch)
monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "b" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY})
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda _: pytest.fail("source mismatch must block mutation"))
instance.do_POST()
assert responses == [(409, {"status": "controller_source_mismatch"})]
def test_specialization_uses_only_signed_current_main_script(controller, monkeypatch):
instance, responses, signatures = handler(controller, monkeypatch)
commands = []
scripts = []
monkeypatch.setattr(controller.subprocess, "run", lambda command, **_: commands.append(command) or SimpleNamespace(returncode=0, stdout=b"exact source script"))
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda script: scripts.append(script) or {"status": "specialized", "verifiedRunners": [], "resourceEnvelope": {"ready": True}})
instance.do_POST()
assert signatures == [{"method": "POST", "path": controller.RUNNER_FLEET_SPECIALIZE_PATH}]
assert commands == [["git", "--git-dir=fixed-repo.git", "show", "a" * 40 + ":scripts/upgrade_ci_runner_fleet_specialized.sh"]]
assert scripts == [b"exact source script"]
assert responses[0][0] == 200
assert responses[0][1]["revision"] == "a" * 40
assert "burstRunning" not in responses[0][1]
def test_pending_protected_validation_blocks_specialization(controller, monkeypatch):
instance, responses, _ = handler(controller, monkeypatch)
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _: {"status": "preflight_rejected"})
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda _: pytest.fail("pending validation must block mutation"))
instance.do_POST()
assert responses[0][0] == 409
assert responses[0][1]["status"] == "preflight_rejected"
def test_caller_cannot_choose_script_image_or_runner(controller, monkeypatch):
instance, responses, _ = handler(controller, monkeypatch, {"reason": "Specialize CI profiles", "script": "untrusted.sh"})
instance.do_POST()
assert responses == [(400, {"status": "invalid_request"})]
@pytest.mark.parametrize("state", ["blocked", "unknown"])
def test_maintenance_never_falls_back_to_legacy_during_cutover(controller, monkeypatch, state):
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: state)
monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("legacy maintenance must be fenced"))
assert controller._runner_maintenance_cycle() == "deferred"
def test_cutover_and_maintenance_share_single_owner_lock(controller, monkeypatch):
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: pytest.fail("maintenance must not inspect during cutover"))
with controller._runner_fleet_upgrade_lock:
assert controller._runner_maintenance_cycle() == "migration_busy"
def test_active_maintenance_calls_only_admission_controller(controller, monkeypatch):
import runner_admission_controller
calls = []
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: "active")
monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "cycle", lambda _: calls.append("admission"))
monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("legacy worker must stop"))
assert controller._runner_maintenance_cycle() == "specialized"
assert calls == ["admission"]
def test_statistics_get_returns_persisted_projection_without_a_cycle(controller, monkeypatch):
import runner_admission_controller
instance, responses, _ = handler(controller, monkeypatch)
instance.path = "/v1/runner-fleet/statistics"
monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "cycle", lambda _: pytest.fail("GET may not start collection or mutate fleet"))
monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "snapshot", lambda _: {"schemaVersion": 1, "status": "unavailable"})
instance.do_GET()
assert responses == [(200, {"schemaVersion": 1, "status": "unavailable"})]
def test_statistics_configuration_uses_same_protected_source_and_fixed_signed_route(controller, monkeypatch):
instance, responses, signatures = handler(controller, monkeypatch)
instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH
calls = []
monkeypatch.setattr(controller, 'configure_runner_statistics_binding',
lambda **kwargs: calls.append(kwargs) or {'status': 'configured', 'credentialReturned': False})
monkeypatch.setattr(controller, 'runner_fleet_specialized_topology', lambda: pytest.fail('configuration must not mutate fleet'))
instance.do_POST()
assert signatures == [{'method': 'POST', 'path': controller.RUNNER_STATISTICS_CONFIGURE_PATH}]
assert calls == [{'revision': 'a' * 40, 'projects': {controller.SELF_UPGRADE_REPOSITORY: {'branch': 'main'}}}]
assert responses == [(200, {'status': 'configured', 'credentialReturned': False})]
def test_statistics_configuration_source_mismatch_writes_nothing(controller, monkeypatch):
instance, responses, _ = handler(controller, monkeypatch)
instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH
monkeypatch.setattr(controller, '_require_controller_release', lambda: {'sourceRevision': 'b' * 40, 'sourceRepository': controller.SELF_UPGRADE_REPOSITORY})
monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration'))
instance.do_POST()
assert responses == [(409, {'status': 'controller_source_mismatch'})]
def test_statistics_configuration_caller_cannot_select_credential_or_endpoint(controller, monkeypatch):
instance, responses, _ = handler(controller, monkeypatch, {'reason': 'configure stats', 'token': 'caller-credential'})
instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH
monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration'))
instance.do_POST()
assert responses == [(400, {'status': 'invalid_request'})]
backend/tests/test_runner_statistics_binding.py
+391
View File
@@ -0,0 +1,391 @@
from __future__ import annotations
import importlib.util
import json
import os
import subprocess
import tempfile
from pathlib import Path
import pytest
pytestmark = pytest.mark.skipif(os.name != 'posix', reason='Linux controller ownership/flock contract')
@pytest.fixture
def binding(tmp_path, monkeypatch):
path = Path(__file__).resolve().parents[2] / 'ops/gitea-deploy-broker/runner_statistics_binding.py'
spec = importlib.util.spec_from_file_location('binding_test', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
anchor = tmp_path / 'controller-anchor'
state = anchor / 'state'
(state / 'locks').mkdir(parents=True)
source = tmp_path / 'brain'
source.mkdir()
data = tmp_path / 'appops'
data.mkdir()
monkeypatch.setattr(module, 'ROOT_UID', os.getuid())
if os.geteuid() != 0:
# The production controller is root. Keep portable unit coverage for
# non-root Linux runs while root-only tests assert 100:101.
monkeypatch.setattr(module, 'APPOPS_UID', os.geteuid())
monkeypatch.setattr(module, 'APPOPS_GID', os.getegid())
for key, value in {'STATE_ROOT': state, 'BINDING_ROOT': state / 'runner-statistics-binding',
'PB_SOURCE': source, 'PB_ENV': source / '.env', 'APPOPS_DATA': data,
'TOKEN_FILE': data / 'runner-statistics-token',
'ADMIN_TOKEN_FILE': data / 'mcp-token', 'GLOBAL_LOCK': anchor / 'deploy.lock'}.items():
monkeypatch.setattr(module, key, value)
module.PB_ENV.write_bytes(b'EXISTING_PRIVATE_VALUE=keep-me\nPROJECTBRAIN_APPOPS_MCP_URL=\nPROJECTBRAIN_APPOPS_MCP_TOKEN=\n')
module.PB_ENV.chmod(0o600)
module.ADMIN_TOKEN_FILE.write_text('admin-capability-separate-secret\n')
module.ADMIN_TOKEN_FILE.chmod(0o600)
module.project = {'jens/projectbrain': {'branch': 'main', 'slug': 'projectbrain',
'source_dir': str(source), 'containers': ['projectbrain'],
'deploy_script': 'scripts/deploy-unraid-single.sh',
'validation': {'required': True, 'contexts': ['Managed validation / full (push)']}}}
return module
def invoke(binding):
return binding.configure(revision='a' * 40, projects=binding.project)
def test_configuration_is_separate_private_and_idempotent(binding):
before = binding.PB_ENV.read_bytes()
result = invoke(binding)
assert result['status'] == 'configured'
assert result['projectbrainDeliveryRequired'] is True
token = binding.TOKEN_FILE.read_text().strip()
assert len(token) == 64
assert token != binding.ADMIN_TOKEN_FILE.read_text().strip()
assert token not in json.dumps(result)
assert binding._configured_values(binding.PB_ENV.read_bytes()) == {
binding.URL_KEY: binding.ENDPOINT, binding.TOKEN_KEY: token}
assert b'EXISTING_PRIVATE_VALUE=keep-me\n' in binding.PB_ENV.read_bytes()
assert (binding.BINDING_ROOT / 'environment-before').read_bytes() == before
for path in (binding.PB_ENV, binding.TOKEN_FILE, binding.BINDING_ROOT / 'receipt.json'):
assert path.stat().st_mode & 0o777 == 0o600
token_info = binding.TOKEN_FILE.stat()
assert (token_info.st_uid, token_info.st_gid) == (binding.APPOPS_UID, binding.APPOPS_GID)
assert token_info.st_nlink == 1
assert invoke(binding)['status'] == 'already_configured'
@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0,
reason='fixed AppOps ownership requires a privileged Linux controller')
def test_statistics_token_is_appops_readable_while_binding_state_stays_root_private(binding):
assert os.geteuid() != 100
result = invoke(binding)
token_info = binding.TOKEN_FILE.stat()
assert (token_info.st_uid, token_info.st_gid) == (100, 101)
assert token_info.st_mode & 0o777 == 0o600
assert token_info.st_nlink == 1
assert len(binding.TOKEN_FILE.read_text().strip()) == 64
assert result['status'] == 'configured'
for path in (binding.PB_ENV, binding.BINDING_ROOT / 'receipt.json',
binding.BINDING_ROOT / 'environment-before'):
info = path.stat()
assert info.st_uid == binding.ROOT_UID
assert info.st_mode & 0o777 == 0o600
assert info.st_nlink == 1
assert invoke(binding)['status'] == 'already_configured'
token_info = binding.TOKEN_FILE.stat()
assert (token_info.st_uid, token_info.st_gid) == (100, 101)
assert token_info.st_mode & 0o777 == 0o600
assert token_info.st_nlink == 1
@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0,
reason='dropping to the fixed AppOps identity requires a privileged Linux controller')
def test_private_secret_reader_accepts_statistics_token_as_dropped_appops_identity(binding):
invoke(binding)
repository = Path(__file__).resolve().parents[2]
code = (
'import sys; from pathlib import Path; '
'sys.path.insert(0, sys.argv[2]); '
'from backend.app.core.config import _private_secret_file; '
'value = _private_secret_file(sys.argv[1]); '
'raise SystemExit(0 if value is not None and len(value) == 64 else 1)'
)
with tempfile.TemporaryDirectory(prefix='appops-statistics-reader-', dir='/tmp') as directory:
os.chmod(directory, 0o711)
token_copy = Path(directory) / 'runner-statistics-token'
token_copy.write_bytes(binding.TOKEN_FILE.read_bytes())
source_info = binding.TOKEN_FILE.stat()
os.chown(token_copy, source_info.st_uid, source_info.st_gid)
token_copy.chmod(source_info.st_mode & 0o777)
def drop_to_appops():
os.setgroups([])
os.setgid(101)
os.setuid(100)
result = subprocess.run(
['/usr/bin/python3', '-c', code, str(token_copy), str(repository)],
cwd=directory, preexec_fn=drop_to_appops,
capture_output=True, text=True, check=False,
)
token = binding.TOKEN_FILE.read_text().strip()
assert token not in result.stdout and token not in result.stderr
assert result.stdout == ''
assert result.returncode == 0, (
'AppOps config private-secret reader rejected the fixed token: '
+ result.stderr[-500:]
)
@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0,
reason='wrong-owner validation requires a privileged Linux controller')
def test_configured_root_owned_statistics_token_fails_closed(binding):
assert invoke(binding)['status'] == 'configured'
token = binding.TOKEN_FILE.read_bytes()
os.chown(binding.TOKEN_FILE, binding.ROOT_UID, os.getgid())
with pytest.raises(binding.StatisticsBindingError, match='file_untrusted'):
invoke(binding)
assert binding.TOKEN_FILE.read_bytes() == token
info = binding.TOKEN_FILE.stat()
assert (info.st_uid, info.st_gid) == (binding.ROOT_UID, os.getgid())
@pytest.mark.parametrize('key', ['source_dir', 'deploy_script', 'containers', 'validation'])
def test_unknown_project_configuration_is_rejected_before_write(binding, key):
binding.project['jens/projectbrain'][key] = 'untrusted'
with pytest.raises(binding.StatisticsBindingError):
invoke(binding)
assert not binding.TOKEN_FILE.exists()
assert not binding.BINDING_ROOT.exists()
def test_existing_operator_configuration_is_preserved(binding):
before = b'PROJECTBRAIN_APPOPS_MCP_URL=https://operator.example/mcp\nOTHER=preserve\n'
binding.PB_ENV.write_bytes(before)
with pytest.raises(binding.StatisticsBindingError, match='existing_operator_configuration'):
invoke(binding)
assert binding.PB_ENV.read_bytes() == before
assert not binding.TOKEN_FILE.exists()
def test_existing_operator_token_is_never_adopted(binding):
binding.TOKEN_FILE.write_text('existing-operator-secret')
binding.TOKEN_FILE.chmod(0o600)
os.chown(binding.TOKEN_FILE, binding.APPOPS_UID, binding.APPOPS_GID)
with pytest.raises(binding.StatisticsBindingError, match='existing_operator_credential'):
invoke(binding)
assert binding.TOKEN_FILE.read_text() == 'existing-operator-secret'
@pytest.mark.parametrize('fault_path', ['token', 'environment', 'receipt'])
def test_crash_after_each_durable_boundary_recovers_same_credential(binding, monkeypatch, fault_path):
atomic = binding._atomic
interrupted = False
def crash(path, value, expected):
nonlocal interrupted
atomic(path, value, expected)
selected = {'token': binding.TOKEN_FILE, 'environment': binding.PB_ENV,
'receipt': binding.BINDING_ROOT / 'receipt.json'}[fault_path]
if path == selected and not interrupted:
interrupted = True
raise OSError('simulated interruption after fsync')
monkeypatch.setattr(binding, '_atomic', crash)
with pytest.raises(OSError):
invoke(binding)
journal = json.loads((binding.BINDING_ROOT / 'receipt.json').read_bytes())
expected = journal['token']
monkeypatch.setattr(binding, '_atomic', atomic)
assert invoke(binding)['status'] == 'configured'
assert binding.TOKEN_FILE.read_text().strip() == expected
@pytest.mark.skipif(os.name != 'posix' or os.geteuid() != 0,
reason='simulated ownership failure requires a privileged Linux controller')
def test_statistics_token_owner_failure_closes_fd_and_never_publishes(binding, monkeypatch, capsys):
opened_token_fds = []
def fail_chown(fd, uid, gid):
opened_token_fds.append(fd)
raise PermissionError('simulated AppOps ownership failure')
monkeypatch.setattr(binding.os, 'fchown', fail_chown)
with pytest.raises(PermissionError):
invoke(binding)
assert len(opened_token_fds) == 1
with pytest.raises(OSError):
binding.os.fstat(opened_token_fds[0])
assert not binding.TOKEN_FILE.exists()
assert [path.name for path in binding.APPOPS_DATA.iterdir()] == ['mcp-token']
assert capsys.readouterr().out == ''
def test_crash_after_backup_before_receipt_has_safe_retry(binding, monkeypatch):
atomic = binding._atomic
def crash(path, value, expected):
atomic(path, value, expected)
if path.name == 'environment-before':
raise OSError('interruption after backup')
monkeypatch.setattr(binding, '_atomic', crash)
with pytest.raises(OSError):
invoke(binding)
monkeypatch.setattr(binding, '_atomic', atomic)
assert invoke(binding)['status'] == 'configured'
def test_pending_environment_cas_conflict_preserves_operator_change(binding, monkeypatch):
atomic = binding._atomic
def crash(path, value, expected):
atomic(path, value, expected)
if path == binding.TOKEN_FILE:
raise OSError('interruption')
monkeypatch.setattr(binding, '_atomic', crash)
with pytest.raises(OSError):
invoke(binding)
binding.PB_ENV.write_bytes(b'OPERATOR_CHANGED=value\n')
monkeypatch.setattr(binding, '_atomic', atomic)
with pytest.raises(binding.StatisticsBindingError, match='environment_cas_conflict'):
invoke(binding)
assert binding.PB_ENV.read_bytes() == b'OPERATOR_CHANGED=value\n'
def test_completed_binding_keeps_unrelated_later_operator_edits(binding):
invoke(binding)
with binding.PB_ENV.open('ab') as stream:
stream.write(b'NEW_OPERATOR_VALUE=preserved\n')
before = binding.PB_ENV.read_bytes()
assert invoke(binding)['status'] == 'already_configured'
assert binding.PB_ENV.read_bytes() == before
@pytest.mark.parametrize('unsafe', ['symlink', 'public_file', 'writable_anchor', 'duplicate_key'])
def test_unsafe_configuration_blocks_before_credential_write(binding, unsafe):
if unsafe == 'symlink':
target = binding.PB_ENV.with_name('other.env')
binding.PB_ENV.rename(target)
binding.PB_ENV.symlink_to(target)
elif unsafe == 'public_file':
binding.PB_ENV.chmod(0o644)
elif unsafe == 'writable_anchor':
binding.PB_SOURCE.chmod(0o777)
else:
binding.PB_ENV.write_text(f'{binding.URL_KEY}=\n{binding.URL_KEY}=\n')
with pytest.raises((binding.StatisticsBindingError, OSError)):
invoke(binding)
assert not binding.TOKEN_FILE.exists()
def test_active_deployment_lock_blocks_without_configuration_changes(binding):
import fcntl
with binding.GLOBAL_LOCK.open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
with pytest.raises(binding.StatisticsBindingError, match='deployment_busy'):
invoke(binding)
assert not binding.TOKEN_FILE.exists()
@pytest.mark.parametrize('owner,accepted', [(100, True), (200, False)])
def test_existing_fixed_admin_file_owner_contract(binding, monkeypatch, owner, accepted):
actual = binding.os.fstat
inode = binding.ADMIN_TOKEN_FILE.stat().st_ino
def metadata(fd):
info = actual(fd)
if info.st_ino == inode:
fields = list(info)
fields[4] = owner
return os.stat_result(fields)
return info
monkeypatch.setattr(binding.os, 'fstat', metadata)
if accepted:
assert invoke(binding)['status'] == 'configured'
else:
with pytest.raises(binding.StatisticsBindingError, match='file_untrusted'):
invoke(binding)
assert not binding.TOKEN_FILE.exists()
def test_parent_swap_cannot_redirect_a_root_read(binding, monkeypatch):
actual_open = binding.os.open
original = binding.PB_ENV.read_bytes()
moved = binding.PB_SOURCE.with_name('moved-brain')
attacker = binding.PB_SOURCE.with_name('attacker')
attacker.mkdir()
(attacker / '.env').write_bytes(b'ATTACKER=must-not-be-read\n')
(attacker / '.env').chmod(0o600)
swapped = False
def swap(path, flags, *args, **kwargs):
nonlocal swapped
if path == '.env' and 'dir_fd' in kwargs and not swapped:
swapped = True
binding.PB_SOURCE.rename(moved)
binding.PB_SOURCE.symlink_to(attacker, target_is_directory=True)
return actual_open(path, flags, *args, **kwargs)
monkeypatch.setattr(binding.os, 'open', swap)
assert binding._read(binding.PB_ENV, binding.MAX_ENV_BYTES) == original
assert (attacker / '.env').read_bytes() == b'ATTACKER=must-not-be-read\n'
with pytest.raises(OSError):
binding._read(binding.PB_ENV, binding.MAX_ENV_BYTES)
def test_parent_swap_cannot_redirect_atomic_replace(binding, monkeypatch):
actual_replace = binding.os.replace
before = binding.PB_ENV.read_bytes()
updated = before + b'APPENDED=private\n'
moved = binding.PB_SOURCE.with_name('moved-brain')
attacker = binding.PB_SOURCE.with_name('attacker')
attacker.mkdir()
(attacker / '.env').write_bytes(b'ATTACKER=must-not-be-overwritten\n')
(attacker / '.env').chmod(0o600)
def swap(source, destination, **kwargs):
assert source.startswith('.statistics-')
assert destination == '.env'
assert kwargs['src_dir_fd'] == kwargs['dst_dir_fd']
binding.PB_SOURCE.rename(moved)
binding.PB_SOURCE.symlink_to(attacker, target_is_directory=True)
return actual_replace(source, destination, **kwargs)
monkeypatch.setattr(binding.os, 'replace', swap)
binding._atomic(binding.PB_ENV, updated, before)
assert (moved / '.env').read_bytes() == updated
assert (attacker / '.env').read_bytes() == b'ATTACKER=must-not-be-overwritten\n'
with pytest.raises(OSError):
binding._read(binding.PB_ENV, binding.MAX_ENV_BYTES)
def test_parent_swap_cannot_replace_the_authentication_secret(binding, monkeypatch):
secret = b'original-administration-capability-1234567890'
binding.ADMIN_TOKEN_FILE.write_bytes(secret)
actual_open = binding.os.open
moved = binding.APPOPS_DATA.with_name('moved-appops')
attacker = binding.APPOPS_DATA.with_name('attacker-appops')
attacker.mkdir()
(attacker / 'mcp-token').write_bytes(b'attacker-chosen-authentication-capability-12345')
(attacker / 'mcp-token').chmod(0o600)
swapped = False
def swap(path, flags, *args, **kwargs):
nonlocal swapped
if path == 'mcp-token' and 'dir_fd' in kwargs and not swapped:
swapped = True
binding.APPOPS_DATA.rename(moved)
binding.APPOPS_DATA.symlink_to(attacker, target_is_directory=True)
return actual_open(path, flags, *args, **kwargs)
monkeypatch.setattr(binding.os, 'open', swap)
assert binding.read_administration_secret() == secret
assert binding.read_administration_secret() is None
backend/tests/test_runner_statistics_mcp.py
+301
View File
@@ -0,0 +1,301 @@
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from httpx import ASGITransport, AsyncClient
from mcp import Client
from backend.app.core.config import Settings
from backend.app.core.security import require_service_request
from backend.app.runner_statistics_mcp import RUNNER_STATISTICS_TOOL, create_runner_statistics_mcp
STATISTICS_TOKEN = "runner-statistics-token-that-is-distinct-and-long"
GOVERNED_MCP_TOKEN = "governed-operations-token-that-is-distinct-and-long"
def _request(authorization: str | None = None):
from starlette.requests import Request
headers = [(b"authorization", authorization.encode("ascii"))] if authorization else []
return Request(
{
"type": "http",
"method": "POST",
"path": "/mcp-runner-statistics/",
"headers": headers,
"client": ("127.0.0.1", 12345),
}
)
def _snapshot() -> dict[str, object]:
return {
"schemaVersion": 1,
"status": "healthy",
"scope": "ci-fleet",
"generatedAt": "2026-10-01T12:00:00+00:00",
"budget": {
"cpuLimit": 8,
"memoryLimitBytes": 17_179_869_184,
"activeJobLimit": 2,
"heavyJobLimit": 1,
"reservedCpu": 1,
"reservedMemoryBytes": 2_147_483_648,
"activeJobs": 0,
"heavyJobs": 0,
},
"host": {
"cpuBusyPercent": None,
"loadRatio": None,
"availableMemoryBytes": None,
"cacheFreeBytes": None,
},
"profiles": [],
"window": {
"startAt": None,
"endAt": None,
"completedJobs": 0,
"samples": 0,
"coverage": "unavailable",
},
"history": [],
}
def test_separate_mcp_catalog_and_tool_are_read_only():
server, _http_app = create_runner_statistics_mcp(
snapshot_provider=_snapshot,
allowed_hosts=["testserver"],
allowed_origins=[],
)
async def exercise() -> None:
async with Client(server) as client:
listing = await client.list_tools()
assert [tool.name for tool in listing.tools] == [RUNNER_STATISTICS_TOOL]
tool = listing.tools[0]
assert tool.annotations.read_only_hint is True
assert tool.annotations.destructive_hint is False
result = await client.call_tool(RUNNER_STATISTICS_TOOL, {})
assert result.structured_content["scope"] == "ci-fleet"
unknown = await client.call_tool("container.restart", {"target": "anything"})
assert unknown.is_error is True
asyncio.run(exercise())
def test_dedicated_service_auth_requires_strong_distinct_token(monkeypatch):
from backend.app.core import security
from backend.app.core.config import settings
monkeypatch.setattr(security, "settings", settings)
monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: STATISTICS_TOKEN)
monkeypatch.setattr(settings, "mcp_token", GOVERNED_MCP_TOKEN)
assert require_service_request(_request(f"Bearer {STATISTICS_TOKEN}"), "mcp_runner_statistics") == "mcp_runner_statistics"
with pytest.raises(HTTPException) as wrong:
require_service_request(_request(f"Bearer {GOVERNED_MCP_TOKEN}"), "mcp_runner_statistics")
assert wrong.value.status_code == 403
monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: GOVERNED_MCP_TOKEN)
with pytest.raises(HTTPException) as collision:
require_service_request(_request(f"Bearer {GOVERNED_MCP_TOKEN}"), "mcp_runner_statistics")
assert collision.value.status_code == 503
monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: "short")
with pytest.raises(HTTPException) as weak:
require_service_request(_request("Bearer short"), "mcp_runner_statistics")
assert weak.value.status_code == 503
def test_statistics_token_file_is_private_and_uses_data_dir_default(tmp_path: Path, monkeypatch):
monkeypatch.delenv("APPOPS_RUNNER_STATISTICS_TOKEN_FILE", raising=False)
monkeypatch.delenv("APPOPS_RUNNER_STATISTICS_TOKEN", raising=False)
token_path = tmp_path / "runner-statistics-token"
token_path.write_text(STATISTICS_TOKEN, encoding="utf-8")
if __import__("os").name != "nt":
token_path.chmod(0o600)
configured = Settings(data_dir=str(tmp_path), mcp_token=GOVERNED_MCP_TOKEN)
assert configured.runner_statistics_token_file == str(token_path)
assert configured.read_runner_statistics_token() == STATISTICS_TOKEN
assert STATISTICS_TOKEN not in str(configured.redact_env_preview())
def test_statistics_token_reader_rejects_oversized_and_symlink_files(tmp_path: Path):
from backend.app.core.config import _private_secret_file
oversized = tmp_path / "oversized-token"
oversized.write_text("x" * 513, encoding="utf-8")
assert _private_secret_file(str(oversized)) is None
target = tmp_path / "target-token"
target.write_text(STATISTICS_TOKEN, encoding="utf-8")
if __import__("os").name != "nt":
target.chmod(0o600)
linked = tmp_path / "linked-token"
try:
linked.symlink_to(target)
except (OSError, NotImplementedError):
pytest.skip("symlink creation is unavailable on this host")
assert _private_secret_file(str(linked)) is None
def test_token_file_can_be_provisioned_after_app_start_without_restart(tmp_path: Path, monkeypatch):
monkeypatch.delenv("APPOPS_RUNNER_STATISTICS_TOKEN_FILE", raising=False)
token_path = tmp_path / "runner-statistics-token"
configured = Settings(data_dir=str(tmp_path), mcp_token=GOVERNED_MCP_TOKEN)
assert configured.read_runner_statistics_token() is None
token_path.write_text(STATISTICS_TOKEN, encoding="utf-8")
if __import__("os").name != "nt":
token_path.chmod(0o600)
assert configured.read_runner_statistics_token() == STATISTICS_TOKEN
def test_plaintext_environment_value_is_not_a_runner_statistics_credential(tmp_path: Path, monkeypatch):
monkeypatch.setenv("APPOPS_RUNNER_STATISTICS_TOKEN", STATISTICS_TOKEN)
monkeypatch.setenv("APPOPS_RUNNER_STATISTICS_TOKEN_FILE", str(tmp_path / "missing-token"))
configured = Settings(data_dir=str(tmp_path), mcp_token=GOVERNED_MCP_TOKEN)
assert configured.read_runner_statistics_token() is None
def test_readonly_transport_adds_only_validated_auth_public_host(monkeypatch):
from backend.app.core.config import settings
from backend.app.runner_statistics_mcp import _readonly_allowed_hosts, _readonly_allowed_origins
monkeypatch.setattr(settings, "mcp_allowed_hosts", ["localhost:*"])
monkeypatch.setattr(settings, "auth_public_url", "https://appops.example.test")
monkeypatch.setattr(settings, "frontend_origin", "https://frontend.example.test")
assert _readonly_allowed_hosts() == ["localhost:*", "appops.example.test"]
assert _readonly_allowed_origins() == ["https://frontend.example.test", "https://appops.example.test"]
assert settings.mcp_allowed_hosts == ["localhost:*"]
monkeypatch.setattr(settings, "auth_public_url", "http://attacker.example.test")
assert _readonly_allowed_hosts() == ["localhost:*"]
assert _readonly_allowed_origins() == ["https://frontend.example.test"]
monkeypatch.setattr(settings, "auth_public_url", "https://attacker_host.example.test")
assert _readonly_allowed_hosts() == ["localhost:*"]
def test_transport_factory_uses_readonly_hosts_without_widening_full_mcp(monkeypatch):
from backend.app.core.config import settings
from mcp.server import MCPServer
from starlette.applications import Starlette
captured: dict[str, object] = {}
def capture_transport(self, **kwargs):
captured.update(kwargs)
return Starlette()
monkeypatch.setattr(MCPServer, "streamable_http_app", capture_transport)
monkeypatch.setattr(settings, "mcp_allowed_hosts", ["localhost:*"])
monkeypatch.setattr(settings, "auth_public_url", "https://appops.example.test")
create_runner_statistics_mcp()
transport_security = captured["transport_security"]
assert transport_security.allowed_hosts == ["localhost:*", "appops.example.test"]
assert settings.mcp_allowed_hosts == ["localhost:*"]
def test_readonly_transport_preserves_explicit_port_and_ipv6(monkeypatch):
from backend.app.core.config import settings
from backend.app.runner_statistics_mcp import _readonly_allowed_hosts
monkeypatch.setattr(settings, "mcp_allowed_hosts", [])
monkeypatch.setattr(settings, "auth_public_url", "https://[2001:db8::1]:9443")
assert _readonly_allowed_hosts() == ["[2001:db8::1]:9443"]
def test_main_http_boundary_rejects_cross_tokens_and_hides_missing_endpoint(monkeypatch):
from backend.app import main
from backend.app.core.config import settings
def no_session_for_machine_token(_request):
raise HTTPException(401, "Sign in with Authentik or Microsoft Entra")
monkeypatch.setattr(main, "authorize_request", no_session_for_machine_token)
monkeypatch.setattr(settings, "mcp_enabled", True)
monkeypatch.setattr(settings, "mcp_token", GOVERNED_MCP_TOKEN)
monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: STATISTICS_TOKEN)
monkeypatch.setattr(settings, "mcp_allowed_hosts", ["testserver"])
async def exercise() -> None:
transport = ASGITransport(app=main.app)
async with AsyncClient(transport=transport, base_url="http://testserver") as client:
stats_with_main_token = await client.post(
"/mcp-runner-statistics/",
headers={"Authorization": f"Bearer {GOVERNED_MCP_TOKEN}"},
json={},
)
main_with_stats_token = await client.post(
"/mcp/",
headers={"Authorization": f"Bearer {STATISTICS_TOKEN}"},
json={},
)
native_api_with_stats_token = await client.get(
"/api",
headers={"Authorization": f"Bearer {STATISTICS_TOKEN}"},
)
assert stats_with_main_token.status_code == 403
assert main_with_stats_token.status_code == 403
assert native_api_with_stats_token.status_code == 401
monkeypatch.setattr(settings, "read_runner_statistics_token", lambda: None)
disabled = await client.post("/mcp-runner-statistics/", json={})
assert disabled.status_code == 404
asyncio.run(exercise())
@pytest.mark.parametrize(
("main_enabled", "expected"),
[(False, ["statistics"]), (True, ["main", "statistics"])],
)
def test_lifespan_always_runs_readonly_mcp_and_gates_governed_mcp(monkeypatch, main_enabled, expected):
from backend.app import main
from backend.app.core.config import settings
entered: list[str] = []
@asynccontextmanager
async def run_manager(name: str):
entered.append(name)
try:
yield
finally:
entered.remove(name)
monkeypatch.setattr(main, "appops_mcp", SimpleNamespace(session_manager=SimpleNamespace(run=lambda: run_manager("main"))))
monkeypatch.setattr(main, "run_runner_statistics_mcp", lambda: run_manager("statistics"))
monkeypatch.setattr(main, "init_db", lambda: None)
monkeypatch.setattr(main, "Session", lambda _engine: SimpleNamespace(__enter__=lambda self: self, __exit__=lambda *_args: False))
monkeypatch.setattr(main, "restore_request_metrics", lambda _session: None)
monkeypatch.setattr(main, "restore_tool_metrics", lambda _session: None)
monkeypatch.setattr(main.knowledge_service, "ensure_index_current", lambda _session: None)
monkeypatch.setattr(main, "persist_observability", lambda: None)
monkeypatch.setattr(main, "set_session_running", lambda _value: None)
monkeypatch.setattr(settings, "mcp_enabled", main_enabled)
for field in (
"notification_interval_seconds",
"connector_health_interval_seconds",
"observability_persist_interval_seconds",
"restore_drill_interval_seconds",
"mcp_canary_interval_seconds",
"discovery_interval_seconds",
):
monkeypatch.setattr(settings, field, 0)
async def exercise() -> None:
async with main.lifespan(main.app):
assert entered == expected
asyncio.run(exercise())
backend/tests/test_runner_statistics_surface.py
+234
View File
@@ -0,0 +1,234 @@
from __future__ import annotations
import copy
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
import httpx
import pytest
from sqlmodel import Session, SQLModel, create_engine, select
from backend.app.db.models import AuditEvent, JobRun
from backend.app.services import action_execution, deployment_broker, runner_statistics
def snapshot():
at = datetime.now(UTC).isoformat()
return {
"schemaVersion": 1, "status": "healthy", "scope": "ci-fleet", "generatedAt": at,
"budget": {"cpuLimit": 8, "memoryLimitBytes": 16 * 1024**3, "activeJobLimit": 2,
"heavyJobLimit": 1, "reservedCpu": 2, "reservedMemoryBytes": 3 * 1024**3,
"activeJobs": 0, "heavyJobs": 0},
"host": {"cpuBusyPercent": None, "loadRatio": None,
"availableMemoryBytes": None, "cacheFreeBytes": None},
"profiles": [{"id": "checks", "displayName": "ITWorx CI · Checks",
"runnerName": "itworx-ci-checks", "labels": ["itworx-ci-checks"],
"status": "capacity_wait", "reason": "host_headroom",
"cpuLimit": 1, "memoryLimitBytes": 1536 * 1024**2,
"queueDepth": 1, "activeJobs": 0, "cpuUsed": None,
"memoryUsedBytes": None, "oldestQueuedSeconds": 12,
"completedJobs": 0, "p50WaitSeconds": None, "p95WaitSeconds": None,
"p50DurationSeconds": None, "p95DurationSeconds": None}],
"window": {"startAt": at, "endAt": at, "completedJobs": 0, "samples": 1,
"coverage": "bounded"},
"history": [{"at": at, "activeJobs": 0, "queuedJobs": 1,
"cpuUsed": None, "memoryUsedBytes": None}],
}
@pytest.fixture(autouse=True)
def reset_cache(monkeypatch):
monkeypatch.setattr(runner_statistics, "_cached", None)
monkeypatch.setattr(runner_statistics, "_cached_at", 0.0)
def test_unknown_measurements_survive_and_private_fields_are_removed():
payload = snapshot()
payload["rawJobs"] = [{"secret": "fixture-secret"}]
payload["profiles"][0]["workspacePath"] = "/private/credentials"
result = runner_statistics.sanitize_statistics(payload)
assert result["status"] == "healthy"
assert result["profiles"][0]["cpuUsed"] is None
assert result["profiles"][0]["queueDepth"] == 1
assert "rawJobs" not in result
assert "workspacePath" not in result["profiles"][0]
def test_unattested_profile_limits_remain_unknown():
payload = snapshot()
payload["status"] = "degraded"
payload["profiles"][0].update(cpuLimit=None, memoryLimitBytes=None)
result = runner_statistics.sanitize_statistics(payload)
assert result["status"] == "degraded"
assert result["profiles"][0]["cpuLimit"] is None
assert result["profiles"][0]["memoryLimitBytes"] is None
@pytest.mark.parametrize("mutation", [
lambda p: p.update(schemaVersion=2),
lambda p: p.update(generatedAt="2026-10-01T12:00:00"),
lambda p: p.update(generatedAt=(datetime.now(UTC) + timedelta(minutes=1)).isoformat()),
lambda p: p["profiles"].append(copy.deepcopy(p["profiles"][0])),
lambda p: p["profiles"][0].update(cpuUsed=float("nan")),
lambda p: p["profiles"][0].update(activeJobs=True),
lambda p: p["profiles"][0].update(runnerName="secret-token"),
lambda p: p["profiles"][0].update(labels=["security-validation"]),
lambda p: p["profiles"][0].update(reason="Bearer fixture-secret-token"),
lambda p: p["window"].update(startAt=None),
lambda p: p["history"].append(copy.deepcopy(p["history"][0])),
lambda p: p.update(history=[p["history"][0]] * 289),
])
def test_untrusted_or_ambiguous_statistics_fail_to_honest_unknown(mutation):
payload = snapshot()
mutation(payload)
result = runner_statistics.sanitize_statistics(payload)
assert result["status"] == "unavailable"
assert result["budget"]["cpuLimit"] is None
assert result["profiles"] == []
def test_old_sample_is_degraded_without_retimestamping():
payload = snapshot()
old = (datetime.now(UTC) - timedelta(minutes=2)).isoformat()
payload["generatedAt"] = old
payload["history"][0]["at"] = old
payload["window"].update(startAt=old, endAt=old)
result = runner_statistics.sanitize_statistics(payload)
assert result["status"] == "degraded"
assert result["generatedAt"] == old
def test_singleflight_cache_and_caller_mutation_isolation(monkeypatch):
calls = []
class Broker:
def is_configured(self):
return True
def runner_fleet_statistics(self):
calls.append(1)
return snapshot()
monkeypatch.setattr(runner_statistics, "DeploymentBrokerClient", Broker)
service = runner_statistics.RunnerStatisticsService()
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: service.snapshot(), range(8)))
assert calls == [1]
results[0]["budget"]["cpuLimit"] = 999
assert service.snapshot()["budget"]["cpuLimit"] == 8
@pytest.mark.parametrize("configured,raises,expected", [
(False, False, "not_configured"), (True, True, "unavailable"),
])
def test_unavailable_backend_never_leaks_error_or_invents_budget(monkeypatch, configured, raises, expected):
class Broker:
def is_configured(self):
return configured
def runner_fleet_statistics(self):
assert raises
raise RuntimeError("Bearer fixture-private-secret /private/key")
monkeypatch.setattr(runner_statistics, "DeploymentBrokerClient", Broker)
result = runner_statistics.RunnerStatisticsService().snapshot()
assert result["status"] == expected
assert result["budget"]["reservedCpu"] is None
assert "fixture-private" not in str(result)
def test_broker_statistics_fixed_read_path_has_deadline_and_no_credentials(monkeypatch):
calls = []
real_client = httpx.Client
def handle(request):
calls.append(request)
return httpx.Response(200, json=snapshot())
def client(**kwargs):
assert kwargs == {"timeout": 5, "verify": True, "follow_redirects": False}
return real_client(**kwargs, transport=httpx.MockTransport(handle))
monkeypatch.setattr(deployment_broker.httpx, "Client", client)
broker = deployment_broker.DeploymentBrokerClient(base_url="https://controller.test", verify_tls=True)
assert broker.runner_fleet_statistics()["schemaVersion"] == 1
request = calls[0]
assert request.method == "GET"
assert str(request.url) == "https://controller.test/v1/runner-fleet/statistics"
assert "authorization" not in request.headers
assert not any(key.startswith("x-itworx") for key in request.headers)
assert broker._runner_fleet_read("/v1/arbitrary")["reason"] == "runner_fleet_path_invalid"
assert len(calls) == 1
def test_specialize_broker_caller_cannot_choose_script_revision_or_target(monkeypatch):
calls = []
broker = deployment_broker.DeploymentBrokerClient()
monkeypatch.setattr(broker, "_request", lambda *args, **kwargs: calls.append((args, kwargs)) or {})
broker.runner_fleet_specialize(reason="x" * 1100)
args, kwargs = calls[0]
assert args == ("POST", "/v1/appops/runner-fleet/specialize")
assert kwargs["payload"] == {"reason": "x" * 1000}
@pytest.mark.parametrize("status,success", [("specialized", True), ("already_applied", True),
("upgraded", False), ("failed", False)])
def test_specialize_execution_persists_actual_success_or_failure(tmp_path, monkeypatch, status, success):
engine = create_engine(f"sqlite:///{tmp_path / 'specialize.db'}")
SQLModel.metadata.create_all(engine)
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_fleet_specialize",
lambda self, **kwargs: {"status": status, "revision": "a" * 40})
with Session(engine) as session:
result = action_execution.execute_governed_action(
session, action="runner.fleet.specialize", target="tower/runner-fleet-specialized",
source="chatgpt-mcp", risk="high", reason="approved fleet specialization",
original_policy_decision="allowed",
)
job = session.get(JobRun, result["jobId"])
audit = session.get(AuditEvent, result["auditId"])
assert result["success"] is success
assert job.status == ("succeeded" if success else "failed")
assert audit.result == ("success" if success else "failed")
assert audit.target == "tower/runner-fleet-specialized"
assert ("verified" in result["message"]) is success
def test_wrong_specialization_target_is_blocked_before_job_or_broker(tmp_path, monkeypatch):
engine = create_engine(f"sqlite:///{tmp_path / 'blocked.db'}")
SQLModel.metadata.create_all(engine)
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_fleet_specialize",
lambda *args, **kwargs: pytest.fail("must not call broker"))
with Session(engine) as session:
result = action_execution.execute_governed_action(
session, action="runner.fleet.specialize", target="other/runner",
source="chatgpt-mcp", risk="high", reason="wrong target",
original_policy_decision="allowed",
)
assert result["policyDecision"] == "blocked"
assert session.exec(select(JobRun)).all() == []
@pytest.mark.parametrize('status,success', [('configured', True), ('already_configured', True),
('specialized', False), ('failed', False)])
def test_statistics_configuration_persists_actual_status_without_secret(tmp_path, monkeypatch, status, success):
engine = create_engine(f"sqlite:///{tmp_path / 'binding.db'}")
SQLModel.metadata.create_all(engine)
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, 'runner_statistics_configure',
lambda self, **kwargs: {'status': status, 'revision': 'a' * 40, 'projectbrainDeliveryRequired': True})
with Session(engine) as session:
result = action_execution.execute_governed_action(
session, action='runner.statistics.configure', target='tower/projectbrain-runner-statistics',
source='chatgpt-mcp', risk='high', reason='configure read-only metrics', original_policy_decision='allowed')
assert result['success'] is success
assert session.get(JobRun, result['jobId']).status == ('succeeded' if success else 'failed')
assert session.get(AuditEvent, result['auditId']).target == 'tower/projectbrain-runner-statistics'
assert 'token' not in result
def test_statistics_configuration_broker_has_fixed_signed_body(monkeypatch):
calls = []
broker = deployment_broker.DeploymentBrokerClient()
monkeypatch.setattr(broker, '_request', lambda *args, **kwargs: calls.append((args, kwargs)) or {})
broker.runner_statistics_configure(reason='configure read-only stats')
assert calls[0][0] == ('POST', '/v1/appops/runner-statistics/configure')
assert calls[0][1]['payload'] == {'reason': 'configure read-only stats'}
backend/tests/test_runner_task_image_producer.py
+230
View File
@@ -0,0 +1,230 @@
"""Exercise the real producer script against Docker's inherited-label shape."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tarfile
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
PROFILES = ("checks", "backend", "frontend", "security", "browser", "build")
# Captured from the SHA-verified immutable amd64 base image config c474d435...
BASE_LABELS = {
"io.buildah.version": "1.35.0",
"org.opencontainers.image.authors": "Gitea",
"org.opencontainers.image.created": "2026-10-01T00:01:57Z",
"org.opencontainers.image.description": "Official docker images used by Gitea runner to run workflows.",
"org.opencontainers.image.documentation": "https://gitea.com/gitea/runner-images/src/branch/main/README.md",
"org.opencontainers.image.licenses": "MIT",
"org.opencontainers.image.title": "Official Gitea Actions runner images",
"org.opencontainers.image.url": "https://gitea.com/gitea/runner-images",
"org.opencontainers.image.vendor": "Gitea",
"org.opencontainers.image.version": "v26.10.01",
}
def _run_producer(tmp_path: Path, mutation: str = ""):
repo = tmp_path / "repository"
repo.mkdir()
shutil.copytree(ROOT / "ops" / "runner-task-images", repo / "ops" / "runner-task-images")
code = repo / "ops" / "gitea-deploy-broker"
code.mkdir()
shutil.copyfile(ROOT / "ops" / "gitea-deploy-broker" / "runner_profiles.py", code / "runner_profiles.py")
scripts = repo / "scripts"
scripts.mkdir()
shutil.copyfile(
ROOT / "scripts" / "normalize_ci_task_image_archive.py",
scripts / "normalize_ci_task_image_archive.py",
)
for args in (
["init", "-q"],
["add", "ops", "scripts"],
["-c", "user.name=Producer test", "-c", "user.email=producer-test@localhost", "commit", "-qm", "fixture"],
):
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True)
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], cwd=repo, text=True).strip()
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
docker = fake_bin / "docker"
docker.write_text(
f"#!{sys.executable}\n"
"import hashlib,io,json,os,sys,tarfile\n"
"from pathlib import Path\n"
"a=sys.argv[1:]; p=Path(os.environ['FAKE_DOCKER_ROWS'])\n"
"layer_buffer=io.BytesIO()\n"
"with tarfile.open(fileobj=layer_buffer,mode='w:') as layer_tar:\n"
" payload=b'producer fixture layer'; member=tarfile.TarInfo('fixture.txt'); member.size=len(payload); layer_tar.addfile(member,io.BytesIO(payload))\n"
"layer=layer_buffer.getvalue(); layer_sha=hashlib.sha256(layer).hexdigest()\n"
"if a==['info']: sys.exit(0)\n"
"if a[0]=='build':\n"
" builds=Path(os.environ['FAKE_DOCKER_BUILD_ARGS'])\n"
" argv=json.loads(builds.read_text()) if builds.exists() else []\n"
" argv.append(a); builds.write_text(json.dumps(argv))\n"
" pairs=[a[i+1].split('=',1) for i,x in enumerate(a) if x=='--build-arg']\n"
" b=dict(pairs); profile=b['PROFILE']; ref=a[a.index('--tag')+1]\n"
f" labels={BASE_LABELS!r}.copy()\n"
" labels.update({'org.opencontainers.image.source':'https://gitea.example.com/Jens/Unraid-AppOps-Gateway',"
"'org.opencontainers.image.revision':b['SOURCE_REVISION'],'tech.itworx.source-tree':b['SOURCE_TREE'],'tech.itworx.ci.profile':profile})\n"
" mutation=os.environ.get('FAKE_LABEL_MUTATION','')\n"
" if mutation=='extra': labels['unexpected.label']='unexpected'\n"
" if mutation=='missing-inherited': labels.pop('org.opencontainers.image.vendor')\n"
" if mutation=='changed-inherited': labels['org.opencontainers.image.created']='different'\n"
" if mutation=='wrong-source': labels['org.opencontainers.image.source']='https://example.invalid'\n"
" if mutation=='missing-profile': labels.pop('tech.itworx.ci.profile')\n"
" config={'architecture':'amd64','os':'linux','config':{'User':'ubuntu','Labels':labels},'rootfs':{'type':'layers','diff_ids':['sha256:'+layer_sha]}}\n"
" config_bytes=json.dumps(config,sort_keys=True,separators=(',',':')).encode(); image_id='sha256:'+hashlib.sha256(config_bytes).hexdigest()\n"
" rows=json.loads(p.read_text()) if p.exists() else []\n"
" rows.append({'Id':image_id,'Os':'linux','Architecture':'amd64','RepoTags':[ref],'Config':{'User':'ubuntu','Labels':labels}})\n"
" configs=Path(os.environ['FAKE_DOCKER_CONFIGS']); configs.mkdir(exist_ok=True); (configs/(profile+'.json')).write_bytes(config_bytes)\n"
" p.write_text(json.dumps(rows)); sys.exit(0)\n"
"if a[:2]==['image','inspect']: print(p.read_text()); sys.exit(0)\n"
"if a[:2]==['image','save']:\n"
" selected=set(a[a.index('--output')+2:]); rows=json.loads(p.read_text()); members={'blobs/sha256/'+layer_sha:layer}; manifest=[]\n"
" for row in rows:\n"
" if row['RepoTags'][0] not in selected: continue\n"
" reference=row['RepoTags'][0]; profile=reference.split('itworx-ci-',1)[1].split(':sha-',1)[0]; config=(Path(os.environ['FAKE_DOCKER_CONFIGS'])/(profile+'.json')).read_bytes(); config_sha=hashlib.sha256(config).hexdigest(); config_path='blobs/sha256/'+config_sha; members[config_path]=config\n"
" manifest.append({'Config':config_path,'RepoTags':[reference],'Layers':['blobs/sha256/'+layer_sha],'LayerSources':{'sha256:'+layer_sha:{'mediaType':'application/vnd.oci.image.layer.v1.tar','size':len(layer),'digest':'sha256:'+layer_sha}}})\n"
" members['manifest.json']=json.dumps(manifest,separators=(',',':')).encode(); members['oci-layout']=json.dumps({'imageLayoutVersion':'1.0.0'}).encode(); members['index.json']=json.dumps({'schemaVersion':2,'manifests':[]}).encode()\n"
" with tarfile.open(a[a.index('--output')+1],mode='w:') as archive:\n"
" for name,payload in members.items():\n"
" member=tarfile.TarInfo(name); member.size=len(payload); archive.addfile(member,io.BytesIO(payload))\n"
" sys.exit(0)\n"
"sys.exit('unexpected Docker invocation')\n",
encoding="utf-8",
)
docker.chmod(0o755)
output = tmp_path / "output"
env = {
**os.environ,
"PATH": str(fake_bin) + os.pathsep + os.environ["PATH"],
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
"GITHUB_REPOSITORY": "Jens/Unraid-AppOps-Gateway",
"FAKE_DOCKER_ROWS": str(tmp_path / "docker-rows.json"),
"FAKE_DOCKER_BUILD_ARGS": str(tmp_path / "docker-build-args.json"),
"FAKE_DOCKER_CONFIGS": str(tmp_path / "docker-configs"),
"FAKE_LABEL_MUTATION": mutation,
}
Path(env["FAKE_DOCKER_CONFIGS"]).mkdir()
result = subprocess.run(
["bash", str(ROOT / "scripts" / "build_ci_task_images.sh"), revision, "123", str(output)],
cwd=repo,
env=env,
capture_output=True,
text=True,
timeout=30,
)
return result, output, revision, tree
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
def test_producer_accepts_exact_pinned_base_metadata_and_keeps_provenance_manifest(tmp_path):
result, output, revision, tree = _run_producer(tmp_path)
assert result.returncode == 0, result.stderr
build_args = json.loads((tmp_path / "docker-build-args.json").read_text())
assert len(build_args) == len(PROFILES)
expected_limits = {
"--cpu-period=100000",
"--cpu-quota=300000",
"--memory=5120m",
"--memory-swap=5120m",
"--cgroup-parent=docker",
"--rm=true",
"--force-rm=true",
}
profiles = [
next(
args[index + 1]
for index, arg in enumerate(args[:-1])
if arg == "--build-arg" and args[index + 1].startswith("PROFILE=")
)
for args in build_args
]
assert profiles == [f"PROFILE={profile}" for profile in PROFILES]
for args in build_args:
resource_flags = [
arg
for arg in args
if arg.startswith(
(
"--cpu-period",
"--cpu-quota",
"--memory",
"--cgroup-parent",
"--rm",
"--force-rm",
"--cpus",
)
)
]
assert set(resource_flags) == expected_limits
assert len(resource_flags) == len(expected_limits)
manifest = json.loads((output / "task-image-manifest.json").read_text())
assert manifest["sourceRevision"] == revision
assert manifest["sourceTree"] == tree
assert tuple(row["profile"] for row in manifest["images"]) == PROFILES
for row in manifest["images"]:
assert row["labels"] == {
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
"org.opencontainers.image.revision": revision,
"tech.itworx.source-tree": tree,
"tech.itworx.ci.profile": row["profile"],
}
archive_path = output / "task-images.tar"
archive_bytes = archive_path.read_bytes()
assert manifest["dockerTarSha256"] == hashlib.sha256(archive_bytes).hexdigest()
assert manifest["dockerTarBytes"] == len(archive_bytes)
inspected = {
image["RepoTags"][0]: image
for image in json.loads((tmp_path / "docker-rows.json").read_text())
}
with tarfile.open(archive_path, mode="r:") as archive:
members = {member.name: member for member in archive}
normalized = json.loads(archive.extractfile(members["manifest.json"]).read())
assert len(normalized) == len(PROFILES)
assert all(set(item) == {"Config", "RepoTags", "Layers"} for item in normalized)
for item, row in zip(normalized, manifest["images"], strict=True):
assert item["RepoTags"] == [row["reference"]]
assert item["Config"] == row["imageId"].removeprefix("sha256:") + ".json"
assert len(item["Layers"]) == 1
image = inspected[row["reference"]]
config_bytes = archive.extractfile(members[item["Config"]]).read()
config = json.loads(config_bytes)
assert "sha256:" + hashlib.sha256(config_bytes).hexdigest() == image["Id"] == row["imageId"]
assert config["config"]["User"] == "ubuntu"
assert config["config"]["Labels"] == image["Config"]["Labels"] == {
**BASE_LABELS,
**row["labels"],
}
assert config["rootfs"]["type"] == "layers"
assert len(config["rootfs"]["diff_ids"]) == 1
diff_id = config["rootfs"]["diff_ids"][0]
layer_name = item["Layers"][0]
assert layer_name == diff_id.removeprefix("sha256:") + "/layer.tar"
layer_bytes = archive.extractfile(members[layer_name]).read()
assert "sha256:" + hashlib.sha256(layer_bytes).hexdigest() == diff_id
assert not any(name.startswith("blobs/") for name in members)
assert not (output / ".task-images.raw.tar").exists()
assert not (output / "images.inspect.json").exists()
@pytest.mark.skipif(os.name == "nt", reason="producer is a POSIX shell script")
@pytest.mark.parametrize("mutation", ("extra", "missing-inherited", "changed-inherited", "wrong-source", "missing-profile"))
def test_producer_rejects_any_untrusted_image_label_change(tmp_path, mutation):
result, output, _revision, _tree = _run_producer(tmp_path, mutation)
assert result.returncode != 0
expected_error = (
"task image archive normalization failed: image inspection provenance invalid"
if mutation in {"wrong-source", "missing-profile"}
else "image identity verification failed"
)
assert expected_error in result.stderr
assert not (output / "task-image-manifest.json").exists()