"""Journaled legacy cutover through normal broker functions; fixed two targets only. No provenance is written for the original runtime. Recovery retains every data clone and original image/container. Completed migrations are never automatically rolled back: new writes may have reached candidate data after commit. """ from contextlib import ExitStack import json import os from pathlib import Path import secrets import shutil import time from urllib.request import urlopen from execute import (JOURNAL_ROOT, assert_original, checked, docker_json_template, mount_inventory, original_identity, require_real_child, verify_image) from plan import ATTEMPT, SHA, TARGETS, build_plan from storage import clone_cold, digest, dotenv, private_json, private_write, tree_manifest STEPS = ("stopping", "copying", "renaming", "deploying", "verifying", "committing") def run_transaction(ops): """Write intent before every side effect, allowing recovery after abrupt exit.""" try: for phase in STEPS: ops.save(phase) getattr(ops, phase)() ops.save("committed") except Exception: try: ops.recover() ops.save("rolled_back") except Exception: ops.save("recovery_failed") raise RuntimeError("Migration failed and rollback requires explicit recovery") from None raise RuntimeError("Migration failed; original runtime restored and candidate data retained") from None # Outside rollback scope: releasing workers permits new external effects/data. ops.activate() def recover_transaction(ops): if ops.journal["phase"] == "committed": ops.verify_committed() ops.activate() return "committed" if ops.journal["phase"] == "rolled_back": ops.original(healthy=True) ops.http_health() return "rolled_back" ops.recover() ops.save("rolled_back") return "rolled_back" def inspect_fields(runner, target, expressions): template = docker_json_template(expressions) return [json.loads(line) for line in runner(["docker", "inspect", "--format", template, target]).splitlines()] def mounts(runner, target): raw = inspect_fields(runner, target, [".Mounts"])[0] return sorted(({"type": item["Type"], "source": item["Source"], "destination": item["Destination"], "rw": item["RW"]} for item in raw), key=lambda item: item["destination"]) def candidate_project(original, prepared, plan, nonce, env_path): slug = plan["repository"].split("/")[1] by_target = {item["destination"]: item["source"] for item in plan["candidateMounts"]} names = ({"MIGRATION_LOCAL_ROOT": "/app/local"} if slug == "vacatureradar" else { "MIGRATION_CONFIG_ROOT": "/config", "MIGRATION_OPENRGB_ROOT": "/config/openrgb", "MIGRATION_LUMA_CONFIG_ROOT": "/config/lumaops", "MIGRATION_DATA_ROOT": "/data", "MIGRATION_LOGS_ROOT": "/logs"}) environment = {**dict(original.get("environment", {})), **{key: by_target[target] for key, target in names.items()}, "MIGRATION_ATTEMPT": nonce, "MIGRATION_IMAGE": f"gitea-managed/{slug}:{{revision}}", "MIGRATION_NETWORK": slug + "_default", "MIGRATION_SOURCE_REVISION": "{revision}", "MIGRATION_SOURCE_TREE": "{tree}", "MIGRATION_BUILD_ID": "{build_id}", "MIGRATION_BUILD_DATE": "{build_date}", "MIGRATION_ENV_FILE": str(env_path), "VACATURERADAR_ENV_FILE": str(env_path)} result = dict(original) for key in ("deploy_script", "archive_deploy_script", "manual", "clean_untracked_paths", "legacy_containers"): result.pop(key, None) result.update(source_dir=prepared["source"], compose_project=plan["candidateComposeProject"], exact_source=True, verify_oci_revision=True, env_source=str(env_path), environment=environment, compose_files=[*original["compose_files"], "docker-compose.migration.yml"]) return result def render_contract(broker, project, prepared, plan, runner): cwd, command = broker.compose_command(project) environment = broker.deployment_environment(project, prepared["revision"], plan["repository"], prepared["metadata"]) rendered = json.loads(runner(command + ["config", "--format", "json"], cwd=cwd, env=environment)) services = rendered.get("services", {}) if len(services) != 1: raise ValueError("Migration supports exactly one reviewed application container") service = next(iter(services.values())) if service.get("container_name") != plan["originalName"]: raise ValueError("Candidate canonical name differs from the plan") network = plan["repository"].split("/")[1] + "_default" original_networks = inspect_fields(runner, prepared["originalContainerId"], [".NetworkSettings.Networks"])[0] if set(original_networks) != {network} or service.get("network_mode") != network: raise ValueError("Candidate network must exactly preserve the reviewed original network") if prepared.get("originalNetwork") not in {None, network}: raise ValueError("Original network changed since preparation") host = inspect_fields(runner, prepared["originalContainerId"], [".HostConfig"])[0] validate_runtime_safety(host, service) expected = {item["destination"]: item["source"] for item in plan["candidateMounts"]} observed = {} for item in service.get("volumes", []): if item.get("type") != "bind" or item.get("read_only") or item.get("bind", {}).get("create_host_path") is not False: raise ValueError("Candidate contains an unreviewed writable mount") observed[item["target"]] = item["source"] if observed != expected or len(observed) != len(service.get("volumes", [])): raise ValueError("Rendered candidate mounts do not exactly match cold clones") bindings = inspect_fields(runner, prepared["originalContainerId"], [".HostConfig.PortBindings"])[0] old_ports = sorted((port, item.get("HostIp") or "0.0.0.0", str(item["HostPort"])) for port, items in bindings.items() for item in items or []) new_ports = sorted((f"{item['target']}/{item.get('protocol', 'tcp')}", item.get("host_ip") or "0.0.0.0", str(item["published"])) for item in service.get("ports", [])) if old_ports != new_ports: raise ValueError("Candidate published ports differ from the original runtime") if service.get("labels", {}).get("io.itworx.migration-attempt") != environment["MIGRATION_ATTEMPT"]: raise ValueError("Candidate ownership correlation label is absent") return cwd, command, environment def validate_runtime_safety(host, service): """Compare nonsecret hardware/security bindings; unsupported shapes fail closed.""" old_devices = sorted((item["PathOnHost"], item["PathInContainer"], item["CgroupPermissions"]) for item in host.get("Devices") or []) new_devices = sorted((item["source"], item["target"], item.get("permissions", "rwm")) for item in service.get("devices", [])) if old_devices != new_devices or host.get("DeviceRequests") or host.get("DeviceCgroupRules"): raise ValueError("Candidate device contract differs or is unsupported") for actual, desired in (("CapAdd", "cap_add"), ("CapDrop", "cap_drop")): normalize = lambda values: sorted(value.upper().removeprefix("CAP_") for value in values or []) if normalize(host.get(actual)) != normalize(service.get(desired)): raise ValueError("Candidate capability contract differs") for actual, desired in (("ReadonlyRootfs", "read_only"), ("Privileged", "privileged")): if bool(host.get(actual)) != bool(service.get(desired)): raise ValueError("Candidate filesystem/privilege contract differs") if sorted(host.get("SecurityOpt") or []) != sorted(service.get("security_opt") or []): raise ValueError("Candidate security options differ") new_tmpfs = {} for item in service.get("tmpfs") or []: target, separator, options = item.partition(":") if target in new_tmpfs: raise ValueError("Duplicate candidate tmpfs target") new_tmpfs[target] = options if separator else "" if (host.get("Tmpfs") or {}) != new_tmpfs: raise ValueError("Candidate tmpfs contract differs") def prepare_compose(broker, slug, revision, attempt, original, prepared, workspace, runner): if slug == "vacatureradar": source = Path(prepared["source"]) required = { "scripts/migration_worker_gate.py": (".migration-worker-hold", "while check():"), "deployment/unraid/supervisord.conf": ("migration_worker_gate.py worker", "migration_worker_gate.py scheduler"), "config/settings.py": ("apps.core.middleware.MigrationMaintenanceMiddleware",), "apps/core/middleware.py": ("class MigrationMaintenanceMiddleware", "migration_maintenance"), } for filename, fragments in required.items(): content = (source / filename).read_text() if any(fragment not in content for fragment in fragments): raise ValueError("Exact source lacks the reviewed maintenance contract") observed = {"name": TARGETS[slug][0], **original_identity(prepared["originalContainerId"], runner), "mounts": prepared["originalMounts"]} plan = build_plan(slug, revision, attempt, observed) if Path(plan["workspace"]).resolve() != workspace: raise ValueError("Plan workspace differs from the protected preparation directory") nonce = secrets.token_hex(16) env_path = workspace / "runtime.env" entries = inspect_fields(runner, prepared["originalContainerId"], [".Config.Env"])[0] private_write(env_path, dotenv(entries)) prepared["originalNetwork"] = slug + "_default" candidate = candidate_project(original, prepared, plan, nonce, env_path) repo = broker.repository_path(plan["repository"], original) broker.validate_project_config(repo, candidate, revision) cwd, command, environment = render_contract(broker, candidate, prepared, plan, runner) return candidate, command + ["build"], cwd, environment, plan, nonce class Production: def __init__(self, broker, slug, prepared, workspace, journal, runner=checked): self.broker, self.slug, self.prepared = broker, slug, prepared self.workspace, self.journal, self.runner = workspace, journal, runner self.repository = f"jens/{slug}" self.plan = journal["plan"] self.project = dict(journal["candidateProject"]) self.project.update(_repository_name=self.repository, _repository_path=journal["repositoryPath"], _deployment_metadata=prepared["metadata"]) self.state_path = broker.STATE_ROOT / f"{slug}.json" def save(self, phase): self.journal["phase"] = phase private_json(self.workspace / "cutover.json", self.journal) def canonical_id(self, name): result = self.runner(["docker", "container", "ls", "--all", "--quiet", "--no-trunc", "--filter", f"name=^/{name}$"]).strip().splitlines() if len(result) > 1: raise ValueError("Ambiguous container identity") return result[0] if result else None def original(self, healthy=False): container_id = self.prepared["originalContainerId"] observed = original_identity(container_id, self.runner) if observed["imageId"] != self.prepared["originalImageId"] or observed["composeProject"] != self.slug: raise ValueError("Original immutable runtime identity changed") if mounts(self.runner, container_id) != self.journal["originalMounts"]: raise ValueError("Original mounts changed") networks = inspect_fields(self.runner, container_id, [".NetworkSettings.Networks"])[0] if set(networks) != {self.slug + "_default"}: raise ValueError("Original network changed") if healthy: assert_original(observed, self.slug, container_id, self.prepared["originalImageId"]) return observed def stopping(self): self.original(healthy=True) self.runner(["docker", "stop", "--time", "60", self.prepared["originalContainerId"]], timeout=90) if self.original()["running"]: raise ValueError("Original did not stop") def copying(self): if self.original()["running"]: raise ValueError("Cold copy requires stopped original") checksums = {} for item in self.plan["coldCopyRoots"]: original = Path(item["original"]) if original.is_symlink() or original.resolve() != original: raise ValueError("Unreviewed symlink in original cold-copy root") target = require_real_child(Path(item["candidate"]), self.workspace) checksums[item["original"]] = clone_cold(original, target, self.runner) self.journal["coldCopyChecksums"] = checksums self.save("copying") if self.slug == "vacatureradar": marker = self.hold_path() if marker.exists() or marker.is_symlink(): raise ValueError("Cold clone already contains a maintenance hold") private_write(marker, self.hold_bytes()) def hold_path(self): roots = [item["source"] for item in self.plan["candidateMounts"] if item["destination"] == "/app/local"] if len(roots) != 1: raise ValueError("Maintenance hold requires one exact cloned local root") return require_real_child(Path(roots[0]) / ".migration-worker-hold", self.workspace) def hold_bytes(self): return ("legacy-migration.v1:" + self.journal["nonce"] + "\n").encode() def verify_hold(self): if self.slug == "vacatureradar" and private_read(self.hold_path()) != self.hold_bytes(): raise ValueError("Maintenance hold identity does not match this attempt") def activate(self): if self.journal.get("phase") != "committed": raise ValueError("Activation requires a durable committed transaction") self.verify_committed() self.assert_candidate(self.journal["candidateContainerId"]) if self.slug == "vacatureradar": marker = self.hold_path() if marker.exists() or marker.is_symlink(): self.verify_hold() self.journal["activationIntent"] = True self.save("committed") marker.unlink() descriptor = os.open(marker.parent, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) elif not self.journal.get("activationIntent"): raise ValueError("Maintenance hold disappeared before activation intent") self.journal["activated"] = True self.save("committed") def renaming(self): if self.canonical_id(self.plan["retainedName"]): raise ValueError("Retained container name is already occupied") if self.canonical_id(self.plan["originalName"]) != self.prepared["originalContainerId"]: raise ValueError("Canonical container identity changed") self.runner(["docker", "rename", self.prepared["originalContainerId"], self.plan["retainedName"]]) def deploying(self): self.verify_hold() if self.canonical_id(self.plan["originalName"]): raise ValueError("Canonical name was claimed before candidate deployment") # True first deploy relative to the distinct managed Compose project. if self.broker.capture_compose_rollback(self.project) is not None: raise ValueError("Candidate project is not a first deployment") self.broker.revalidate_deployment_candidate(Path(self.journal["repositoryPath"]), self.project, self.prepared["revision"], self.repository, self.prepared["metadata"]) cwd, command, environment = render_contract(self.broker, self.project, self.prepared, self.plan, self.runner) image_id = verify_image(environment["MIGRATION_IMAGE"], self.prepared["metadata"], self.runner) if image_id != self.prepared["candidateImageId"]: raise ValueError("Prepared image tag changed before candidate start") self.runner(command + ["up", "--detach", "--no-build", "--pull", "never"], cwd=cwd, env=environment, timeout=180) container_id = self.canonical_id(self.plan["originalName"]) if not container_id: raise ValueError("Candidate deployment produced no canonical container") self.assert_candidate(container_id) self.journal["candidateContainerId"] = container_id self.save("deploying") def assert_candidate(self, container_id): values = inspect_fields(self.runner, container_id, [".Id", ".Image", 'index .Config.Labels "com.docker.compose.project"', 'index .Config.Labels "io.itworx.migration-attempt"']) if (values[0] != container_id or container_id == self.prepared["originalContainerId"] or values[2] != self.plan["candidateComposeProject"] or values[3] != self.journal["nonce"]): raise ValueError("Refusing an unrelated candidate container") recorded = self.journal.get("candidateContainerId") if recorded and recorded != container_id: raise ValueError("Recorded candidate container was replaced") def http_health(self): path = "http://127.0.0.1:1226/health/ready/" if self.slug == "vacatureradar" else "http://127.0.0.1:1223/health/ready" with urlopen(path, timeout=10) as response: if response.status != 200 or response.geturl() != path: raise ValueError("Representative readiness probe failed") def verifying(self): self.verify_hold() self.assert_candidate(self.journal["candidateContainerId"]) expected = sorted(self.plan["candidateMounts"], key=lambda item: item["destination"]) if mounts(self.runner, self.journal["candidateContainerId"]) != expected: raise ValueError("Candidate mounts differ from the isolated cold-copy plan") networks = inspect_fields(self.runner, self.journal["candidateContainerId"], [".NetworkSettings.Networks"])[0] if set(networks) != {self.slug + "_default"}: raise ValueError("Candidate runtime network differs from the original") receipt = self.broker.verify_deployment(self.project, self.prepared["metadata"], expected_image_ids={self.plan["originalName"]: self.prepared["candidateImageId"]}) self.broker.validate_recorded_receipt(self.project, receipt, self.repository, self.prepared["revision"]) self.http_health() self.journal["receipt"] = receipt self.save("verifying") def committing(self): self.broker.write_state(self.repository, self.project, self.prepared["revision"], "success", "Controlled legacy migration passed immutable image and readiness verification", live_receipt=self.journal["receipt"], attempt_metadata=self.prepared["metadata"]) # Never alter another project's configuration. Both deployment locks held. current = self.broker.CONFIG_PATH.read_bytes() if digest(current) != self.journal["originalConfigSha256"]: raise ValueError("Broker configuration changed during cutover") private_write(self.broker.CONFIG_PATH, (self.workspace / "candidate-projects.json").read_bytes()) self.verify_committed() def verify_committed(self): if digest(self.broker.CONFIG_PATH.read_bytes()) != self.journal["candidateConfigSha256"]: raise ValueError("Committed broker configuration no longer matches") receipt = self.broker.verified_current_receipt(self.repository, self.project, self.prepared["revision"], self.prepared["metadata"]["tree"]) if receipt is None: raise ValueError("Committed runtime receipt does not reverify") self.http_health() def recover(self): if self.journal.get("phase") == "committed" or self.journal.get("activationIntent"): raise ValueError("Committed activation must never roll back") for filename, key in (("original-projects.json", "originalConfigSha256"), ("original-state.json", "originalStateSha256")): if digest((self.workspace / filename).read_bytes()) != self.journal[key]: raise ValueError("Protected rollback snapshot integrity failed") config = self.broker.CONFIG_PATH.read_bytes() if digest(config) not in {self.journal["originalConfigSha256"], self.journal["candidateConfigSha256"]}: raise ValueError("Unrelated broker configuration change blocks automatic recovery") state = self.state_path.read_bytes() if digest(state) != self.journal["originalStateSha256"]: payload = json.loads(state) if payload.get("live_receipt") != self.journal.get("receipt") or not self.journal.get("receipt"): raise ValueError("Unrelated deployment state blocks automatic recovery") self.original() if not self.journal.get("originalRestartIntent"): for source, expected in self.journal.get("coldCopyChecksums", {}).items(): if digest(json.dumps(tree_manifest(source), sort_keys=True).encode()) != expected: raise ValueError("Original data changed after verified cold copy") container_id = self.canonical_id(self.plan["originalName"]) if container_id and container_id != self.prepared["originalContainerId"]: self.assert_candidate(container_id) # Persist discovered ownership before removing a crash-created container. self.journal["candidateContainerId"] = container_id self.save("recovering") self.runner(["docker", "rm", "--force", container_id], timeout=90) if self.canonical_id(self.plan["originalName"]) not in {None, self.prepared["originalContainerId"]}: raise ValueError("Canonical name is occupied by unrelated runtime") current_name = inspect_fields(self.runner, self.prepared["originalContainerId"], [".Name"])[0].lstrip("/") if current_name not in {self.plan["originalName"], self.plan["retainedName"]}: raise ValueError("Original runtime was renamed by another operator") if current_name != self.plan["originalName"]: self.runner(["docker", "rename", self.prepared["originalContainerId"], self.plan["originalName"]]) private_write(self.broker.CONFIG_PATH, (self.workspace / "original-projects.json").read_bytes()) private_write(self.state_path, (self.workspace / "original-state.json").read_bytes()) self.journal["originalRestartIntent"] = True self.save("restoring_original") if not self.original()["running"]: self.runner(["docker", "start", self.prepared["originalContainerId"]], timeout=90) deadline = time.monotonic() + 240 while time.monotonic() < deadline: try: self.original(healthy=True) self.http_health() return except (ValueError, OSError): time.sleep(3) raise ValueError("Original runtime recovery health did not verify") def private_read(path): import os import stat info = path.lstat() if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) & 0o077: raise ValueError("Expected a root-only regular transaction file") return path.read_bytes() def operate(broker, mode, slug, revision, attempt, container_id, image_id, runner=checked): if mode not in {"cutover", "recover"} or slug not in TARGETS or not SHA.fullmatch(revision) or not ATTEMPT.fullmatch(attempt): raise ValueError("Invalid fixed migration operation") workspace = require_real_child(JOURNAL_ROOT / slug / attempt, JOURNAL_ROOT) repository = f"jens/{slug}" with ExitStack() as locks: locks.enter_context(broker.exclusive_lock(broker.STATE_ROOT / "locks" / f"{slug}.lock", "migration project")) locks.enter_context(broker.exclusive_lock(broker.GLOBAL_DEPLOY_LOCK, "migration production")) prepared_bytes = private_read(workspace / "prepare.json") prepared = json.loads(prepared_bytes) if (prepared.get("phase") != "build_verified" or prepared.get("repository") != repository or prepared.get("revision") != revision or prepared.get("originalContainerId") != container_id or prepared.get("originalImageId") != image_id): raise ValueError("Preparation identity does not match the requested operation") journal_path = workspace / "cutover.json" if mode == "recover": journal = json.loads(private_read(journal_path)) if journal.get("preparedSha256") != digest(prepared_bytes): raise ValueError("Prepared artifact changed after transaction snapshot") ops = Production(broker, slug, prepared, workspace, journal, runner) phase = recover_transaction(ops) return {"phase": phase, "repository": repository, "candidateDataRetained": True} if journal_path.exists(): raise ValueError("Transaction already exists; use explicit recover, never repeat cutover") projects_bytes = broker.CONFIG_PATH.read_bytes() projects = json.loads(projects_bytes) original = projects[repository] state_bytes = (broker.STATE_ROOT / f"{slug}.json").read_bytes() if (digest(json.dumps(original, sort_keys=True).encode()) != prepared.get("projectConfigSha256") or digest(state_bytes) != prepared.get("stateSha256")): raise ValueError("Original broker config/state changed since preparation") if broker.state_live_revision(json.loads(state_bytes)): raise ValueError("This transaction only supports unattested legacy first deployments") repo = broker.repository_path(repository, original) broker.validate_commit(repo, str(original["branch"]), revision) if broker.output(["git", f"--git-dir={repo}", "rev-parse", f"refs/heads/{original['branch']}"]) != revision: raise ValueError("Prepared revision is no longer the configured branch head") source = require_real_child(Path(prepared["source"]), broker.EXACT_SOURCE_ROOT) if not broker.exact_source_matches_revision(repo, revision, source): raise ValueError("Prepared source changed before cutover") if verify_image(prepared["candidateImage"], prepared["metadata"], runner) != prepared["candidateImageId"]: raise ValueError("Prepared immutable image changed") assert_original(original_identity(container_id, runner), slug, container_id, image_id) if mount_inventory(container_id, runner) != prepared["originalMounts"]: raise ValueError("Original mounts changed after preparation") if digest(private_read(workspace / "runtime.env")) != prepared["runtimeEnvSha256"]: raise ValueError("Private runtime environment changed after preparation") plan = build_plan(slug, revision, attempt, {"name": TARGETS[slug][0], **original_identity(container_id, runner), "mounts": prepared["originalMounts"]}) if plan != prepared["plan"]: raise ValueError("Cold-copy plan changed after preparation") candidate = candidate_project(original, prepared, plan, prepared["nonce"], workspace / "runtime.env") if candidate != prepared["candidateProject"]: raise ValueError("Candidate project differs from the build configuration") broker.validate_project_config(repo, candidate, revision) render_contract(broker, candidate, prepared, plan, runner) required = 8 * 1024**3 for item in plan["coldCopyRoots"]: root = Path(item["original"]) if root.is_symlink() or root.resolve() != root or not root.is_dir(): raise ValueError("Cold-copy source is not an unchanged real directory") required += sum(path.stat().st_size for path in root.rglob("*") if path.is_file()) if Path(item["candidate"]).exists(): raise ValueError("Cold-copy target already exists") if shutil.disk_usage(workspace).free < required: raise ValueError("Insufficient space for retained cold clones plus safety reserve") updated = {**projects, repository: candidate} candidate_bytes = json.dumps(updated, sort_keys=True).encode() private_write(workspace / "original-projects.json", projects_bytes) private_write(workspace / "original-state.json", state_bytes) private_write(workspace / "candidate-projects.json", candidate_bytes) # Configuration may contain secrets: stored privately, never in user output. snapshot = inspect_fields(runner, container_id, [".Config", ".HostConfig", ".NetworkSettings.Networks"]) private_json(workspace / "original-container.json", snapshot) journal = {"schema": "legacy-cutover.v1", "phase": "snapshotted", "plan": plan, "preparedSha256": digest(prepared_bytes), "candidateProject": candidate, "repositoryPath": str(repo), "nonce": prepared["nonce"], "originalMounts": prepared["originalMounts"], "originalConfigSha256": digest(projects_bytes), "originalStateSha256": digest(state_bytes), "candidateConfigSha256": digest(candidate_bytes)} ops = Production(broker, slug, prepared, workspace, journal, runner) ops.http_health() if ops.canonical_id(plan["retainedName"]) or broker.compose_project_container_names(plan["candidateComposeProject"]): raise ValueError("Attempt runtime/project is already occupied") runner(["docker", "image", "tag", image_id, f"gitea-legacy-rollback/{slug}:{prepared['nonce']}"]) ops.save("snapshotted") run_transaction(ops) return {"phase": "committed", "repository": repository, "revision": revision, "originalContainerRetained": True, "originalDataRetained": True, "receiptVerified": True, "journal": str(journal_path)}