Initial curated projectbrain source from 8a7b1e0651dbdab97d744cbd2e653ab3970ac6c5
This commit is contained in:
Public Source Publisher committed
commit
5aa3e8d685
700 files changed
+132801
No files matched your search
@@ -0,0 +1,517 @@
|
||||
"""Journaled legacy cutover through normal broker functions; fixed two targets only.
|
||||
|
||||
No provenance is written for the original runtime. Recovery retains every data
|
||||
clone and original image/container. Completed migrations are never automatically
|
||||
rolled back: new writes may have reached candidate data after commit.
|
||||
"""
|
||||
|
||||
from contextlib import ExitStack
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import time
|
||||
from urllib.request import urlopen
|
||||
|
||||
from execute import (JOURNAL_ROOT, assert_original, checked, docker_json_template, mount_inventory, original_identity,
|
||||
require_real_child, verify_image)
|
||||
from plan import ATTEMPT, SHA, TARGETS, build_plan
|
||||
from storage import clone_cold, digest, dotenv, private_json, private_write, tree_manifest
|
||||
|
||||
|
||||
STEPS = ("stopping", "copying", "renaming", "deploying", "verifying", "committing")
|
||||
|
||||
|
||||
def run_transaction(ops):
|
||||
"""Write intent before every side effect, allowing recovery after abrupt exit."""
|
||||
try:
|
||||
for phase in STEPS:
|
||||
ops.save(phase)
|
||||
getattr(ops, phase)()
|
||||
ops.save("committed")
|
||||
except Exception:
|
||||
try:
|
||||
ops.recover()
|
||||
ops.save("rolled_back")
|
||||
except Exception:
|
||||
ops.save("recovery_failed")
|
||||
raise RuntimeError("Migration failed and rollback requires explicit recovery") from None
|
||||
raise RuntimeError("Migration failed; original runtime restored and candidate data retained") from None
|
||||
# Outside rollback scope: releasing workers permits new external effects/data.
|
||||
ops.activate()
|
||||
|
||||
|
||||
def recover_transaction(ops):
|
||||
if ops.journal["phase"] == "committed":
|
||||
ops.verify_committed()
|
||||
ops.activate()
|
||||
return "committed"
|
||||
if ops.journal["phase"] == "rolled_back":
|
||||
ops.original(healthy=True)
|
||||
ops.http_health()
|
||||
return "rolled_back"
|
||||
ops.recover()
|
||||
ops.save("rolled_back")
|
||||
return "rolled_back"
|
||||
|
||||
|
||||
def inspect_fields(runner, target, expressions):
|
||||
template = docker_json_template(expressions)
|
||||
return [json.loads(line) for line in runner(["docker", "inspect", "--format", template, target]).splitlines()]
|
||||
|
||||
|
||||
def mounts(runner, target):
|
||||
raw = inspect_fields(runner, target, [".Mounts"])[0]
|
||||
return sorted(({"type": item["Type"], "source": item["Source"],
|
||||
"destination": item["Destination"], "rw": item["RW"]} for item in raw),
|
||||
key=lambda item: item["destination"])
|
||||
|
||||
|
||||
def candidate_project(original, prepared, plan, nonce, env_path):
|
||||
slug = plan["repository"].split("/")[1]
|
||||
by_target = {item["destination"]: item["source"] for item in plan["candidateMounts"]}
|
||||
names = ({"MIGRATION_LOCAL_ROOT": "/app/local"} if slug == "vacatureradar" else {
|
||||
"MIGRATION_CONFIG_ROOT": "/config", "MIGRATION_OPENRGB_ROOT": "/config/openrgb",
|
||||
"MIGRATION_LUMA_CONFIG_ROOT": "/config/lumaops", "MIGRATION_DATA_ROOT": "/data",
|
||||
"MIGRATION_LOGS_ROOT": "/logs"})
|
||||
environment = {**dict(original.get("environment", {})),
|
||||
**{key: by_target[target] for key, target in names.items()},
|
||||
"MIGRATION_ATTEMPT": nonce, "MIGRATION_IMAGE": f"gitea-managed/{slug}:{{revision}}",
|
||||
"MIGRATION_NETWORK": slug + "_default",
|
||||
"MIGRATION_SOURCE_REVISION": "{revision}", "MIGRATION_SOURCE_TREE": "{tree}",
|
||||
"MIGRATION_BUILD_ID": "{build_id}", "MIGRATION_BUILD_DATE": "{build_date}",
|
||||
"MIGRATION_ENV_FILE": str(env_path), "VACATURERADAR_ENV_FILE": str(env_path)}
|
||||
result = dict(original)
|
||||
for key in ("deploy_script", "archive_deploy_script", "manual", "clean_untracked_paths", "legacy_containers"):
|
||||
result.pop(key, None)
|
||||
result.update(source_dir=prepared["source"], compose_project=plan["candidateComposeProject"],
|
||||
exact_source=True, verify_oci_revision=True, env_source=str(env_path),
|
||||
environment=environment,
|
||||
compose_files=[*original["compose_files"], "docker-compose.migration.yml"])
|
||||
return result
|
||||
|
||||
|
||||
def render_contract(broker, project, prepared, plan, runner):
|
||||
cwd, command = broker.compose_command(project)
|
||||
environment = broker.deployment_environment(project, prepared["revision"], plan["repository"], prepared["metadata"])
|
||||
rendered = json.loads(runner(command + ["config", "--format", "json"], cwd=cwd, env=environment))
|
||||
services = rendered.get("services", {})
|
||||
if len(services) != 1:
|
||||
raise ValueError("Migration supports exactly one reviewed application container")
|
||||
service = next(iter(services.values()))
|
||||
if service.get("container_name") != plan["originalName"]:
|
||||
raise ValueError("Candidate canonical name differs from the plan")
|
||||
network = plan["repository"].split("/")[1] + "_default"
|
||||
original_networks = inspect_fields(runner, prepared["originalContainerId"], [".NetworkSettings.Networks"])[0]
|
||||
if set(original_networks) != {network} or service.get("network_mode") != network:
|
||||
raise ValueError("Candidate network must exactly preserve the reviewed original network")
|
||||
if prepared.get("originalNetwork") not in {None, network}:
|
||||
raise ValueError("Original network changed since preparation")
|
||||
host = inspect_fields(runner, prepared["originalContainerId"], [".HostConfig"])[0]
|
||||
validate_runtime_safety(host, service)
|
||||
expected = {item["destination"]: item["source"] for item in plan["candidateMounts"]}
|
||||
observed = {}
|
||||
for item in service.get("volumes", []):
|
||||
if item.get("type") != "bind" or item.get("read_only") or item.get("bind", {}).get("create_host_path") is not False:
|
||||
raise ValueError("Candidate contains an unreviewed writable mount")
|
||||
observed[item["target"]] = item["source"]
|
||||
if observed != expected or len(observed) != len(service.get("volumes", [])):
|
||||
raise ValueError("Rendered candidate mounts do not exactly match cold clones")
|
||||
bindings = inspect_fields(runner, prepared["originalContainerId"], [".HostConfig.PortBindings"])[0]
|
||||
old_ports = sorted((port, item.get("HostIp") or "0.0.0.0", str(item["HostPort"]))
|
||||
for port, items in bindings.items() for item in items or [])
|
||||
new_ports = sorted((f"{item['target']}/{item.get('protocol', 'tcp')}", item.get("host_ip") or "0.0.0.0", str(item["published"]))
|
||||
for item in service.get("ports", []))
|
||||
if old_ports != new_ports:
|
||||
raise ValueError("Candidate published ports differ from the original runtime")
|
||||
if service.get("labels", {}).get("io.itworx.migration-attempt") != environment["MIGRATION_ATTEMPT"]:
|
||||
raise ValueError("Candidate ownership correlation label is absent")
|
||||
return cwd, command, environment
|
||||
|
||||
|
||||
def validate_runtime_safety(host, service):
|
||||
"""Compare nonsecret hardware/security bindings; unsupported shapes fail closed."""
|
||||
old_devices = sorted((item["PathOnHost"], item["PathInContainer"], item["CgroupPermissions"])
|
||||
for item in host.get("Devices") or [])
|
||||
new_devices = sorted((item["source"], item["target"], item.get("permissions", "rwm"))
|
||||
for item in service.get("devices", []))
|
||||
if old_devices != new_devices or host.get("DeviceRequests") or host.get("DeviceCgroupRules"):
|
||||
raise ValueError("Candidate device contract differs or is unsupported")
|
||||
for actual, desired in (("CapAdd", "cap_add"), ("CapDrop", "cap_drop")):
|
||||
normalize = lambda values: sorted(value.upper().removeprefix("CAP_") for value in values or [])
|
||||
if normalize(host.get(actual)) != normalize(service.get(desired)):
|
||||
raise ValueError("Candidate capability contract differs")
|
||||
for actual, desired in (("ReadonlyRootfs", "read_only"), ("Privileged", "privileged")):
|
||||
if bool(host.get(actual)) != bool(service.get(desired)):
|
||||
raise ValueError("Candidate filesystem/privilege contract differs")
|
||||
if sorted(host.get("SecurityOpt") or []) != sorted(service.get("security_opt") or []):
|
||||
raise ValueError("Candidate security options differ")
|
||||
new_tmpfs = {}
|
||||
for item in service.get("tmpfs") or []:
|
||||
target, separator, options = item.partition(":")
|
||||
if target in new_tmpfs:
|
||||
raise ValueError("Duplicate candidate tmpfs target")
|
||||
new_tmpfs[target] = options if separator else ""
|
||||
if (host.get("Tmpfs") or {}) != new_tmpfs:
|
||||
raise ValueError("Candidate tmpfs contract differs")
|
||||
|
||||
|
||||
def prepare_compose(broker, slug, revision, attempt, original, prepared, workspace, runner):
|
||||
if slug == "vacatureradar":
|
||||
source = Path(prepared["source"])
|
||||
required = {
|
||||
"scripts/migration_worker_gate.py": (".migration-worker-hold", "while check():"),
|
||||
"deployment/unraid/supervisord.conf": ("migration_worker_gate.py worker", "migration_worker_gate.py scheduler"),
|
||||
"config/settings.py": ("apps.core.middleware.MigrationMaintenanceMiddleware",),
|
||||
"apps/core/middleware.py": ("class MigrationMaintenanceMiddleware", "migration_maintenance"),
|
||||
}
|
||||
for filename, fragments in required.items():
|
||||
content = (source / filename).read_text()
|
||||
if any(fragment not in content for fragment in fragments):
|
||||
raise ValueError("Exact source lacks the reviewed maintenance contract")
|
||||
observed = {"name": TARGETS[slug][0], **original_identity(prepared["originalContainerId"], runner),
|
||||
"mounts": prepared["originalMounts"]}
|
||||
plan = build_plan(slug, revision, attempt, observed)
|
||||
if Path(plan["workspace"]).resolve() != workspace:
|
||||
raise ValueError("Plan workspace differs from the protected preparation directory")
|
||||
nonce = secrets.token_hex(16)
|
||||
env_path = workspace / "runtime.env"
|
||||
entries = inspect_fields(runner, prepared["originalContainerId"], [".Config.Env"])[0]
|
||||
private_write(env_path, dotenv(entries))
|
||||
prepared["originalNetwork"] = slug + "_default"
|
||||
candidate = candidate_project(original, prepared, plan, nonce, env_path)
|
||||
repo = broker.repository_path(plan["repository"], original)
|
||||
broker.validate_project_config(repo, candidate, revision)
|
||||
cwd, command, environment = render_contract(broker, candidate, prepared, plan, runner)
|
||||
return candidate, command + ["build"], cwd, environment, plan, nonce
|
||||
|
||||
|
||||
class Production:
|
||||
def __init__(self, broker, slug, prepared, workspace, journal, runner=checked):
|
||||
self.broker, self.slug, self.prepared = broker, slug, prepared
|
||||
self.workspace, self.journal, self.runner = workspace, journal, runner
|
||||
self.repository = f"jens/{slug}"
|
||||
self.plan = journal["plan"]
|
||||
self.project = dict(journal["candidateProject"])
|
||||
self.project.update(_repository_name=self.repository,
|
||||
_repository_path=journal["repositoryPath"],
|
||||
_deployment_metadata=prepared["metadata"])
|
||||
self.state_path = broker.STATE_ROOT / f"{slug}.json"
|
||||
|
||||
def save(self, phase):
|
||||
self.journal["phase"] = phase
|
||||
private_json(self.workspace / "cutover.json", self.journal)
|
||||
|
||||
def canonical_id(self, name):
|
||||
result = self.runner(["docker", "container", "ls", "--all", "--quiet", "--no-trunc",
|
||||
"--filter", f"name=^/{name}$"]).strip().splitlines()
|
||||
if len(result) > 1:
|
||||
raise ValueError("Ambiguous container identity")
|
||||
return result[0] if result else None
|
||||
|
||||
def original(self, healthy=False):
|
||||
container_id = self.prepared["originalContainerId"]
|
||||
observed = original_identity(container_id, self.runner)
|
||||
if observed["imageId"] != self.prepared["originalImageId"] or observed["composeProject"] != self.slug:
|
||||
raise ValueError("Original immutable runtime identity changed")
|
||||
if mounts(self.runner, container_id) != self.journal["originalMounts"]:
|
||||
raise ValueError("Original mounts changed")
|
||||
networks = inspect_fields(self.runner, container_id, [".NetworkSettings.Networks"])[0]
|
||||
if set(networks) != {self.slug + "_default"}:
|
||||
raise ValueError("Original network changed")
|
||||
if healthy:
|
||||
assert_original(observed, self.slug, container_id, self.prepared["originalImageId"])
|
||||
return observed
|
||||
|
||||
def stopping(self):
|
||||
self.original(healthy=True)
|
||||
self.runner(["docker", "stop", "--time", "60", self.prepared["originalContainerId"]], timeout=90)
|
||||
if self.original()["running"]:
|
||||
raise ValueError("Original did not stop")
|
||||
|
||||
def copying(self):
|
||||
if self.original()["running"]:
|
||||
raise ValueError("Cold copy requires stopped original")
|
||||
checksums = {}
|
||||
for item in self.plan["coldCopyRoots"]:
|
||||
original = Path(item["original"])
|
||||
if original.is_symlink() or original.resolve() != original:
|
||||
raise ValueError("Unreviewed symlink in original cold-copy root")
|
||||
target = require_real_child(Path(item["candidate"]), self.workspace)
|
||||
checksums[item["original"]] = clone_cold(original, target, self.runner)
|
||||
self.journal["coldCopyChecksums"] = checksums
|
||||
self.save("copying")
|
||||
if self.slug == "vacatureradar":
|
||||
marker = self.hold_path()
|
||||
if marker.exists() or marker.is_symlink():
|
||||
raise ValueError("Cold clone already contains a maintenance hold")
|
||||
private_write(marker, self.hold_bytes())
|
||||
|
||||
def hold_path(self):
|
||||
roots = [item["source"] for item in self.plan["candidateMounts"]
|
||||
if item["destination"] == "/app/local"]
|
||||
if len(roots) != 1:
|
||||
raise ValueError("Maintenance hold requires one exact cloned local root")
|
||||
return require_real_child(Path(roots[0]) / ".migration-worker-hold", self.workspace)
|
||||
|
||||
def hold_bytes(self):
|
||||
return ("legacy-migration.v1:" + self.journal["nonce"] + "\n").encode()
|
||||
|
||||
def verify_hold(self):
|
||||
if self.slug == "vacatureradar" and private_read(self.hold_path()) != self.hold_bytes():
|
||||
raise ValueError("Maintenance hold identity does not match this attempt")
|
||||
|
||||
def activate(self):
|
||||
if self.journal.get("phase") != "committed":
|
||||
raise ValueError("Activation requires a durable committed transaction")
|
||||
self.verify_committed()
|
||||
self.assert_candidate(self.journal["candidateContainerId"])
|
||||
if self.slug == "vacatureradar":
|
||||
marker = self.hold_path()
|
||||
if marker.exists() or marker.is_symlink():
|
||||
self.verify_hold()
|
||||
self.journal["activationIntent"] = True
|
||||
self.save("committed")
|
||||
marker.unlink()
|
||||
descriptor = os.open(marker.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
elif not self.journal.get("activationIntent"):
|
||||
raise ValueError("Maintenance hold disappeared before activation intent")
|
||||
self.journal["activated"] = True
|
||||
self.save("committed")
|
||||
|
||||
def renaming(self):
|
||||
if self.canonical_id(self.plan["retainedName"]):
|
||||
raise ValueError("Retained container name is already occupied")
|
||||
if self.canonical_id(self.plan["originalName"]) != self.prepared["originalContainerId"]:
|
||||
raise ValueError("Canonical container identity changed")
|
||||
self.runner(["docker", "rename", self.prepared["originalContainerId"], self.plan["retainedName"]])
|
||||
|
||||
def deploying(self):
|
||||
self.verify_hold()
|
||||
if self.canonical_id(self.plan["originalName"]):
|
||||
raise ValueError("Canonical name was claimed before candidate deployment")
|
||||
# True first deploy relative to the distinct managed Compose project.
|
||||
if self.broker.capture_compose_rollback(self.project) is not None:
|
||||
raise ValueError("Candidate project is not a first deployment")
|
||||
self.broker.revalidate_deployment_candidate(Path(self.journal["repositoryPath"]), self.project,
|
||||
self.prepared["revision"], self.repository, self.prepared["metadata"])
|
||||
cwd, command, environment = render_contract(self.broker, self.project, self.prepared, self.plan, self.runner)
|
||||
image_id = verify_image(environment["MIGRATION_IMAGE"], self.prepared["metadata"], self.runner)
|
||||
if image_id != self.prepared["candidateImageId"]:
|
||||
raise ValueError("Prepared image tag changed before candidate start")
|
||||
self.runner(command + ["up", "--detach", "--no-build", "--pull", "never"],
|
||||
cwd=cwd, env=environment, timeout=180)
|
||||
container_id = self.canonical_id(self.plan["originalName"])
|
||||
if not container_id:
|
||||
raise ValueError("Candidate deployment produced no canonical container")
|
||||
self.assert_candidate(container_id)
|
||||
self.journal["candidateContainerId"] = container_id
|
||||
self.save("deploying")
|
||||
|
||||
def assert_candidate(self, container_id):
|
||||
values = inspect_fields(self.runner, container_id, [".Id", ".Image",
|
||||
'index .Config.Labels "com.docker.compose.project"',
|
||||
'index .Config.Labels "io.itworx.migration-attempt"'])
|
||||
if (values[0] != container_id or container_id == self.prepared["originalContainerId"]
|
||||
or values[2] != self.plan["candidateComposeProject"] or values[3] != self.journal["nonce"]):
|
||||
raise ValueError("Refusing an unrelated candidate container")
|
||||
recorded = self.journal.get("candidateContainerId")
|
||||
if recorded and recorded != container_id:
|
||||
raise ValueError("Recorded candidate container was replaced")
|
||||
|
||||
def http_health(self):
|
||||
path = "http://127.0.0.1:1226/health/ready/" if self.slug == "vacatureradar" else "http://127.0.0.1:1223/health/ready"
|
||||
with urlopen(path, timeout=10) as response:
|
||||
if response.status != 200 or response.geturl() != path:
|
||||
raise ValueError("Representative readiness probe failed")
|
||||
|
||||
def verifying(self):
|
||||
self.verify_hold()
|
||||
self.assert_candidate(self.journal["candidateContainerId"])
|
||||
expected = sorted(self.plan["candidateMounts"], key=lambda item: item["destination"])
|
||||
if mounts(self.runner, self.journal["candidateContainerId"]) != expected:
|
||||
raise ValueError("Candidate mounts differ from the isolated cold-copy plan")
|
||||
networks = inspect_fields(self.runner, self.journal["candidateContainerId"], [".NetworkSettings.Networks"])[0]
|
||||
if set(networks) != {self.slug + "_default"}:
|
||||
raise ValueError("Candidate runtime network differs from the original")
|
||||
receipt = self.broker.verify_deployment(self.project, self.prepared["metadata"],
|
||||
expected_image_ids={self.plan["originalName"]: self.prepared["candidateImageId"]})
|
||||
self.broker.validate_recorded_receipt(self.project, receipt, self.repository, self.prepared["revision"])
|
||||
self.http_health()
|
||||
self.journal["receipt"] = receipt
|
||||
self.save("verifying")
|
||||
|
||||
def committing(self):
|
||||
self.broker.write_state(self.repository, self.project, self.prepared["revision"], "success",
|
||||
"Controlled legacy migration passed immutable image and readiness verification",
|
||||
live_receipt=self.journal["receipt"], attempt_metadata=self.prepared["metadata"])
|
||||
# Never alter another project's configuration. Both deployment locks held.
|
||||
current = self.broker.CONFIG_PATH.read_bytes()
|
||||
if digest(current) != self.journal["originalConfigSha256"]:
|
||||
raise ValueError("Broker configuration changed during cutover")
|
||||
private_write(self.broker.CONFIG_PATH, (self.workspace / "candidate-projects.json").read_bytes())
|
||||
self.verify_committed()
|
||||
|
||||
def verify_committed(self):
|
||||
if digest(self.broker.CONFIG_PATH.read_bytes()) != self.journal["candidateConfigSha256"]:
|
||||
raise ValueError("Committed broker configuration no longer matches")
|
||||
receipt = self.broker.verified_current_receipt(self.repository, self.project,
|
||||
self.prepared["revision"], self.prepared["metadata"]["tree"])
|
||||
if receipt is None:
|
||||
raise ValueError("Committed runtime receipt does not reverify")
|
||||
self.http_health()
|
||||
|
||||
def recover(self):
|
||||
if self.journal.get("phase") == "committed" or self.journal.get("activationIntent"):
|
||||
raise ValueError("Committed activation must never roll back")
|
||||
for filename, key in (("original-projects.json", "originalConfigSha256"),
|
||||
("original-state.json", "originalStateSha256")):
|
||||
if digest((self.workspace / filename).read_bytes()) != self.journal[key]:
|
||||
raise ValueError("Protected rollback snapshot integrity failed")
|
||||
config = self.broker.CONFIG_PATH.read_bytes()
|
||||
if digest(config) not in {self.journal["originalConfigSha256"], self.journal["candidateConfigSha256"]}:
|
||||
raise ValueError("Unrelated broker configuration change blocks automatic recovery")
|
||||
state = self.state_path.read_bytes()
|
||||
if digest(state) != self.journal["originalStateSha256"]:
|
||||
payload = json.loads(state)
|
||||
if payload.get("live_receipt") != self.journal.get("receipt") or not self.journal.get("receipt"):
|
||||
raise ValueError("Unrelated deployment state blocks automatic recovery")
|
||||
self.original()
|
||||
if not self.journal.get("originalRestartIntent"):
|
||||
for source, expected in self.journal.get("coldCopyChecksums", {}).items():
|
||||
if digest(json.dumps(tree_manifest(source), sort_keys=True).encode()) != expected:
|
||||
raise ValueError("Original data changed after verified cold copy")
|
||||
container_id = self.canonical_id(self.plan["originalName"])
|
||||
if container_id and container_id != self.prepared["originalContainerId"]:
|
||||
self.assert_candidate(container_id)
|
||||
# Persist discovered ownership before removing a crash-created container.
|
||||
self.journal["candidateContainerId"] = container_id
|
||||
self.save("recovering")
|
||||
self.runner(["docker", "rm", "--force", container_id], timeout=90)
|
||||
if self.canonical_id(self.plan["originalName"]) not in {None, self.prepared["originalContainerId"]}:
|
||||
raise ValueError("Canonical name is occupied by unrelated runtime")
|
||||
current_name = inspect_fields(self.runner, self.prepared["originalContainerId"], [".Name"])[0].lstrip("/")
|
||||
if current_name not in {self.plan["originalName"], self.plan["retainedName"]}:
|
||||
raise ValueError("Original runtime was renamed by another operator")
|
||||
if current_name != self.plan["originalName"]:
|
||||
self.runner(["docker", "rename", self.prepared["originalContainerId"], self.plan["originalName"]])
|
||||
private_write(self.broker.CONFIG_PATH, (self.workspace / "original-projects.json").read_bytes())
|
||||
private_write(self.state_path, (self.workspace / "original-state.json").read_bytes())
|
||||
self.journal["originalRestartIntent"] = True
|
||||
self.save("restoring_original")
|
||||
if not self.original()["running"]:
|
||||
self.runner(["docker", "start", self.prepared["originalContainerId"]], timeout=90)
|
||||
deadline = time.monotonic() + 240
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
self.original(healthy=True)
|
||||
self.http_health()
|
||||
return
|
||||
except (ValueError, OSError):
|
||||
time.sleep(3)
|
||||
raise ValueError("Original runtime recovery health did not verify")
|
||||
|
||||
|
||||
def private_read(path):
|
||||
import os
|
||||
import stat
|
||||
info = path.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) & 0o077:
|
||||
raise ValueError("Expected a root-only regular transaction file")
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def operate(broker, mode, slug, revision, attempt, container_id, image_id, runner=checked):
|
||||
if mode not in {"cutover", "recover"} or slug not in TARGETS or not SHA.fullmatch(revision) or not ATTEMPT.fullmatch(attempt):
|
||||
raise ValueError("Invalid fixed migration operation")
|
||||
workspace = require_real_child(JOURNAL_ROOT / slug / attempt, JOURNAL_ROOT)
|
||||
repository = f"jens/{slug}"
|
||||
with ExitStack() as locks:
|
||||
locks.enter_context(broker.exclusive_lock(broker.STATE_ROOT / "locks" / f"{slug}.lock", "migration project"))
|
||||
locks.enter_context(broker.exclusive_lock(broker.GLOBAL_DEPLOY_LOCK, "migration production"))
|
||||
prepared_bytes = private_read(workspace / "prepare.json")
|
||||
prepared = json.loads(prepared_bytes)
|
||||
if (prepared.get("phase") != "build_verified" or prepared.get("repository") != repository
|
||||
or prepared.get("revision") != revision or prepared.get("originalContainerId") != container_id
|
||||
or prepared.get("originalImageId") != image_id):
|
||||
raise ValueError("Preparation identity does not match the requested operation")
|
||||
journal_path = workspace / "cutover.json"
|
||||
if mode == "recover":
|
||||
journal = json.loads(private_read(journal_path))
|
||||
if journal.get("preparedSha256") != digest(prepared_bytes):
|
||||
raise ValueError("Prepared artifact changed after transaction snapshot")
|
||||
ops = Production(broker, slug, prepared, workspace, journal, runner)
|
||||
phase = recover_transaction(ops)
|
||||
return {"phase": phase, "repository": repository, "candidateDataRetained": True}
|
||||
if journal_path.exists():
|
||||
raise ValueError("Transaction already exists; use explicit recover, never repeat cutover")
|
||||
projects_bytes = broker.CONFIG_PATH.read_bytes()
|
||||
projects = json.loads(projects_bytes)
|
||||
original = projects[repository]
|
||||
state_bytes = (broker.STATE_ROOT / f"{slug}.json").read_bytes()
|
||||
if (digest(json.dumps(original, sort_keys=True).encode()) != prepared.get("projectConfigSha256")
|
||||
or digest(state_bytes) != prepared.get("stateSha256")):
|
||||
raise ValueError("Original broker config/state changed since preparation")
|
||||
if broker.state_live_revision(json.loads(state_bytes)):
|
||||
raise ValueError("This transaction only supports unattested legacy first deployments")
|
||||
repo = broker.repository_path(repository, original)
|
||||
broker.validate_commit(repo, str(original["branch"]), revision)
|
||||
if broker.output(["git", f"--git-dir={repo}", "rev-parse", f"refs/heads/{original['branch']}"]) != revision:
|
||||
raise ValueError("Prepared revision is no longer the configured branch head")
|
||||
source = require_real_child(Path(prepared["source"]), broker.EXACT_SOURCE_ROOT)
|
||||
if not broker.exact_source_matches_revision(repo, revision, source):
|
||||
raise ValueError("Prepared source changed before cutover")
|
||||
if verify_image(prepared["candidateImage"], prepared["metadata"], runner) != prepared["candidateImageId"]:
|
||||
raise ValueError("Prepared immutable image changed")
|
||||
assert_original(original_identity(container_id, runner), slug, container_id, image_id)
|
||||
if mount_inventory(container_id, runner) != prepared["originalMounts"]:
|
||||
raise ValueError("Original mounts changed after preparation")
|
||||
if digest(private_read(workspace / "runtime.env")) != prepared["runtimeEnvSha256"]:
|
||||
raise ValueError("Private runtime environment changed after preparation")
|
||||
plan = build_plan(slug, revision, attempt, {"name": TARGETS[slug][0],
|
||||
**original_identity(container_id, runner), "mounts": prepared["originalMounts"]})
|
||||
if plan != prepared["plan"]:
|
||||
raise ValueError("Cold-copy plan changed after preparation")
|
||||
candidate = candidate_project(original, prepared, plan, prepared["nonce"], workspace / "runtime.env")
|
||||
if candidate != prepared["candidateProject"]:
|
||||
raise ValueError("Candidate project differs from the build configuration")
|
||||
broker.validate_project_config(repo, candidate, revision)
|
||||
render_contract(broker, candidate, prepared, plan, runner)
|
||||
required = 8 * 1024**3
|
||||
for item in plan["coldCopyRoots"]:
|
||||
root = Path(item["original"])
|
||||
if root.is_symlink() or root.resolve() != root or not root.is_dir():
|
||||
raise ValueError("Cold-copy source is not an unchanged real directory")
|
||||
required += sum(path.stat().st_size for path in root.rglob("*") if path.is_file())
|
||||
if Path(item["candidate"]).exists():
|
||||
raise ValueError("Cold-copy target already exists")
|
||||
if shutil.disk_usage(workspace).free < required:
|
||||
raise ValueError("Insufficient space for retained cold clones plus safety reserve")
|
||||
updated = {**projects, repository: candidate}
|
||||
candidate_bytes = json.dumps(updated, sort_keys=True).encode()
|
||||
private_write(workspace / "original-projects.json", projects_bytes)
|
||||
private_write(workspace / "original-state.json", state_bytes)
|
||||
private_write(workspace / "candidate-projects.json", candidate_bytes)
|
||||
# Configuration may contain secrets: stored privately, never in user output.
|
||||
snapshot = inspect_fields(runner, container_id, [".Config", ".HostConfig", ".NetworkSettings.Networks"])
|
||||
private_json(workspace / "original-container.json", snapshot)
|
||||
journal = {"schema": "legacy-cutover.v1", "phase": "snapshotted", "plan": plan,
|
||||
"preparedSha256": digest(prepared_bytes), "candidateProject": candidate,
|
||||
"repositoryPath": str(repo), "nonce": prepared["nonce"], "originalMounts": prepared["originalMounts"],
|
||||
"originalConfigSha256": digest(projects_bytes), "originalStateSha256": digest(state_bytes),
|
||||
"candidateConfigSha256": digest(candidate_bytes)}
|
||||
ops = Production(broker, slug, prepared, workspace, journal, runner)
|
||||
ops.http_health()
|
||||
if ops.canonical_id(plan["retainedName"]) or broker.compose_project_container_names(plan["candidateComposeProject"]):
|
||||
raise ValueError("Attempt runtime/project is already occupied")
|
||||
runner(["docker", "image", "tag", image_id, f"gitea-legacy-rollback/{slug}:{prepared['nonce']}"])
|
||||
ops.save("snapshotted")
|
||||
run_transaction(ops)
|
||||
return {"phase": "committed", "repository": repository, "revision": revision,
|
||||
"originalContainerRetained": True, "originalDataRetained": True,
|
||||
"receiptVerified": True, "journal": str(journal_path)}
|
||||
Reference in new issue
Block a user