222 lines
11 KiB
Python
222 lines
11 KiB
Python
"""Explicit fixed-target preparation, journaled cutover and crash recovery.
|
|
|
|
Run on the reviewed Unraid controller host. Builds a verified exact-source candidate
|
|
Preparation leaves live config/state/containers/data untouched. Cutover is a
|
|
separate --apply operation requiring a reviewed, verified preparation. Originals
|
|
and failed candidate data are retained; recover never undoes a committed migration.
|
|
"""
|
|
|
|
import argparse
|
|
from contextlib import ExitStack
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
|
|
from plan import ATTEMPT, ID, SHA, TARGETS
|
|
|
|
|
|
BROKER_PATH = Path("/mnt/pool/appdata/gitea-deploy-runners/controller/deploy.py")
|
|
BROKER_SHA = "b5a0467f53acba5c783781a22bf81cf99f8b96ae99535460ad6236fac038330d"
|
|
JOURNAL_ROOT = Path("/mnt/pool/appdata/gitea-deploy-runners/legacy-migrations")
|
|
DOCKERFILES = {"vacatureradar": "Dockerfile.unraid", "lumaops": "Dockerfile"}
|
|
|
|
|
|
def durable_json(path, payload):
|
|
temporary = path.with_suffix(".tmp")
|
|
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(descriptor, "w") as stream:
|
|
json.dump(payload, stream, sort_keys=True)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, path)
|
|
descriptor = os.open(path.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def require_real_child(path, parent):
|
|
resolved = path.resolve()
|
|
root = parent.resolve()
|
|
if path.is_symlink() or resolved == root or not resolved.is_relative_to(root):
|
|
raise ValueError("Candidate path is not a private child of the fixed root")
|
|
return resolved
|
|
|
|
|
|
def checked(command, *, log=None, timeout=30, env=None, cwd=None):
|
|
# Do not inherit application credentials into builds or print raw subprocess output.
|
|
environment = {key: os.environ[key] for key in ("PATH", "HOME", "DOCKER_CONFIG") if key in os.environ}
|
|
result = subprocess.run(command, env=environment if env is None else env, cwd=cwd, stdout=log or subprocess.PIPE,
|
|
stderr=log or subprocess.PIPE, text=log is None,
|
|
timeout=timeout, check=False)
|
|
if result.returncode:
|
|
raise RuntimeError("Bounded preparation command failed; inspect the protected attempt log")
|
|
return result.stdout if log is None else ""
|
|
|
|
|
|
def docker_json_template(fields):
|
|
# Go templates otherwise pass index/map/key as three separate json args.
|
|
return "\n".join("{{json (" + field + ")}}" for field in fields)
|
|
|
|
|
|
def original_identity(name, runner=checked):
|
|
fields = [".Id", ".Image", ".State.Running", ".State.Health.Status",
|
|
'index .Config.Labels "com.docker.compose.project"']
|
|
template = docker_json_template(fields)
|
|
values = runner(["docker", "inspect", "--format", template, name]).splitlines()
|
|
if len(values) != len(fields):
|
|
raise ValueError("Unexpected sanitized container inspection")
|
|
return dict(zip(("containerId", "imageId", "running", "health", "composeProject"),
|
|
(json.loads(value) for value in values)))
|
|
|
|
|
|
def assert_original(observed, slug, container_id, image_id):
|
|
if observed != {"containerId": container_id, "imageId": image_id,
|
|
"running": True, "health": "healthy", "composeProject": slug}:
|
|
raise ValueError("Original runtime identity/health drift; preparation refused")
|
|
|
|
|
|
def mount_inventory(target, runner=checked):
|
|
raw = json.loads(runner(["docker", "inspect", "--format", "{{json .Mounts}}", target]))
|
|
return sorted(({"type": item["Type"], "source": item["Source"],
|
|
"destination": item["Destination"], "rw": item["RW"]} for item in raw),
|
|
key=lambda item: item["destination"])
|
|
|
|
|
|
def expected_labels(metadata):
|
|
return {
|
|
"org.opencontainers.image.source": metadata["repository"],
|
|
"org.opencontainers.image.revision": metadata["revision"],
|
|
"org.opencontainers.image.source-tree": metadata["tree"],
|
|
"org.opencontainers.image.version": metadata["id"],
|
|
"org.opencontainers.image.created": metadata["builtAt"],
|
|
}
|
|
|
|
|
|
def verify_image(image, metadata, runner=checked):
|
|
image_id = runner(["docker", "image", "inspect", "--format", "{{.Id}}", image]).strip()
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise ValueError("Candidate immutable image identity is invalid")
|
|
for key, expected in expected_labels(metadata).items():
|
|
actual = runner(["docker", "image", "inspect", "--format",
|
|
'{{index .Config.Labels "' + key + '"}}', image_id]).strip()
|
|
if actual != expected:
|
|
raise ValueError("Candidate immutable provenance did not verify")
|
|
return image_id
|
|
|
|
|
|
def prepare(broker, slug, revision, attempt, container_id, image_id, runner=checked):
|
|
if (slug not in TARGETS or not SHA.fullmatch(revision) or not ATTEMPT.fullmatch(attempt)
|
|
or not ID.fullmatch(container_id) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id)):
|
|
raise ValueError("Invalid fixed target, commit, attempt or expected runtime identity")
|
|
repository = f"jens/{slug}"
|
|
original_name = TARGETS[slug][0]
|
|
workspace = require_real_child(JOURNAL_ROOT / slug / attempt, JOURNAL_ROOT)
|
|
source = require_real_child(broker.EXACT_SOURCE_ROOT / f"{slug}-migration-{attempt}", broker.EXACT_SOURCE_ROOT)
|
|
with ExitStack() as locks:
|
|
locks.enter_context(broker.exclusive_lock(broker.STATE_ROOT / "locks" / f"{slug}.lock", "migration project"))
|
|
locks.enter_context(broker.exclusive_lock(broker.GLOBAL_DEPLOY_LOCK, "migration preparation build"))
|
|
# Read config only after lock acquisition, unlike starting another broker process.
|
|
project = dict(broker.load_projects()[repository])
|
|
repo = broker.repository_path(repository, project)
|
|
broker.validate_commit(repo, str(project["branch"]), revision)
|
|
head = broker.output(["git", f"--git-dir={repo}", "rev-parse", f"refs/heads/{project['branch']}"])
|
|
if head != revision:
|
|
raise ValueError("Migration candidate must be the current configured branch head")
|
|
assert_original(original_identity(original_name, runner), slug, container_id, image_id)
|
|
if workspace.exists() or source.exists():
|
|
raise ValueError("Attempt artifacts already exist; retain them and review before a new attempt")
|
|
metadata = broker.deployment_metadata(repo, revision, repository)
|
|
if not broker.git_path_exists(repo, revision, "docker-compose.migration.yml"):
|
|
raise ValueError("Candidate commit does not contain the reviewed migration Compose contract")
|
|
if not broker.git_path_exists(repo, revision, DOCKERFILES[slug]):
|
|
raise ValueError("Candidate Dockerfile missing")
|
|
workspace.mkdir(parents=True, mode=0o700)
|
|
workspace.chmod(0o700)
|
|
journal_path = workspace / "prepare.json"
|
|
journal = {"schema": "legacy-prepare.v1", "phase": "preparing", "repository": repository,
|
|
"revision": revision, "metadata": metadata, "source": str(source),
|
|
"originalContainerId": container_id, "originalImageId": image_id,
|
|
"originalMounts": mount_inventory(container_id, runner),
|
|
"projectConfigSha256": hashlib.sha256(json.dumps(project, sort_keys=True).encode()).hexdigest(),
|
|
"stateSha256": hashlib.sha256((broker.STATE_ROOT / f"{slug}.json").read_bytes()).hexdigest(),
|
|
"originalProvenance": "unknown-legacy", "productionMutated": False}
|
|
durable_json(journal_path, journal)
|
|
try:
|
|
broker.sync_source(repo, revision, source, slug, exact=True)
|
|
if not broker.exact_source_matches_revision(repo, revision, source):
|
|
raise ValueError("Candidate source is not the exact committed tree")
|
|
from transaction import prepare_compose
|
|
candidate, command, cwd, environment, plan, nonce = prepare_compose(
|
|
broker, slug, revision, attempt, project, journal, workspace, runner)
|
|
image = environment["MIGRATION_IMAGE"]
|
|
journal.update(candidateProject=candidate, plan=plan, nonce=nonce)
|
|
journal["runtimeEnvSha256"] = hashlib.sha256((workspace / "runtime.env").read_bytes()).hexdigest()
|
|
journal.update(phase="building", candidateImage=image)
|
|
durable_json(journal_path, journal)
|
|
descriptor = os.open(workspace / "build.log", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(descriptor, "wb") as log:
|
|
runner(command, cwd=cwd, env=environment, log=log, timeout=3600)
|
|
log.flush()
|
|
os.fsync(log.fileno())
|
|
immutable = verify_image(image, metadata, runner)
|
|
if not broker.exact_source_matches_revision(repo, revision, source):
|
|
raise ValueError("Candidate source changed during build")
|
|
assert_original(original_identity(original_name, runner), slug, container_id, image_id)
|
|
if mount_inventory(container_id, runner) != journal["originalMounts"]:
|
|
raise ValueError("Original mount inventory changed during preparation")
|
|
journal.update(phase="build_verified", candidateImageId=immutable)
|
|
durable_json(journal_path, journal)
|
|
return {"phase": journal["phase"], "repository": repository, "revision": revision,
|
|
"candidateImageId": immutable, "journal": str(journal_path),
|
|
"productionMutated": False, "cutoverAvailable": True}
|
|
except Exception:
|
|
journal["phase"] = "preparation_failed"
|
|
durable_json(journal_path, journal)
|
|
raise
|
|
|
|
|
|
def load_broker():
|
|
if hashlib.sha256(BROKER_PATH.read_bytes().replace(b"\r\n", b"\n")).hexdigest() != BROKER_SHA:
|
|
raise ValueError("Live broker source differs from the reviewed artifact")
|
|
spec = importlib.util.spec_from_file_location("reviewed_deploy_broker", BROKER_PATH)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("mode", choices=["prepare", "cutover", "recover"])
|
|
parser.add_argument("slug", choices=sorted(TARGETS))
|
|
parser.add_argument("revision")
|
|
parser.add_argument("attempt")
|
|
parser.add_argument("--container-id", required=True)
|
|
parser.add_argument("--image-id", required=True)
|
|
parser.add_argument("--apply", action="store_true")
|
|
args = parser.parse_args()
|
|
if not args.apply:
|
|
print(json.dumps({"apply": False, "mode": args.mode, "productionMutated": False}))
|
|
return
|
|
if not hasattr(os, "geteuid") or os.geteuid() != 0:
|
|
raise SystemExit("Run only as root on the configured Unraid controller host")
|
|
try:
|
|
broker = load_broker()
|
|
if args.mode == "prepare":
|
|
result = prepare(broker, args.slug, args.revision, args.attempt, args.container_id, args.image_id)
|
|
else:
|
|
from transaction import operate
|
|
result = operate(broker, args.mode, args.slug, args.revision, args.attempt, args.container_id, args.image_id)
|
|
except Exception:
|
|
raise SystemExit("Operation failed. Inspect the protected attempt journal; do not assume production or rollback succeeded. Use explicit recover for an unfinished cutover.") from None
|
|
print(json.dumps(result))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|