#!/usr/bin/env python3 """Privileged one-shot worker for a previously validated browser OTA package.""" from __future__ import annotations import hashlib import json import os from pathlib import Path import shutil import signal import subprocess import sys import time import traceback from typing import Any import urllib.request from app.ota.diagnostics import DiagnosticLog, clear_failure_log, persist_failure_log, sha256_file from app.ota.package import extract_payload, inspect_package from app.ota.state import read_json, utc_now, write_json, write_last_result from app.ota.versioning import SoftwareVersion from app.system.kernel_recovery import CANDIDATE_RELEASE, KernelRecovery TARGET = Path("/opt/matrix-screen-controller") RELEASES = Path("/opt/matrix-screen-controller.releases") DATA_ROOT = Path("/var/lib/matrix-screen-controller") RUNTIME_ROOT = Path("/run/matrix-screen-controller") OTA_WORK_ROOT = Path("/opt/matrix-screen-controller-ota") UNIT_PATH = Path("/etc/systemd/system/matrix-screen-controller.service") SERVICE = "matrix-screen-controller.service" REQUEST_PATH = RUNTIME_ROOT / "ota-request.json" class UpdateFailed(RuntimeError): pass def verify_kernel_capability_retained( before: dict[str, Any], after: dict[str, Any], performance_before: dict[str, Any], performance_after: dict[str, Any], ) -> None: previously_healthy = ( before.get("release") == CANDIDATE_RELEASE and before.get("available") is True and before.get("marker") is True ) if not previously_healthy: return if (after["release"] != CANDIDATE_RELEASE or not after["available"] or not after["marker"] or set(after["policies"]) != set(before["policies"])): raise UpdateFailed("OTA lost the validated candidate kernel or cpufreq policy") requested = performance_before.get("requested") if (performance_after.get("available") is not True or performance_after.get("requested") is not requested or (requested is True and performance_after.get("effective") is not True)): raise UpdateFailed("OTA changed performance mode availability or effective state") if requested is False and performance_after.get("current_governors") != { name: value["current"] for name, value in before["policies"].items() }: raise UpdateFailed("OTA did not restore the original CPU governors") def snapshot(root: Path, *, ignored: set[str] | None = None) -> dict[str, tuple[int, str]]: records: dict[str, tuple[int, str]] = {} ignored = ignored or set() if not root.exists(): return records for path in sorted((item for item in root.rglob("*") if item.is_file()), key=lambda item: item.as_posix()): relative = path.relative_to(root).as_posix() if relative in ignored: continue digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) records[relative] = (path.stat().st_size, digest.hexdigest()) return records class Transaction: def __init__(self, request: dict[str, Any]) -> None: self.request = request self.job_id = checked_token(request.get("job_id"), "job_id") self.target_version = SoftwareVersion.parse(str(request.get("target_version"))) self.current_version = SoftwareVersion.parse(str(request.get("current_version"))) if self.target_version <= self.current_version: raise UpdateFailed("target version is not newer than the installed version") self.package_path = Path(str(request.get("package_path"))) self.status_path = Path(str(request.get("status_path"))) if self.status_path != RUNTIME_ROOT / "ota-status.json": raise UpdateFailed("unexpected OTA status path") self.work_root = OTA_WORK_ROOT / f"work.{self.job_id}" self.extract_root = self.work_root / "extracted" self.release = RELEASES / f"{self.target_version}-{self.job_id}" self.data_candidate = DATA_ROOT.with_name(f"matrix-screen-controller.ota.{self.job_id}") self.data_backup = DATA_ROOT.with_name(f"matrix-screen-controller.rollback.{self.job_id}") self.unit_backup = self.work_root / "matrix-screen-controller.service.old" self.previous_target: Path | None = None self.previous_directory: Path | None = None self.program_swapped = False self.data_swapped = False self.visual: subprocess.Popen[bytes] | None = None self.diagnostics = DiagnosticLog(RUNTIME_ROOT / "ota-worker.log") self.diagnostics.reset() self.diagnostics.section("更新任务") self.diagnostics.write( f"job_id={self.job_id} current_version={self.current_version} " f"target_version={self.target_version} packaged_at={request.get('packaged_at') or ''}" ) self.kernel_recovery = KernelRecovery(DATA_ROOT) self.kernel_before: dict[str, Any] | None = None self.performance_before: dict[str, Any] | None = None self.job = { "id": self.job_id, "target_version": str(self.target_version), "packaged_at": str(request.get("packaged_at") or ""), "phase": "preparing", "percent": 8, "message": "正在准备更新", "started_at": str(request.get("accepted_at") or utc_now()), "finished_at": None, "error": None, } def update(self, phase: str, percent: int, message: str) -> None: self.diagnostics.write(f"phase={phase} percent={percent} message={message}") self.job.update(phase=phase, percent=max(0, min(100, percent)), message=message) write_json(self.status_path, {"schema_version": 1, "active": True, "job": self.job}) def run( self, command: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, check: bool = True, failure_label: str | None = None, log_output: bool = True, ) -> subprocess.CompletedProcess[bytes]: return self.diagnostics.run( command, cwd=cwd, env=env, check=check, failure_label=failure_label, log_output=log_output, ) def prepare(self) -> None: self.kernel_before = self.kernel_recovery.capability() with urllib.request.urlopen("http://127.0.0.1:8080/api/status", timeout=8) as response: self.performance_before = json.loads(response.read().decode("utf-8"))["system"]["performance_mode"] self.diagnostics.write( f"kernel_before={self.kernel_before['release']} " f"cpufreq_available={self.kernel_before['available']} marker={self.kernel_before['marker']}" ) if any(path.exists() for path in (self.work_root, self.release, self.data_candidate, self.data_backup)): raise UpdateFailed("OTA transaction paths already exist") self.work_root.mkdir(parents=True, mode=0o700) test_root = self.work_root / "test-root" test_tmp = self.work_root / "test-tmp" test_root.mkdir(parents=True, mode=0o700) test_tmp.mkdir(parents=True, mode=0o700) self.diagnostics.host_summary(test_root=test_root) self.diagnostics.section("更新包校验") self.diagnostics.write(f"package={self.package_path.name} sha256={sha256_file(self.package_path)}") package = inspect_package(self.package_path, current_version=self.current_version) self.diagnostics.write( f"package_valid=true target_version={package.target_version} expanded_bytes={package.expanded_bytes}" ) if package.target_version != self.target_version: raise UpdateFailed("request and package versions do not match") stat = shutil.disk_usage("/opt") required = max(package.expanded_bytes * 4, 512 * 1024 * 1024) if stat.free < required: raise UpdateFailed("not enough free space to stage and validate the OTA release") self.update("extracting", 12, "正在解压全量更新包") extract_payload(package, self.extract_root) software = self.extract_root / "software" wheelhouse = self.extract_root / "wheelhouse" frpc_bundle = software / "system-dependencies/frpc" if SoftwareVersion.parse((software / "VERSION").read_text(encoding="utf-8")) != self.target_version: raise UpdateFailed("extracted software VERSION does not match the package") self.update("dependencies", 22, "正在校验离线依赖并创建运行环境") self.run(["sha256sum", "-c", "SHA256SUMS"], cwd=wheelhouse, failure_label="离线 wheel 摘要校验") if frpc_bundle.is_dir(): self.run(["sha256sum", "-c", "SHA256SUMS"], cwd=frpc_bundle, failure_label="frpc 系统依赖摘要校验") shutil.move(str(software), self.release) self.run(["python3", "--version"], failure_label="系统 Python 版本检查") self.run(["python3", "-m", "venv", str(self.release / ".venv")], failure_label="创建虚拟环境") self.run([ str(self.release / ".venv/bin/pip"), "install", "--no-index", "--disable-pip-version-check", "--find-links", str(wheelhouse), "-r", str(self.release / "requirements-dev.txt"), ], failure_label="安装离线依赖") self.run([str(self.release / ".venv/bin/python"), "-m", "pip", "--version"], failure_label="pip 版本检查") self.run([str(self.release / ".venv/bin/python"), "-m", "pytest", "--version"], failure_label="pytest 版本检查") self.update("building", 42, "正在编译并测试屏幕驱动") self.run( ["make", "-C", str(self.release / "app/display/native"), "clean", "all", "test"], failure_label="原生驱动编译与测试", ) self.run( [str(self.release / ".venv/bin/python"), "-m", "compileall", "-q", str(self.release / "app")], failure_label="Python 静态编译", ) self.update("testing", 55, "正在运行新版本自动化测试") test_env = os.environ.copy() test_env.update( MATRIX_DRIVER="mock", MATRIX_TEST_ROOT=str(test_root), MATRIX_DATA_DIR=str(test_root / "data"), MATRIX_RUNTIME_DIR=str(test_root / "runtime"), MATRIX_SOURCE_ONLY_UPDATE_TESTS="1", TMPDIR=str(test_tmp), TEMP=str(test_tmp), TMP=str(test_tmp), ) self.run([ str(self.release / ".venv/bin/python"), "-m", "pytest", "-p", "no:cacheprovider", "-q", f"--basetemp={self.work_root / 'pytest'}", "--tb=short", "--disable-warnings", ], cwd=self.release, env=test_env, failure_label="新版本 pytest") self.run( [str(self.release / ".venv/bin/python"), str(self.release / "scripts/dedicated_host.py"), "check"], failure_label="专用主机检查", ) self.update("migrating", 68, "正在迁移用户数据副本") shutil.copytree(DATA_ROOT, self.data_candidate, symlinks=True) runtime_candidate = self.work_root / "migration-runtime" migration_env = os.environ.copy() migration_env["PYTHONPATH"] = str(self.release) self.run([ str(self.release / ".venv/bin/python"), "-m", "scripts.prepare_data_root", "--data-root", str(self.data_candidate), "--runtime-root", str(runtime_candidate), ], cwd=self.release, env=migration_env, failure_label="用户数据副本迁移") def switch(self) -> None: self.update("switching", 78, "正在切换到新版本") shutil.copy2(UNIT_PATH, self.unit_backup) self.run(["systemctl", "stop", SERVICE], failure_label="停止旧服务") self.start_visual() RELEASES.mkdir(parents=True, exist_ok=True) if TARGET.is_symlink(): self.previous_target = Path(os.readlink(TARGET)) TARGET.unlink() elif TARGET.is_dir(): self.previous_directory = RELEASES / f"{self.current_version}-pre-ota-{self.job_id}" TARGET.rename(self.previous_directory) else: raise UpdateFailed("production target is neither a release symlink nor a directory") os.symlink(self.release, TARGET, target_is_directory=True) self.program_swapped = True DATA_ROOT.rename(self.data_backup) self.data_candidate.rename(DATA_ROOT) self.data_swapped = True shutil.copy2(self.release / "systemd/matrix-screen-controller.service", UNIT_PATH) shutil.copy2(self.release / "systemd/matrix-screen-controller-ota.service", Path("/etc/systemd/system/matrix-screen-controller-ota.service")) # prepare_data_root owns the component transaction, including for legacy # workers. Its independent guardian commits after our final health result. self.run(["systemctl", "daemon-reload"], failure_label="重载 systemd unit") self.stop_visual() self.run(["systemctl", "start", SERVICE], failure_label="启动新服务") def verify(self) -> None: self.update("verifying", 90, "正在验证新版本和真实屏幕驱动") deadline = time.monotonic() + 45 last_error = "service did not respond" attempt = 0 while time.monotonic() < deadline: attempt += 1 try: response = self.run( [ "curl", "--fail", "--silent", "--max-time", "5", "http://127.0.0.1:8080/api/status", ], check=False, failure_label=f"健康检查第 {attempt} 次", log_output=False, ) if response.returncode != 0: raise UpdateFailed(f"health endpoint returned exit code {response.returncode}") status = json.loads(response.stdout.decode("utf-8", errors="strict")) screen = status.get("screen", {}) driver = screen.get("driver_status") or {} if status.get("service", {}).get("software_version") != str(self.target_version): raise UpdateFailed("new service reports the wrong software version") if screen.get("driver") != "walnutpi-h618-hub75": raise UpdateFailed("new service did not start the production HUB75 driver") if screen.get("hardware_mapping") != "walnutpi-pi-bank-pwm-oe-v2": raise UpdateFailed("new service reports the wrong hardware mapping") if driver.get("oe_timing_backend") != "h618-pwm4" or driver.get("oe_pulse_faults") != 0: raise UpdateFailed("new service did not pass the PWM4 OE health check") before = self.kernel_before or {} if (before.get("available") and before.get("marker") and before.get("release") == CANDIDATE_RELEASE): verify_kernel_capability_retained( before, self.kernel_recovery.capability(), self.performance_before or {}, (status.get("system") or {}).get("performance_mode") or {}, ) else: self.diagnostics.write("kernel_preexisting_degraded=true; application update may proceed") self.diagnostics.write( "health_summary=" f"version:{status.get('service', {}).get('software_version')} " f"driver:{screen.get('driver')} mapping:{screen.get('hardware_mapping')} " f"oe_backend:{driver.get('oe_timing_backend')} oe_pulse_faults:{driver.get('oe_pulse_faults')}" ) self.diagnostics.write(f"health_check_passed=true attempts={attempt}") return except (UpdateFailed, OSError, UnicodeError, json.JSONDecodeError) as exc: last_error = str(exc) self.diagnostics.write(f"health_check_attempt={attempt} failed={exc}") time.sleep(1) raise UpdateFailed(f"new release health check timed out: {last_error}") def succeed(self) -> None: self.diagnostics.section("更新成功") self.diagnostics.write("health_check=passed rollback_required=false") result = { "status": "success", "target_version": str(self.target_version), "packaged_at": str(self.request.get("packaged_at") or ""), "installed_at": utc_now(), "release_notes": str(self.request.get("release_notes") or ""), "error": None, } clear_failure_log(DATA_ROOT) write_last_result(DATA_ROOT, result) self.job.update( phase="complete", percent=100, message="更新完成", finished_at=result["installed_at"], error=None, ) write_json(self.status_path, {"schema_version": 1, "active": False, "job": self.job}) self.package_path.unlink(missing_ok=True) self.request_path().unlink(missing_ok=True) shutil.rmtree(self.data_backup, ignore_errors=True) if self.previous_directory is not None: shutil.rmtree(self.previous_directory, ignore_errors=True) if self.previous_target is not None: previous_release = self.previous_target if self.previous_target.is_absolute() else TARGET.parent / self.previous_target if previous_release.parent == RELEASES: shutil.rmtree(previous_release, ignore_errors=True) shutil.rmtree(self.work_root, ignore_errors=True) prune_empty(self.package_path.parent) self.diagnostics.path.unlink(missing_ok=True) def rollback(self, error: BaseException) -> None: self.diagnostics.section("失败与回滚") self.diagnostics.write(f"failure={error.__class__.__name__}: {error}") self.diagnostics.write( "traceback_short:\n" + "".join(traceback.format_exception(error, limit=8)).rstrip() ) rollback_errors: list[str] = [] def attempt(label: str, action) -> None: try: action() self.diagnostics.write(f"rollback_step={label} result=ok") except BaseException as rollback_error: detail = f"{label}: {rollback_error.__class__.__name__}: {rollback_error}" rollback_errors.append(detail) self.diagnostics.write(f"rollback_step={label} result=failed detail={detail}") attempt("停止独立显示进程", self.stop_visual) if self.program_swapped or self.data_swapped: attempt( "停止待回滚服务", lambda: self.run(["systemctl", "stop", SERVICE], check=False, failure_label="回滚停止服务"), ) if self.data_swapped: failed_data = DATA_ROOT.with_name(f"matrix-screen-controller.failed.{self.job_id}") if DATA_ROOT.exists(): attempt("移开失败数据", lambda: DATA_ROOT.rename(failed_data)) if self.data_backup.exists(): attempt("恢复原数据", lambda: self.data_backup.rename(DATA_ROOT)) attempt("清理失败数据", lambda: shutil.rmtree(failed_data, ignore_errors=True)) if self.program_swapped: if TARGET.is_symlink(): attempt("移除失败版本链接", TARGET.unlink) if self.previous_directory is not None and self.previous_directory.exists(): attempt("恢复原程序目录", lambda: self.previous_directory.rename(TARGET)) elif self.previous_target is not None: attempt( "恢复原程序链接", lambda: os.symlink(self.previous_target, TARGET, target_is_directory=True), ) if self.unit_backup.exists(): attempt("恢复原 systemd unit", lambda: shutil.copy2(self.unit_backup, UNIT_PATH)) attempt( "回滚后重载 systemd", lambda: self.run(["systemctl", "daemon-reload"], check=False, failure_label="回滚重载 systemd"), ) if self.program_swapped or self.data_swapped: attempt( "启动原服务", lambda: self.run(["systemctl", "start", SERVICE], check=False, failure_label="回滚启动原服务"), ) attempt("清理失败程序候选", lambda: shutil.rmtree(self.release, ignore_errors=True)) attempt("清理数据候选", lambda: shutil.rmtree(self.data_candidate, ignore_errors=True)) recovery = "回滚完成" if not rollback_errors else "回滚存在错误：" + "；".join(rollback_errors) self.diagnostics.write(f"final_recovery_state={recovery}") brief_error = str(error).splitlines()[0][:1000] or error.__class__.__name__ result = { "status": "failed", "target_version": str(self.target_version), "packaged_at": str(self.request.get("packaged_at") or ""), "installed_at": utc_now(), "release_notes": str(self.request.get("release_notes") or ""), "error": brief_error, } try: self.diagnostics.persist(DATA_ROOT) self.diagnostics.write("persistent_failure_log=written") except OSError as persist_error: self.diagnostics.write( f"persistent_failure_log=failed detail={persist_error.__class__.__name__}: {persist_error}" ) try: write_last_result(DATA_ROOT, result) except OSError: pass self.job.update( phase="failed", percent=0, message="更新失败，已恢复原版本", finished_at=result["installed_at"], error=brief_error, ) write_json(self.status_path, {"schema_version": 1, "active": False, "job": self.job}) self.package_path.unlink(missing_ok=True) self.request_path().unlink(missing_ok=True) shutil.rmtree(self.work_root, ignore_errors=True) self.diagnostics.path.unlink(missing_ok=True) def start_visual(self) -> None: self.diagnostics.section("启动独立更新显示") python = TARGET / ".venv/bin/python" self.visual = subprocess.Popen([ str(python), "-m", "scripts.hub75_visual_hold", "--mode", "ota", "--brightness", "40", "--progress-file", str(self.status_path), "--orientation", str(int(self.request.get("orientation", 0))), ]) time.sleep(1) if self.visual.poll() is not None: raise UpdateFailed("independent OTA display process failed to start") self.diagnostics.write("ota_visual_process=running") def stop_visual(self) -> None: if self.visual is None: return if self.visual.poll() is None: self.visual.send_signal(signal.SIGTERM) try: self.visual.wait(timeout=5) except subprocess.TimeoutExpired: self.visual.kill() self.visual.wait(timeout=2) self.visual = None self.diagnostics.write("ota_visual_process=stopped") def request_path(self) -> Path: return Path(str(self.request.get("request_path") or REQUEST_PATH)) def checked_token(value: Any, name: str) -> str: candidate = str(value or "") if not candidate or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-" for character in candidate): raise UpdateFailed(f"invalid {name}") return candidate def prune_empty(path: Path) -> None: try: path.rmdir() except OSError: pass def main() -> int: if os.geteuid() != 0 or os.uname().machine != "aarch64": print("OTA worker must run as root on the WalnutPi AArch64 host", file=sys.stderr) return 2 request = read_json(REQUEST_PATH) if request is None or request.get("schema_version") != 1: print("OTA request is missing or invalid", file=sys.stderr) return 2 transaction: Transaction | None = None try: transaction = Transaction(request) transaction.prepare() transaction.switch() transaction.verify() transaction.succeed() return 0 except BaseException as exc: if transaction is not None: transaction.rollback(exc) print(f"OTA update failed: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())