483 lines
22 KiB
Python
483 lines
22 KiB
Python
#!/usr/bin/env python3
|
||
"""Privileged one-shot worker for a previously validated browser OTA package."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import shutil
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import traceback
|
||
from typing import Any
|
||
|
||
from app.ota.diagnostics import DiagnosticLog, clear_failure_log, persist_failure_log, sha256_file
|
||
from app.ota.package import extract_payload, inspect_package
|
||
from app.ota.state import read_json, utc_now, write_json, write_last_result
|
||
from app.ota.versioning import SoftwareVersion
|
||
|
||
TARGET = Path("/opt/matrix-screen-controller")
|
||
RELEASES = Path("/opt/matrix-screen-controller.releases")
|
||
DATA_ROOT = Path("/var/lib/matrix-screen-controller")
|
||
RUNTIME_ROOT = Path("/run/matrix-screen-controller")
|
||
OTA_WORK_ROOT = Path("/opt/matrix-screen-controller-ota")
|
||
UNIT_PATH = Path("/etc/systemd/system/matrix-screen-controller.service")
|
||
SERVICE = "matrix-screen-controller.service"
|
||
REQUEST_PATH = RUNTIME_ROOT / "ota-request.json"
|
||
|
||
|
||
class UpdateFailed(RuntimeError):
|
||
pass
|
||
|
||
|
||
def snapshot(root: Path, *, ignored: set[str] | None = None) -> dict[str, tuple[int, str]]:
|
||
records: dict[str, tuple[int, str]] = {}
|
||
ignored = ignored or set()
|
||
if not root.exists():
|
||
return records
|
||
for path in sorted((item for item in root.rglob("*") if item.is_file()), key=lambda item: item.as_posix()):
|
||
relative = path.relative_to(root).as_posix()
|
||
if relative in ignored:
|
||
continue
|
||
digest = hashlib.sha256()
|
||
with path.open("rb") as handle:
|
||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||
digest.update(chunk)
|
||
records[relative] = (path.stat().st_size, digest.hexdigest())
|
||
return records
|
||
|
||
|
||
class Transaction:
|
||
def __init__(self, request: dict[str, Any]) -> None:
|
||
self.request = request
|
||
self.job_id = checked_token(request.get("job_id"), "job_id")
|
||
self.target_version = SoftwareVersion.parse(str(request.get("target_version")))
|
||
self.current_version = SoftwareVersion.parse(str(request.get("current_version")))
|
||
if self.target_version <= self.current_version:
|
||
raise UpdateFailed("target version is not newer than the installed version")
|
||
self.package_path = Path(str(request.get("package_path")))
|
||
self.status_path = Path(str(request.get("status_path")))
|
||
if self.status_path != RUNTIME_ROOT / "ota-status.json":
|
||
raise UpdateFailed("unexpected OTA status path")
|
||
self.work_root = OTA_WORK_ROOT / f"work.{self.job_id}"
|
||
self.extract_root = self.work_root / "extracted"
|
||
self.release = RELEASES / f"{self.target_version}-{self.job_id}"
|
||
self.data_candidate = DATA_ROOT.with_name(f"matrix-screen-controller.ota.{self.job_id}")
|
||
self.data_backup = DATA_ROOT.with_name(f"matrix-screen-controller.rollback.{self.job_id}")
|
||
self.unit_backup = self.work_root / "matrix-screen-controller.service.old"
|
||
self.previous_target: Path | None = None
|
||
self.previous_directory: Path | None = None
|
||
self.program_swapped = False
|
||
self.data_swapped = False
|
||
self.visual: subprocess.Popen[bytes] | None = None
|
||
self.diagnostics = DiagnosticLog(RUNTIME_ROOT / "ota-worker.log")
|
||
self.diagnostics.reset()
|
||
self.diagnostics.section("更新任务")
|
||
self.diagnostics.write(
|
||
f"job_id={self.job_id} current_version={self.current_version} "
|
||
f"target_version={self.target_version} packaged_at={request.get('packaged_at') or ''}"
|
||
)
|
||
self.job = {
|
||
"id": self.job_id,
|
||
"target_version": str(self.target_version),
|
||
"packaged_at": str(request.get("packaged_at") or ""),
|
||
"phase": "preparing",
|
||
"percent": 8,
|
||
"message": "正在准备更新",
|
||
"started_at": str(request.get("accepted_at") or utc_now()),
|
||
"finished_at": None,
|
||
"error": None,
|
||
}
|
||
|
||
def update(self, phase: str, percent: int, message: str) -> None:
|
||
self.diagnostics.write(f"phase={phase} percent={percent} message={message}")
|
||
self.job.update(phase=phase, percent=max(0, min(100, percent)), message=message)
|
||
write_json(self.status_path, {"schema_version": 1, "active": True, "job": self.job})
|
||
|
||
def run(
|
||
self,
|
||
command: list[str],
|
||
*,
|
||
cwd: Path | None = None,
|
||
env: dict[str, str] | None = None,
|
||
check: bool = True,
|
||
failure_label: str | None = None,
|
||
log_output: bool = True,
|
||
) -> subprocess.CompletedProcess[bytes]:
|
||
return self.diagnostics.run(
|
||
command,
|
||
cwd=cwd,
|
||
env=env,
|
||
check=check,
|
||
failure_label=failure_label,
|
||
log_output=log_output,
|
||
)
|
||
|
||
def prepare(self) -> None:
|
||
if any(path.exists() for path in (self.work_root, self.release, self.data_candidate, self.data_backup)):
|
||
raise UpdateFailed("OTA transaction paths already exist")
|
||
self.work_root.mkdir(parents=True, mode=0o700)
|
||
test_root = self.work_root / "test-root"
|
||
test_tmp = self.work_root / "test-tmp"
|
||
test_root.mkdir(parents=True, mode=0o700)
|
||
test_tmp.mkdir(parents=True, mode=0o700)
|
||
self.diagnostics.host_summary(test_root=test_root)
|
||
self.diagnostics.section("更新包校验")
|
||
self.diagnostics.write(f"package={self.package_path.name} sha256={sha256_file(self.package_path)}")
|
||
package = inspect_package(self.package_path, current_version=self.current_version)
|
||
self.diagnostics.write(
|
||
f"package_valid=true target_version={package.target_version} expanded_bytes={package.expanded_bytes}"
|
||
)
|
||
if package.target_version != self.target_version:
|
||
raise UpdateFailed("request and package versions do not match")
|
||
stat = shutil.disk_usage("/opt")
|
||
required = max(package.expanded_bytes * 4, 512 * 1024 * 1024)
|
||
if stat.free < required:
|
||
raise UpdateFailed("not enough free space to stage and validate the OTA release")
|
||
|
||
self.update("extracting", 12, "正在解压全量更新包")
|
||
extract_payload(package, self.extract_root)
|
||
software = self.extract_root / "software"
|
||
wheelhouse = self.extract_root / "wheelhouse"
|
||
frpc_bundle = software / "system-dependencies/frpc"
|
||
if SoftwareVersion.parse((software / "VERSION").read_text(encoding="utf-8")) != self.target_version:
|
||
raise UpdateFailed("extracted software VERSION does not match the package")
|
||
|
||
self.update("dependencies", 22, "正在校验离线依赖并创建运行环境")
|
||
self.run(["sha256sum", "-c", "SHA256SUMS"], cwd=wheelhouse, failure_label="离线 wheel 摘要校验")
|
||
if frpc_bundle.is_dir():
|
||
self.run(["sha256sum", "-c", "SHA256SUMS"], cwd=frpc_bundle, failure_label="frpc 系统依赖摘要校验")
|
||
shutil.move(str(software), self.release)
|
||
self.run(["python3", "--version"], failure_label="系统 Python 版本检查")
|
||
self.run(["python3", "-m", "venv", str(self.release / ".venv")], failure_label="创建虚拟环境")
|
||
self.run([
|
||
str(self.release / ".venv/bin/pip"),
|
||
"install",
|
||
"--no-index",
|
||
"--disable-pip-version-check",
|
||
"--find-links",
|
||
str(wheelhouse),
|
||
"-r",
|
||
str(self.release / "requirements-dev.txt"),
|
||
], failure_label="安装离线依赖")
|
||
self.run([str(self.release / ".venv/bin/python"), "-m", "pip", "--version"], failure_label="pip 版本检查")
|
||
self.run([str(self.release / ".venv/bin/python"), "-m", "pytest", "--version"], failure_label="pytest 版本检查")
|
||
|
||
self.update("building", 42, "正在编译并测试屏幕驱动")
|
||
self.run(
|
||
["make", "-C", str(self.release / "app/display/native"), "clean", "all", "test"],
|
||
failure_label="原生驱动编译与测试",
|
||
)
|
||
self.run(
|
||
[str(self.release / ".venv/bin/python"), "-m", "compileall", "-q", str(self.release / "app")],
|
||
failure_label="Python 静态编译",
|
||
)
|
||
|
||
self.update("testing", 55, "正在运行新版本自动化测试")
|
||
test_env = os.environ.copy()
|
||
test_env.update(
|
||
MATRIX_DRIVER="mock",
|
||
MATRIX_TEST_ROOT=str(test_root),
|
||
MATRIX_DATA_DIR=str(test_root / "data"),
|
||
MATRIX_RUNTIME_DIR=str(test_root / "runtime"),
|
||
MATRIX_SOURCE_ONLY_UPDATE_TESTS="1",
|
||
TMPDIR=str(test_tmp),
|
||
TEMP=str(test_tmp),
|
||
TMP=str(test_tmp),
|
||
)
|
||
self.run([
|
||
str(self.release / ".venv/bin/python"), "-m", "pytest",
|
||
"-p", "no:cacheprovider", "-q", f"--basetemp={self.work_root / 'pytest'}",
|
||
"--tb=short", "--disable-warnings",
|
||
], cwd=self.release, env=test_env, failure_label="新版本 pytest")
|
||
self.run(
|
||
[str(self.release / ".venv/bin/python"), str(self.release / "scripts/dedicated_host.py"), "check"],
|
||
failure_label="专用主机检查",
|
||
)
|
||
|
||
self.update("migrating", 68, "正在迁移用户数据副本")
|
||
shutil.copytree(DATA_ROOT, self.data_candidate, symlinks=True)
|
||
runtime_candidate = self.work_root / "migration-runtime"
|
||
migration_env = os.environ.copy()
|
||
migration_env["PYTHONPATH"] = str(self.release)
|
||
self.run([
|
||
str(self.release / ".venv/bin/python"),
|
||
"-m",
|
||
"scripts.prepare_data_root",
|
||
"--data-root",
|
||
str(self.data_candidate),
|
||
"--runtime-root",
|
||
str(runtime_candidate),
|
||
], cwd=self.release, env=migration_env, failure_label="用户数据副本迁移")
|
||
|
||
def switch(self) -> None:
|
||
self.update("switching", 78, "正在切换到新版本")
|
||
shutil.copy2(UNIT_PATH, self.unit_backup)
|
||
self.run(["systemctl", "stop", SERVICE], failure_label="停止旧服务")
|
||
self.start_visual()
|
||
|
||
RELEASES.mkdir(parents=True, exist_ok=True)
|
||
if TARGET.is_symlink():
|
||
self.previous_target = Path(os.readlink(TARGET))
|
||
TARGET.unlink()
|
||
elif TARGET.is_dir():
|
||
self.previous_directory = RELEASES / f"{self.current_version}-pre-ota-{self.job_id}"
|
||
TARGET.rename(self.previous_directory)
|
||
else:
|
||
raise UpdateFailed("production target is neither a release symlink nor a directory")
|
||
os.symlink(self.release, TARGET, target_is_directory=True)
|
||
self.program_swapped = True
|
||
|
||
DATA_ROOT.rename(self.data_backup)
|
||
self.data_candidate.rename(DATA_ROOT)
|
||
self.data_swapped = True
|
||
shutil.copy2(self.release / "systemd/matrix-screen-controller.service", UNIT_PATH)
|
||
shutil.copy2(self.release / "systemd/matrix-screen-controller-ota.service", Path("/etc/systemd/system/matrix-screen-controller-ota.service"))
|
||
# prepare_data_root owns the component transaction, including for legacy
|
||
# workers. Its independent guardian commits after our final health result.
|
||
self.run(["systemctl", "daemon-reload"], failure_label="重载 systemd unit")
|
||
self.stop_visual()
|
||
self.run(["systemctl", "start", SERVICE], failure_label="启动新服务")
|
||
|
||
def verify(self) -> None:
|
||
self.update("verifying", 90, "正在验证新版本和真实屏幕驱动")
|
||
deadline = time.monotonic() + 45
|
||
last_error = "service did not respond"
|
||
attempt = 0
|
||
while time.monotonic() < deadline:
|
||
attempt += 1
|
||
try:
|
||
response = self.run(
|
||
[
|
||
"curl", "--fail", "--silent", "--max-time", "5",
|
||
"http://127.0.0.1:8080/api/status",
|
||
],
|
||
check=False,
|
||
failure_label=f"健康检查第 {attempt} 次",
|
||
log_output=False,
|
||
)
|
||
if response.returncode != 0:
|
||
raise UpdateFailed(f"health endpoint returned exit code {response.returncode}")
|
||
status = json.loads(response.stdout.decode("utf-8", errors="strict"))
|
||
screen = status.get("screen", {})
|
||
driver = screen.get("driver_status") or {}
|
||
if status.get("service", {}).get("software_version") != str(self.target_version):
|
||
raise UpdateFailed("new service reports the wrong software version")
|
||
if screen.get("driver") != "walnutpi-h618-hub75":
|
||
raise UpdateFailed("new service did not start the production HUB75 driver")
|
||
if screen.get("hardware_mapping") != "walnutpi-pi-bank-pwm-oe-v2":
|
||
raise UpdateFailed("new service reports the wrong hardware mapping")
|
||
if driver.get("oe_timing_backend") != "h618-pwm4" or driver.get("oe_pulse_faults") != 0:
|
||
raise UpdateFailed("new service did not pass the PWM4 OE health check")
|
||
self.diagnostics.write(
|
||
"health_summary="
|
||
f"version:{status.get('service', {}).get('software_version')} "
|
||
f"driver:{screen.get('driver')} mapping:{screen.get('hardware_mapping')} "
|
||
f"oe_backend:{driver.get('oe_timing_backend')} oe_pulse_faults:{driver.get('oe_pulse_faults')}"
|
||
)
|
||
self.diagnostics.write(f"health_check_passed=true attempts={attempt}")
|
||
return
|
||
except (UpdateFailed, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||
last_error = str(exc)
|
||
self.diagnostics.write(f"health_check_attempt={attempt} failed={exc}")
|
||
time.sleep(1)
|
||
raise UpdateFailed(f"new release health check timed out: {last_error}")
|
||
|
||
def succeed(self) -> None:
|
||
self.diagnostics.section("更新成功")
|
||
self.diagnostics.write("health_check=passed rollback_required=false")
|
||
result = {
|
||
"status": "success",
|
||
"target_version": str(self.target_version),
|
||
"packaged_at": str(self.request.get("packaged_at") or ""),
|
||
"installed_at": utc_now(),
|
||
"release_notes": str(self.request.get("release_notes") or ""),
|
||
"error": None,
|
||
}
|
||
clear_failure_log(DATA_ROOT)
|
||
write_last_result(DATA_ROOT, result)
|
||
self.job.update(
|
||
phase="complete",
|
||
percent=100,
|
||
message="更新完成",
|
||
finished_at=result["installed_at"],
|
||
error=None,
|
||
)
|
||
write_json(self.status_path, {"schema_version": 1, "active": False, "job": self.job})
|
||
self.package_path.unlink(missing_ok=True)
|
||
self.request_path().unlink(missing_ok=True)
|
||
shutil.rmtree(self.data_backup, ignore_errors=True)
|
||
if self.previous_directory is not None:
|
||
shutil.rmtree(self.previous_directory, ignore_errors=True)
|
||
if self.previous_target is not None:
|
||
previous_release = self.previous_target if self.previous_target.is_absolute() else TARGET.parent / self.previous_target
|
||
if previous_release.parent == RELEASES:
|
||
shutil.rmtree(previous_release, ignore_errors=True)
|
||
shutil.rmtree(self.work_root, ignore_errors=True)
|
||
prune_empty(self.package_path.parent)
|
||
self.diagnostics.path.unlink(missing_ok=True)
|
||
|
||
def rollback(self, error: BaseException) -> None:
|
||
self.diagnostics.section("失败与回滚")
|
||
self.diagnostics.write(f"failure={error.__class__.__name__}: {error}")
|
||
self.diagnostics.write(
|
||
"traceback_short:\n" + "".join(traceback.format_exception(error, limit=8)).rstrip()
|
||
)
|
||
rollback_errors: list[str] = []
|
||
|
||
def attempt(label: str, action) -> None:
|
||
try:
|
||
action()
|
||
self.diagnostics.write(f"rollback_step={label} result=ok")
|
||
except BaseException as rollback_error:
|
||
detail = f"{label}: {rollback_error.__class__.__name__}: {rollback_error}"
|
||
rollback_errors.append(detail)
|
||
self.diagnostics.write(f"rollback_step={label} result=failed detail={detail}")
|
||
|
||
attempt("停止独立显示进程", self.stop_visual)
|
||
if self.program_swapped or self.data_swapped:
|
||
attempt(
|
||
"停止待回滚服务",
|
||
lambda: self.run(["systemctl", "stop", SERVICE], check=False, failure_label="回滚停止服务"),
|
||
)
|
||
if self.data_swapped:
|
||
failed_data = DATA_ROOT.with_name(f"matrix-screen-controller.failed.{self.job_id}")
|
||
if DATA_ROOT.exists():
|
||
attempt("移开失败数据", lambda: DATA_ROOT.rename(failed_data))
|
||
if self.data_backup.exists():
|
||
attempt("恢复原数据", lambda: self.data_backup.rename(DATA_ROOT))
|
||
attempt("清理失败数据", lambda: shutil.rmtree(failed_data, ignore_errors=True))
|
||
if self.program_swapped:
|
||
if TARGET.is_symlink():
|
||
attempt("移除失败版本链接", TARGET.unlink)
|
||
if self.previous_directory is not None and self.previous_directory.exists():
|
||
attempt("恢复原程序目录", lambda: self.previous_directory.rename(TARGET))
|
||
elif self.previous_target is not None:
|
||
attempt(
|
||
"恢复原程序链接",
|
||
lambda: os.symlink(self.previous_target, TARGET, target_is_directory=True),
|
||
)
|
||
if self.unit_backup.exists():
|
||
attempt("恢复原 systemd unit", lambda: shutil.copy2(self.unit_backup, UNIT_PATH))
|
||
attempt(
|
||
"回滚后重载 systemd",
|
||
lambda: self.run(["systemctl", "daemon-reload"], check=False, failure_label="回滚重载 systemd"),
|
||
)
|
||
if self.program_swapped or self.data_swapped:
|
||
attempt(
|
||
"启动原服务",
|
||
lambda: self.run(["systemctl", "start", SERVICE], check=False, failure_label="回滚启动原服务"),
|
||
)
|
||
attempt("清理失败程序候选", lambda: shutil.rmtree(self.release, ignore_errors=True))
|
||
attempt("清理数据候选", lambda: shutil.rmtree(self.data_candidate, ignore_errors=True))
|
||
recovery = "回滚完成" if not rollback_errors else "回滚存在错误：" + "；".join(rollback_errors)
|
||
self.diagnostics.write(f"final_recovery_state={recovery}")
|
||
brief_error = str(error).splitlines()[0][:1000] or error.__class__.__name__
|
||
result = {
|
||
"status": "failed",
|
||
"target_version": str(self.target_version),
|
||
"packaged_at": str(self.request.get("packaged_at") or ""),
|
||
"installed_at": utc_now(),
|
||
"release_notes": str(self.request.get("release_notes") or ""),
|
||
"error": brief_error,
|
||
}
|
||
try:
|
||
self.diagnostics.persist(DATA_ROOT)
|
||
self.diagnostics.write("persistent_failure_log=written")
|
||
except OSError as persist_error:
|
||
self.diagnostics.write(
|
||
f"persistent_failure_log=failed detail={persist_error.__class__.__name__}: {persist_error}"
|
||
)
|
||
try:
|
||
write_last_result(DATA_ROOT, result)
|
||
except OSError:
|
||
pass
|
||
self.job.update(
|
||
phase="failed",
|
||
percent=0,
|
||
message="更新失败，已恢复原版本",
|
||
finished_at=result["installed_at"],
|
||
error=brief_error,
|
||
)
|
||
write_json(self.status_path, {"schema_version": 1, "active": False, "job": self.job})
|
||
self.package_path.unlink(missing_ok=True)
|
||
self.request_path().unlink(missing_ok=True)
|
||
shutil.rmtree(self.work_root, ignore_errors=True)
|
||
self.diagnostics.path.unlink(missing_ok=True)
|
||
|
||
def start_visual(self) -> None:
|
||
self.diagnostics.section("启动独立更新显示")
|
||
python = TARGET / ".venv/bin/python"
|
||
self.visual = subprocess.Popen([
|
||
str(python), "-m", "scripts.hub75_visual_hold",
|
||
"--mode", "ota",
|
||
"--brightness", "40",
|
||
"--progress-file", str(self.status_path),
|
||
"--orientation", str(int(self.request.get("orientation", 0))),
|
||
])
|
||
time.sleep(1)
|
||
if self.visual.poll() is not None:
|
||
raise UpdateFailed("independent OTA display process failed to start")
|
||
self.diagnostics.write("ota_visual_process=running")
|
||
|
||
def stop_visual(self) -> None:
|
||
if self.visual is None:
|
||
return
|
||
if self.visual.poll() is None:
|
||
self.visual.send_signal(signal.SIGTERM)
|
||
try:
|
||
self.visual.wait(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
self.visual.kill()
|
||
self.visual.wait(timeout=2)
|
||
self.visual = None
|
||
self.diagnostics.write("ota_visual_process=stopped")
|
||
|
||
def request_path(self) -> Path:
|
||
return Path(str(self.request.get("request_path") or REQUEST_PATH))
|
||
|
||
|
||
def checked_token(value: Any, name: str) -> str:
|
||
candidate = str(value or "")
|
||
if not candidate or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-" for character in candidate):
|
||
raise UpdateFailed(f"invalid {name}")
|
||
return candidate
|
||
|
||
|
||
def prune_empty(path: Path) -> None:
|
||
try:
|
||
path.rmdir()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def main() -> int:
|
||
if os.geteuid() != 0 or os.uname().machine != "aarch64":
|
||
print("OTA worker must run as root on the WalnutPi AArch64 host", file=sys.stderr)
|
||
return 2
|
||
request = read_json(REQUEST_PATH)
|
||
if request is None or request.get("schema_version") != 1:
|
||
print("OTA request is missing or invalid", file=sys.stderr)
|
||
return 2
|
||
transaction: Transaction | None = None
|
||
try:
|
||
transaction = Transaction(request)
|
||
transaction.prepare()
|
||
transaction.switch()
|
||
transaction.verify()
|
||
transaction.succeed()
|
||
return 0
|
||
except BaseException as exc:
|
||
if transaction is not None:
|
||
transaction.rollback(exc)
|
||
print(f"OTA update failed: {exc}", file=sys.stderr)
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|