初始化奇妙小屏幕控制器项目
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
"""Offline, browser-uploaded full-release updates.
|
||||
|
||||
Keep package initialization dependency-free: release tooling imports the
|
||||
stdlib-only package builder on bare Linux hosts that do not install the
|
||||
controller's Pillow/FontTools runtime.
|
||||
"""
|
||||
|
||||
from importlib import import_module
|
||||
|
||||
__all__ = [
|
||||
"OtaBusyError",
|
||||
"OtaManager",
|
||||
"OtaPackageError",
|
||||
"OtaUploadError",
|
||||
"SoftwareVersion",
|
||||
"inspect_package",
|
||||
"read_software_version",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name in {"OtaBusyError", "OtaManager", "OtaUploadError"}:
|
||||
return getattr(import_module(".manager", __name__), name)
|
||||
if name in {"OtaPackageError", "inspect_package"}:
|
||||
return getattr(import_module(".package", __name__), name)
|
||||
if name in {"SoftwareVersion", "read_software_version"}:
|
||||
return getattr(import_module(".versioning", __name__), name)
|
||||
raise AttributeError(name)
|
||||
@@ -0,0 +1,206 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from app.persistence import atomic_write_bytes
|
||||
|
||||
|
||||
MAX_FAILURE_LOG_BYTES = 1024 * 1024
|
||||
FAILURE_LOG_RELATIVE = Path("ota") / "last-failure.log"
|
||||
_TRUNCATION_MARKER = (
|
||||
"\n\n========== 日志已截断 =========="
|
||||
"\n原始日志超过 1 MiB;保留开头和最新的失败尾部。"
|
||||
"\n========== 继续显示最新日志 ==========\n\n"
|
||||
).encode("utf-8")
|
||||
_HEAD_BYTES = 128 * 1024
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def bounded_failure_log(body: bytes, *, maximum: int = MAX_FAILURE_LOG_BYTES) -> bytes:
|
||||
normalized = bytes(body).decode("utf-8", errors="replace").encode("utf-8")
|
||||
if len(normalized) <= maximum:
|
||||
return normalized
|
||||
head = min(_HEAD_BYTES, maximum // 4)
|
||||
tail = maximum - head - len(_TRUNCATION_MARKER)
|
||||
if tail <= 0:
|
||||
return normalized[-maximum:]
|
||||
return normalized[:head] + _TRUNCATION_MARKER + normalized[-tail:]
|
||||
|
||||
|
||||
def failure_log_path(data_root: Path) -> Path:
|
||||
return Path(data_root) / FAILURE_LOG_RELATIVE
|
||||
|
||||
|
||||
def persist_failure_log(data_root: Path, runtime_log: Path) -> Path:
|
||||
source = Path(runtime_log)
|
||||
body = bounded_failure_log(source.read_bytes())
|
||||
target = failure_log_path(data_root)
|
||||
atomic_write_bytes(target, body)
|
||||
try:
|
||||
target.chmod(0o600)
|
||||
except OSError:
|
||||
target.unlink(missing_ok=True)
|
||||
raise
|
||||
return target
|
||||
|
||||
|
||||
def clear_failure_log(data_root: Path) -> None:
|
||||
failure_log_path(data_root).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def failure_log_metadata(data_root: Path, last_result: dict[str, Any] | None) -> tuple[bool, int]:
|
||||
if not isinstance(last_result, dict) or last_result.get("status") != "failed":
|
||||
return False, 0
|
||||
path = failure_log_path(data_root)
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
return False, 0
|
||||
if size <= 0 or size > MAX_FAILURE_LOG_BYTES:
|
||||
return False, 0
|
||||
return True, size
|
||||
|
||||
|
||||
def read_failure_log(data_root: Path, last_result: dict[str, Any] | None) -> bytes | None:
|
||||
available, expected_size = failure_log_metadata(data_root, last_result)
|
||||
if not available:
|
||||
return None
|
||||
try:
|
||||
body = failure_log_path(data_root).read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
if len(body) != expected_size:
|
||||
return None
|
||||
return body
|
||||
|
||||
|
||||
class DiagnosticLog:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = Path(path)
|
||||
self._recent = bytearray()
|
||||
self._degraded = False
|
||||
|
||||
def reset(self) -> None:
|
||||
self._recent.clear()
|
||||
try:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_bytes(b"")
|
||||
except OSError:
|
||||
self._degraded = True
|
||||
self.write("OTA 诊断日志已创建")
|
||||
|
||||
def write(self, message: str) -> None:
|
||||
rendered = str(message).replace("\r\n", "\n").replace("\r", "\n")
|
||||
entry = f"[{_timestamp()}] {rendered}\n".encode("utf-8", errors="replace")
|
||||
self._recent.extend(entry)
|
||||
del self._recent[:-MAX_FAILURE_LOG_BYTES]
|
||||
try:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
restore = self._degraded or not self.path.exists()
|
||||
with self.path.open("wb" if restore else "ab") as handle:
|
||||
handle.write(bytes(self._recent) if restore else entry)
|
||||
self._degraded = False
|
||||
except OSError as exc:
|
||||
self._degraded = True
|
||||
try:
|
||||
# Avoid printing OS messages, paths or the command's content here.
|
||||
sys.stderr.write(f"OTA diagnostic storage unavailable ({type(exc).__name__}); recovery continues\n")
|
||||
except (OSError, UnicodeError):
|
||||
pass
|
||||
|
||||
def persist(self, data_root: Path) -> Path:
|
||||
try:
|
||||
body = self.path.read_bytes() if not self._degraded else bytes(self._recent)
|
||||
except OSError:
|
||||
body = bytes(self._recent)
|
||||
target = failure_log_path(data_root)
|
||||
atomic_write_bytes(target, bounded_failure_log(body))
|
||||
target.chmod(0o600)
|
||||
return target
|
||||
|
||||
def section(self, name: str) -> None:
|
||||
self.write(f"========== {name} ==========")
|
||||
|
||||
def host_summary(self, *, test_root: Path) -> None:
|
||||
self.section("安全主机摘要")
|
||||
self.write(f"system={platform.system()} release={platform.release()} machine={platform.machine()}")
|
||||
self.write(f"python={sys.version.splitlines()[0]}")
|
||||
for path in (Path("/opt"), Path("/run"), Path(test_root)):
|
||||
probe = path if path.exists() else path.parent
|
||||
try:
|
||||
usage = shutil.disk_usage(probe)
|
||||
self.write(f"space path={path} free={usage.free} total={usage.total}")
|
||||
except OSError as exc:
|
||||
self.write(f"space path={path} unavailable={exc.__class__.__name__}")
|
||||
|
||||
def run(
|
||||
self,
|
||||
command: list[str],
|
||||
*,
|
||||
cwd: Path | None = None,
|
||||
env: dict[str, str] | None = None,
|
||||
check: bool = True,
|
||||
failure_label: str | None = None,
|
||||
log_output: bool = True,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
display = (
|
||||
shlex.join([str(item) for item in command])
|
||||
if log_output
|
||||
else f"{shlex.quote(str(command[0]))} <参数已按脱敏规则隐藏>"
|
||||
)
|
||||
self.section(f"命令开始:{failure_label or Path(command[0]).name}")
|
||||
self.write(f"cwd={cwd if cwd is not None else os.getcwd()}")
|
||||
self.write(f"command={display}")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
except OSError as exc:
|
||||
self.write(f"command_start_failed={exc.__class__.__name__}: {exc}")
|
||||
raise RuntimeError(f"{failure_label or '命令'}无法启动:{exc}") from exc
|
||||
output = result.stdout.decode("utf-8", errors="replace")
|
||||
if output and log_output:
|
||||
self.write("stdout_stderr:\n" + output.rstrip("\n"))
|
||||
try:
|
||||
sys.stdout.write(output)
|
||||
sys.stdout.flush()
|
||||
except (UnicodeError, OSError):
|
||||
pass
|
||||
elif output:
|
||||
self.write("stdout_stderr=<已按脱敏规则隐藏>")
|
||||
self.write(f"exit_code={result.returncode}")
|
||||
if check and result.returncode != 0:
|
||||
tail = (
|
||||
output.rstrip()[-12000:] or "命令没有输出。"
|
||||
if log_output
|
||||
else "命令输出已按脱敏规则隐藏。"
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"{failure_label or '命令'}失败(退出码 {result.returncode}):\n{tail}"
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,238 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
from typing import Any, AsyncIterable, Callable
|
||||
from uuid import uuid4
|
||||
|
||||
from app.display.service import DisplayService
|
||||
|
||||
from .diagnostics import failure_log_metadata, read_failure_log
|
||||
from .package import MAX_OTA_UPLOAD_BYTES, OtaPackageError, inspect_package
|
||||
from .state import read_json, read_last_result, utc_now, write_json
|
||||
from .versioning import SoftwareVersion
|
||||
|
||||
_OTA_FILENAME = re.compile(r"^matrix-screen-controller-[0-9]+\.[0-9]+\.[0-9]+\.ota$")
|
||||
|
||||
|
||||
class OtaUploadError(RuntimeError):
|
||||
def __init__(self, message: str, *, status_code: int = 422) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class OtaBusyError(OtaUploadError):
|
||||
def __init__(self, message: str = "an OTA upload or update is already active") -> None:
|
||||
super().__init__(message, status_code=409)
|
||||
|
||||
|
||||
class OtaManager:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
code_root: Path,
|
||||
data_root: Path,
|
||||
runtime_root: Path,
|
||||
software_version: SoftwareVersion,
|
||||
display: DisplayService,
|
||||
worker_starter: Callable[[], None] | None = None,
|
||||
staging_root: Path | None = None,
|
||||
) -> None:
|
||||
self.code_root = Path(code_root)
|
||||
self.data_root = Path(data_root)
|
||||
self.runtime_root = Path(runtime_root)
|
||||
self.software_version = software_version
|
||||
self.display = display
|
||||
configured_staging = os.environ.get("MATRIX_OTA_STAGING_DIR")
|
||||
if staging_root is not None:
|
||||
self.staging_root = Path(staging_root)
|
||||
elif configured_staging:
|
||||
self.staging_root = Path(configured_staging)
|
||||
elif os.name != "nt" and str(self.code_root).startswith("/opt/"):
|
||||
self.staging_root = Path("/opt/matrix-screen-controller-ota")
|
||||
else:
|
||||
self.staging_root = self.runtime_root / "ota-staging"
|
||||
self.status_path = self.runtime_root / "ota-status.json"
|
||||
self.request_path = self.runtime_root / "ota-request.json"
|
||||
self._worker_starter = worker_starter or self._start_systemd_worker
|
||||
self._upload_lock = asyncio.Lock()
|
||||
self._monitor_stop = threading.Event()
|
||||
self._monitor_thread: threading.Thread | None = None
|
||||
self._completion_callback: Callable[[], None] = lambda: None
|
||||
|
||||
def set_completion_callback(self, callback: Callable[[], None]) -> None:
|
||||
self._completion_callback = callback
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
active_document = read_json(self.status_path)
|
||||
job = active_document.get("job") if isinstance(active_document, dict) else None
|
||||
component_pending = (self.data_root / "ota/component-transaction").exists()
|
||||
active = component_pending or bool(active_document and active_document.get("active") is True and isinstance(job, dict))
|
||||
last_result = read_last_result(self.data_root)
|
||||
failure_log_available, failure_log_bytes = failure_log_metadata(self.data_root, last_result)
|
||||
return {
|
||||
"current_version": str(self.software_version),
|
||||
"max_upload_bytes": MAX_OTA_UPLOAD_BYTES,
|
||||
"active": active,
|
||||
"component_recovery_pending": component_pending,
|
||||
"job": dict(job) if active or isinstance(job, dict) else None,
|
||||
"last_result": last_result,
|
||||
"failure_log_available": failure_log_available,
|
||||
"failure_log_bytes": failure_log_bytes,
|
||||
}
|
||||
|
||||
def failure_log(self) -> bytes | None:
|
||||
last_result = read_last_result(self.data_root)
|
||||
return read_failure_log(self.data_root, last_result)
|
||||
|
||||
def is_active(self) -> bool:
|
||||
return self.status()["active"] or self._upload_lock.locked()
|
||||
|
||||
async def accept_upload(
|
||||
self,
|
||||
*,
|
||||
filename: str,
|
||||
chunks: AsyncIterable[bytes],
|
||||
content_length: int | None,
|
||||
orientation: int,
|
||||
) -> dict[str, Any]:
|
||||
if Path(filename).name != filename or not _OTA_FILENAME.fullmatch(filename):
|
||||
raise OtaUploadError(
|
||||
"OTA filename must be matrix-screen-controller-MAJOR.MINOR.PATCH.ota",
|
||||
status_code=400,
|
||||
)
|
||||
if content_length is not None and (content_length <= 0 or content_length > MAX_OTA_UPLOAD_BYTES):
|
||||
raise OtaUploadError("OTA package exceeds the 256 MiB upload limit", status_code=413)
|
||||
if self._upload_lock.locked() or self.status()["active"]:
|
||||
raise OtaBusyError()
|
||||
|
||||
async with self._upload_lock:
|
||||
if self.status()["active"]:
|
||||
raise OtaBusyError()
|
||||
upload_root = self.staging_root / "uploads"
|
||||
upload_root.mkdir(parents=True, exist_ok=True)
|
||||
temporary = upload_root / f".{uuid4().hex}.part"
|
||||
final: Path | None = None
|
||||
total = 0
|
||||
try:
|
||||
with temporary.open("xb") as handle:
|
||||
os.chmod(temporary, 0o600)
|
||||
async for chunk in chunks:
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > MAX_OTA_UPLOAD_BYTES:
|
||||
raise OtaUploadError("OTA package exceeds the 256 MiB upload limit", status_code=413)
|
||||
handle.write(chunk)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if total == 0:
|
||||
raise OtaUploadError("OTA package is empty", status_code=400)
|
||||
if content_length is not None and total != content_length:
|
||||
raise OtaUploadError("OTA upload length does not match Content-Length", status_code=400)
|
||||
try:
|
||||
package = inspect_package(temporary, current_version=self.software_version)
|
||||
except OtaPackageError as exc:
|
||||
message = str(exc)
|
||||
status_code = 409 if "must be newer" in message else 422
|
||||
raise OtaUploadError(message, status_code=status_code) from exc
|
||||
expected_name = f"matrix-screen-controller-{package.target_version}.ota"
|
||||
if filename != expected_name:
|
||||
raise OtaUploadError("OTA filename version does not match its manifest", status_code=422)
|
||||
job_id = uuid4().hex
|
||||
final = upload_root / f"{job_id}.ota"
|
||||
os.replace(temporary, final)
|
||||
request = {
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"package_path": str(final),
|
||||
"status_path": str(self.status_path),
|
||||
"request_path": str(self.request_path),
|
||||
"current_version": str(self.software_version),
|
||||
"target_version": str(package.target_version),
|
||||
"packaged_at": package.created_at,
|
||||
"release_notes": package.release_notes,
|
||||
"orientation": int(orientation),
|
||||
"accepted_at": utc_now(),
|
||||
}
|
||||
job = {
|
||||
"id": job_id,
|
||||
"target_version": str(package.target_version),
|
||||
"packaged_at": package.created_at,
|
||||
"phase": "queued",
|
||||
"percent": 5,
|
||||
"message": "软件安装包已校验,准备检查并安装系统组件" if package.target_version.patch == 0 else "更新包已校验,准备安装",
|
||||
"started_at": request["accepted_at"],
|
||||
"finished_at": None,
|
||||
"error": None,
|
||||
}
|
||||
write_json(self.request_path, request)
|
||||
write_json(self.status_path, {"schema_version": 1, "active": True, "job": job})
|
||||
self.display.start_ota_indicator(self.status_path)
|
||||
try:
|
||||
self._worker_starter()
|
||||
except Exception as exc:
|
||||
failed = dict(job)
|
||||
failed.update(
|
||||
phase="failed",
|
||||
percent=0,
|
||||
message="无法启动独立 OTA 工作服务",
|
||||
finished_at=utc_now(),
|
||||
error=str(exc),
|
||||
)
|
||||
write_json(self.status_path, {"schema_version": 1, "active": False, "job": failed})
|
||||
self.display.stop_ota_indicator()
|
||||
raise OtaUploadError("independent OTA worker could not be started", status_code=500) from exc
|
||||
self.resume_or_start_monitor(self._completion_callback)
|
||||
return {"accepted": True, "job": job}
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
if final is not None and not self.status()["active"]:
|
||||
final.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
def resume_or_start_monitor(self, on_finished: Callable[[], None]) -> bool:
|
||||
if not self.status()["active"]:
|
||||
return False
|
||||
self.display.start_ota_indicator(self.status_path)
|
||||
if self._monitor_thread is not None and self._monitor_thread.is_alive():
|
||||
return True
|
||||
self._monitor_stop.clear()
|
||||
|
||||
def monitor() -> None:
|
||||
while not self._monitor_stop.wait(0.25):
|
||||
if self.status()["active"]:
|
||||
continue
|
||||
try:
|
||||
self.display.stop_ota_indicator()
|
||||
on_finished()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
self._monitor_thread = threading.Thread(target=monitor, name="ota-status-monitor", daemon=True)
|
||||
self._monitor_thread.start()
|
||||
return True
|
||||
|
||||
def close(self) -> None:
|
||||
self._monitor_stop.set()
|
||||
thread = self._monitor_thread
|
||||
if thread is not None and thread is not threading.current_thread():
|
||||
thread.join(timeout=2.0)
|
||||
|
||||
@staticmethod
|
||||
def _start_systemd_worker() -> None:
|
||||
result = subprocess.run(
|
||||
["systemctl", "start", "--no-block", "matrix-screen-controller-ota.service"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError((result.stderr or result.stdout or "systemctl start failed").strip())
|
||||
@@ -0,0 +1,294 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from typing import Any, BinaryIO
|
||||
import zipfile
|
||||
|
||||
from .versioning import SoftwareVersion
|
||||
from .policy import check_upgrade, package_format, release_metadata
|
||||
|
||||
PRODUCT_ID = "matrix-screen-controller-walnutpi"
|
||||
PACKAGE_FORMAT_VERSION = 1
|
||||
MAX_OTA_UPLOAD_BYTES = 256 * 1024 * 1024
|
||||
MAX_OTA_PAYLOAD_BYTES = 256 * 1024 * 1024
|
||||
MAX_OTA_EXPANDED_BYTES = 1024 * 1024 * 1024
|
||||
_MAX_MANIFEST_BYTES = 64 * 1024
|
||||
_PACKAGE_MEMBERS = {"manifest.json", "payload.tar.gz"}
|
||||
_NATIVE_BUILD_OUTPUTS = {
|
||||
"app/display/native/libh618_hub75.so",
|
||||
"app/display/native/hub75_benchmark",
|
||||
"app/display/native/hub75_native_test",
|
||||
"app/display/native/hub75_safeoff",
|
||||
}
|
||||
_EXCLUDED_PARTS = {".venv", "data", "__pycache__", ".pytest_cache", "node_modules"}
|
||||
|
||||
|
||||
class OtaPackageError(ValueError):
|
||||
"""The uploaded archive is not a supported complete release."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OtaPackageInfo:
|
||||
path: Path
|
||||
target_version: SoftwareVersion
|
||||
created_at: str
|
||||
release_notes: str
|
||||
payload_sha256: str
|
||||
payload_bytes: int
|
||||
expanded_bytes: int
|
||||
|
||||
def document(self) -> dict[str, Any]:
|
||||
return {
|
||||
"target_version": str(self.target_version),
|
||||
"created_at": self.created_at,
|
||||
"release_notes": self.release_notes,
|
||||
"payload_sha256": self.payload_sha256,
|
||||
"payload_bytes": self.payload_bytes,
|
||||
"expanded_bytes": self.expanded_bytes,
|
||||
**release_metadata(self.target_version),
|
||||
}
|
||||
|
||||
|
||||
def _validated_manifest(raw: bytes) -> tuple[dict[str, Any], SoftwareVersion]:
|
||||
if len(raw) > _MAX_MANIFEST_BYTES:
|
||||
raise OtaPackageError("OTA manifest is too large")
|
||||
try:
|
||||
manifest = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise OtaPackageError("OTA manifest is not valid UTF-8 JSON") from exc
|
||||
expected = {
|
||||
"format_version",
|
||||
"product",
|
||||
"software_version",
|
||||
"created_at",
|
||||
"release_notes",
|
||||
"payload_sha256",
|
||||
"payload_bytes",
|
||||
"expanded_bytes",
|
||||
}
|
||||
if not isinstance(manifest, dict) or set(manifest) != expected:
|
||||
raise OtaPackageError("OTA manifest fields do not match format version 1")
|
||||
if type(manifest["format_version"]) is not int or manifest["format_version"] not in (1, 2):
|
||||
raise OtaPackageError("OTA package format is unsupported")
|
||||
if manifest["product"] != PRODUCT_ID:
|
||||
raise OtaPackageError("OTA package is for a different product")
|
||||
try:
|
||||
version = SoftwareVersion.parse(manifest["software_version"])
|
||||
if manifest["format_version"] != package_format(version):
|
||||
raise ValueError("OTA version requires a different package format")
|
||||
except ValueError as exc:
|
||||
raise OtaPackageError(str(exc)) from exc
|
||||
if not isinstance(manifest["created_at"], str) or not manifest["created_at"].strip():
|
||||
raise OtaPackageError("OTA package created_at is missing")
|
||||
if not isinstance(manifest["release_notes"], str) or len(manifest["release_notes"]) > 4000:
|
||||
raise OtaPackageError("OTA release notes are invalid")
|
||||
digest = manifest["payload_sha256"]
|
||||
if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest):
|
||||
raise OtaPackageError("OTA payload SHA-256 is invalid")
|
||||
for key, maximum in (
|
||||
("payload_bytes", MAX_OTA_PAYLOAD_BYTES),
|
||||
("expanded_bytes", MAX_OTA_EXPANDED_BYTES),
|
||||
):
|
||||
value = manifest[key]
|
||||
if type(value) is not int or value <= 0 or value > maximum:
|
||||
raise OtaPackageError(f"OTA {key} is outside the supported limit")
|
||||
return manifest, version
|
||||
|
||||
|
||||
def _payload_digest(handle: BinaryIO) -> tuple[str, int]:
|
||||
digest = hashlib.sha256()
|
||||
total = 0
|
||||
while True:
|
||||
chunk = handle.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > MAX_OTA_PAYLOAD_BYTES:
|
||||
raise OtaPackageError("OTA payload exceeds the supported limit")
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest(), total
|
||||
|
||||
|
||||
def inspect_package(path: Path, *, current_version: SoftwareVersion | None = None) -> OtaPackageInfo:
|
||||
package_path = Path(path)
|
||||
try:
|
||||
size = package_path.stat().st_size
|
||||
except OSError as exc:
|
||||
raise OtaPackageError("OTA package cannot be read") from exc
|
||||
if size <= 0 or size > MAX_OTA_UPLOAD_BYTES:
|
||||
raise OtaPackageError("OTA package exceeds the 256 MiB upload limit")
|
||||
try:
|
||||
with zipfile.ZipFile(package_path, "r") as archive:
|
||||
entries = archive.infolist()
|
||||
if {entry.filename for entry in entries} != _PACKAGE_MEMBERS or len(entries) != 2:
|
||||
raise OtaPackageError("OTA package must contain only manifest.json and payload.tar.gz")
|
||||
if any(entry.is_dir() or entry.flag_bits & 0x1 for entry in entries):
|
||||
raise OtaPackageError("OTA package entries must be unencrypted files")
|
||||
manifest_entry = archive.getinfo("manifest.json")
|
||||
payload_entry = archive.getinfo("payload.tar.gz")
|
||||
if manifest_entry.file_size > _MAX_MANIFEST_BYTES:
|
||||
raise OtaPackageError("OTA manifest is too large")
|
||||
if payload_entry.file_size > MAX_OTA_PAYLOAD_BYTES:
|
||||
raise OtaPackageError("OTA payload exceeds the supported limit")
|
||||
manifest, target_version = _validated_manifest(archive.read(manifest_entry))
|
||||
with archive.open(payload_entry, "r") as payload:
|
||||
digest, payload_bytes = _payload_digest(payload)
|
||||
except (OtaPackageError, zipfile.BadZipFile):
|
||||
raise
|
||||
except (KeyError, OSError, RuntimeError) as exc:
|
||||
raise OtaPackageError("OTA package cannot be inspected") from exc
|
||||
if payload_bytes != manifest["payload_bytes"] or digest != manifest["payload_sha256"]:
|
||||
raise OtaPackageError("OTA payload checksum or size does not match the manifest")
|
||||
if current_version is not None:
|
||||
try:
|
||||
check_upgrade(current_version, target_version)
|
||||
except ValueError as exc:
|
||||
raise OtaPackageError(str(exc)) from exc
|
||||
return OtaPackageInfo(
|
||||
path=package_path,
|
||||
target_version=target_version,
|
||||
created_at=manifest["created_at"],
|
||||
release_notes=manifest["release_notes"],
|
||||
payload_sha256=digest,
|
||||
payload_bytes=payload_bytes,
|
||||
expanded_bytes=manifest["expanded_bytes"],
|
||||
)
|
||||
|
||||
|
||||
def _safe_member_path(name: str) -> Path:
|
||||
pure = PurePosixPath(name)
|
||||
if pure.is_absolute() or not pure.parts or any(part in {"", ".", ".."} for part in pure.parts):
|
||||
raise OtaPackageError(f"unsafe OTA payload path: {name}")
|
||||
if pure.parts[0] not in {"software", "wheelhouse", "system-dependencies"}:
|
||||
raise OtaPackageError(f"unexpected OTA payload root: {pure.parts[0]}")
|
||||
return Path(*pure.parts)
|
||||
|
||||
|
||||
def extract_payload(package: OtaPackageInfo, destination: Path) -> None:
|
||||
target = Path(destination)
|
||||
target.mkdir(parents=True, exist_ok=False)
|
||||
expanded = 0
|
||||
try:
|
||||
with zipfile.ZipFile(package.path, "r") as archive:
|
||||
with archive.open("payload.tar.gz", "r") as payload:
|
||||
with tarfile.open(fileobj=payload, mode="r|gz") as tar:
|
||||
for member in tar:
|
||||
relative = _safe_member_path(member.name)
|
||||
output = target / relative
|
||||
if member.isdir():
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
if not member.isfile():
|
||||
raise OtaPackageError("OTA payload may contain only regular files and directories")
|
||||
expanded += member.size
|
||||
if expanded > MAX_OTA_EXPANDED_BYTES or expanded > package.expanded_bytes:
|
||||
raise OtaPackageError("OTA payload expands beyond its declared limit")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = tar.extractfile(member)
|
||||
if source is None:
|
||||
raise OtaPackageError(f"OTA payload member cannot be read: {member.name}")
|
||||
with output.open("xb") as handle:
|
||||
shutil.copyfileobj(source, handle, length=1024 * 1024)
|
||||
output.chmod(member.mode & 0o777 or 0o644)
|
||||
if expanded != package.expanded_bytes:
|
||||
raise OtaPackageError("OTA expanded size does not match the manifest")
|
||||
if not (target / "software" / "VERSION").is_file():
|
||||
raise OtaPackageError("OTA payload does not contain software/VERSION")
|
||||
if not (target / "wheelhouse" / "SHA256SUMS").is_file():
|
||||
raise OtaPackageError("OTA payload does not contain the offline wheel manifest")
|
||||
except BaseException:
|
||||
shutil.rmtree(target, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def _source_file_allowed(path: Path, source_root: Path) -> bool:
|
||||
relative = path.relative_to(source_root)
|
||||
if any(part in _EXCLUDED_PARTS | {"system-dependencies", "output", ".playwright-cli"} for part in relative.parts):
|
||||
return False
|
||||
if relative.as_posix() in _NATIVE_BUILD_OUTPUTS:
|
||||
return False
|
||||
if path.suffix in {".pyc", ".pyo"}:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _tar_add_file(tar: tarfile.TarFile, source: Path, archive_name: str) -> int:
|
||||
info = tar.gettarinfo(str(source), arcname=archive_name)
|
||||
info.uid = info.gid = 0
|
||||
info.uname = info.gname = "root"
|
||||
info.mtime = 0
|
||||
info.mode = 0o755 if source.suffix == ".sh" else 0o644
|
||||
with source.open("rb") as handle:
|
||||
tar.addfile(info, handle)
|
||||
return info.size
|
||||
|
||||
|
||||
def build_package(
|
||||
source_root: Path,
|
||||
wheelhouse: Path,
|
||||
output_path: Path,
|
||||
*,
|
||||
version: SoftwareVersion,
|
||||
release_notes: str,
|
||||
created_at: datetime | None = None,
|
||||
system_dependencies: Path | None = None,
|
||||
) -> OtaPackageInfo:
|
||||
source_root = Path(source_root).resolve()
|
||||
wheelhouse = Path(wheelhouse).resolve()
|
||||
system_dependencies = Path(system_dependencies).resolve() if system_dependencies is not None else None
|
||||
if version.patch != 0 and version >= SoftwareVersion(1, 1, 0) and system_dependencies is not None:
|
||||
raise OtaPackageError("普通补丁包不得携带系统软件,请导出新的 minor .0 安装包")
|
||||
if SoftwareVersion.parse((source_root / "VERSION").read_text(encoding="utf-8")) != version:
|
||||
raise OtaPackageError("requested package version does not match source VERSION")
|
||||
if not (wheelhouse / "SHA256SUMS").is_file():
|
||||
raise OtaPackageError("offline wheelhouse SHA256SUMS is missing")
|
||||
if system_dependencies is not None and not (system_dependencies / "SHA256SUMS").is_file():
|
||||
raise OtaPackageError("offline system dependency SHA256SUMS is missing")
|
||||
output = Path(output_path)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if output.exists():
|
||||
raise FileExistsError(output)
|
||||
timestamp = (created_at or datetime.now(timezone.utc)).astimezone().isoformat(timespec="seconds")
|
||||
with tempfile.TemporaryDirectory(prefix="matrix-ota-build-") as temporary:
|
||||
payload_path = Path(temporary) / "payload.tar.gz"
|
||||
expanded = 0
|
||||
with tarfile.open(payload_path, "w:gz", format=tarfile.PAX_FORMAT) as tar:
|
||||
for path in sorted(source_root.rglob("*"), key=lambda item: item.relative_to(source_root).as_posix()):
|
||||
if path.is_file() and _source_file_allowed(path, source_root):
|
||||
expanded += _tar_add_file(tar, path, f"software/{path.relative_to(source_root).as_posix()}")
|
||||
for path in sorted(wheelhouse.rglob("*"), key=lambda item: item.relative_to(wheelhouse).as_posix()):
|
||||
if path.is_file():
|
||||
expanded += _tar_add_file(tar, path, f"wheelhouse/{path.relative_to(wheelhouse).as_posix()}")
|
||||
if system_dependencies is not None:
|
||||
for path in sorted(system_dependencies.rglob("*"), key=lambda item: item.relative_to(system_dependencies).as_posix()):
|
||||
if path.is_file():
|
||||
expanded += _tar_add_file(
|
||||
tar,
|
||||
path,
|
||||
f"software/system-dependencies/frpc/{path.relative_to(system_dependencies).as_posix()}",
|
||||
)
|
||||
payload_bytes = payload_path.stat().st_size
|
||||
if payload_bytes > MAX_OTA_PAYLOAD_BYTES or expanded > MAX_OTA_EXPANDED_BYTES:
|
||||
raise OtaPackageError("generated OTA payload exceeds the supported limit")
|
||||
digest = hashlib.sha256(payload_path.read_bytes()).hexdigest()
|
||||
manifest = {
|
||||
"format_version": package_format(version),
|
||||
"product": PRODUCT_ID,
|
||||
"software_version": str(version),
|
||||
"created_at": timestamp,
|
||||
"release_notes": str(release_notes).strip(),
|
||||
"payload_sha256": digest,
|
||||
"payload_bytes": payload_bytes,
|
||||
"expanded_bytes": expanded,
|
||||
}
|
||||
with zipfile.ZipFile(output, "x", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
|
||||
archive.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
|
||||
archive.write(payload_path, "payload.tar.gz")
|
||||
return inspect_package(output)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Ordered dependency checkpoints, independent of removable release artifacts."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .versioning import SoftwareVersion
|
||||
|
||||
BRIDGE_VERSION = SoftwareVersion(1, 1, 0)
|
||||
|
||||
|
||||
def check_upgrade(current: SoftwareVersion, target: SoftwareVersion) -> None:
|
||||
if target <= current:
|
||||
raise ValueError(f"OTA target {target} must be newer than installed version {current}")
|
||||
if current.major != target.major:
|
||||
raise ValueError("跨主版本升级尚未登记,请使用明确支持此路径的安装包")
|
||||
if target.minor > current.minor:
|
||||
required = SoftwareVersion(current.major, current.minor + 1, 0)
|
||||
if target != required:
|
||||
raise ValueError(f"请先安装 {required} 软件安装包,不能跳过必经升级版本")
|
||||
|
||||
|
||||
def package_format(version: SoftwareVersion) -> int:
|
||||
return 1 if version <= BRIDGE_VERSION else 2
|
||||
|
||||
|
||||
def release_metadata(version: SoftwareVersion) -> dict:
|
||||
checkpoint = version.patch == 0 and version >= BRIDGE_VERSION
|
||||
predecessor = None
|
||||
if version >= BRIDGE_VERSION:
|
||||
predecessor = str(SoftwareVersion(version.major, version.minor - 1 if checkpoint else version.minor, 0))
|
||||
return {"package_kind": "software-install" if checkpoint else "application",
|
||||
"required_checkpoint": predecessor, "is_checkpoint": checkpoint}
|
||||
|
||||
|
||||
def read_policy(source: Path) -> dict:
|
||||
value = json.loads((source / "UPGRADE_POLICY.json").read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict) or set(value) != {"schema_version", "checkpoints"} or value["schema_version"] != 1:
|
||||
raise ValueError("invalid upgrade policy")
|
||||
previous = SoftwareVersion(1, 0, 0)
|
||||
for entry in value["checkpoints"]:
|
||||
version = SoftwareVersion.parse(entry["version"])
|
||||
if version != SoftwareVersion(previous.major, previous.minor + 1, 0) or not entry["components"]:
|
||||
raise ValueError("dependency checkpoints must be consecutive minor .0 versions")
|
||||
for name, component in entry["components"].items():
|
||||
if name != "frpc" or set(component) != {"version", "sha256", "bundle"}:
|
||||
raise ValueError("component installer is not registered")
|
||||
if len(component["sha256"]) != 64 or any(c not in "0123456789abcdef" for c in component["sha256"]):
|
||||
raise ValueError("invalid component digest")
|
||||
bundle = Path(component["bundle"])
|
||||
if bundle.is_absolute() or ".." in bundle.parts or "\\" in component["bundle"]:
|
||||
raise ValueError("unsafe component bundle path")
|
||||
previous = version
|
||||
return value
|
||||
|
||||
|
||||
def required_components(source: Path, version: SoftwareVersion) -> dict:
|
||||
components = {}
|
||||
for entry in read_policy(source)["checkpoints"]:
|
||||
if SoftwareVersion.parse(entry["version"]) <= version:
|
||||
components.update(entry["components"])
|
||||
return components
|
||||
|
||||
|
||||
def export_bundle(source: Path, dependency_root: Path, current: SoftwareVersion, target: SoftwareVersion) -> Path | None:
|
||||
check_upgrade(current, target)
|
||||
policy = read_policy(source)
|
||||
if policy["checkpoints"] and target < SoftwareVersion.parse(policy["checkpoints"][-1]["version"]):
|
||||
raise ValueError("源码已引入新系统依赖;必须先导出已登记的 minor .0 软件安装包")
|
||||
matching = [e for e in policy["checkpoints"] if e["version"] == str(target)]
|
||||
if target.patch == 0 and not matching:
|
||||
raise ValueError("软件安装包必须登记本次依赖变化")
|
||||
components = required_components(source, target)
|
||||
# Verify the actual offline source, including on patch exports: silently replacing
|
||||
# a binary at the same version must not bypass a dependency checkpoint.
|
||||
for component in components.values():
|
||||
binary = dependency_root / component["bundle"] / "frpc"
|
||||
if hashlib.sha256(binary.read_bytes()).hexdigest() != component["sha256"]:
|
||||
raise ValueError("系统依赖发生变化;请登记新的 minor .0 软件安装版本")
|
||||
if not matching:
|
||||
return None
|
||||
return dependency_root / matching[0]["components"]["frpc"]["bundle"]
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.persistence import atomic_write_bytes
|
||||
|
||||
OTA_STATE_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return None
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def write_json(path: Path, document: dict[str, Any]) -> None:
|
||||
atomic_write_bytes(
|
||||
Path(path),
|
||||
(json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"),
|
||||
)
|
||||
|
||||
|
||||
def read_last_result(data_root: Path) -> dict[str, Any] | None:
|
||||
document = read_json(Path(data_root) / "ota" / "state.json")
|
||||
if not document or document.get("schema_version") != OTA_STATE_SCHEMA_VERSION:
|
||||
return None
|
||||
result = document.get("last_result")
|
||||
return dict(result) if isinstance(result, dict) else None
|
||||
|
||||
|
||||
def write_last_result(data_root: Path, result: dict[str, Any]) -> None:
|
||||
write_json(
|
||||
Path(data_root) / "ota" / "state.json",
|
||||
{"schema_version": OTA_STATE_SCHEMA_VERSION, "last_result": dict(result)},
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
_VERSION_PATTERN = re.compile(r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$")
|
||||
_FEATURE_UPDATED_AT_PATTERN = re.compile(
|
||||
r"^[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]\+08:00$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True)
|
||||
class SoftwareVersion:
|
||||
major: int
|
||||
minor: int
|
||||
patch: int
|
||||
|
||||
@classmethod
|
||||
def parse(cls, value: str) -> "SoftwareVersion":
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("software version must be a string")
|
||||
match = _VERSION_PATTERN.fullmatch(value.strip())
|
||||
if match is None:
|
||||
raise ValueError("software version must use strict MAJOR.MINOR.PATCH digits")
|
||||
return cls(*(int(part) for part in match.groups()))
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.major}.{self.minor}.{self.patch}"
|
||||
|
||||
|
||||
def read_software_version(code_root: Path) -> SoftwareVersion:
|
||||
path = Path(code_root) / "VERSION"
|
||||
try:
|
||||
return SoftwareVersion.parse(path.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
raise RuntimeError(f"software VERSION file is unreadable: {path}") from exc
|
||||
|
||||
|
||||
def read_feature_updated_at(code_root: Path) -> str:
|
||||
path = Path(code_root) / "FEATURE_UPDATED_AT"
|
||||
try:
|
||||
value = path.read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise RuntimeError(f"feature update timestamp file is unreadable: {path}") from exc
|
||||
if _FEATURE_UPDATED_AT_PATTERN.fullmatch(value) is None:
|
||||
raise ValueError("feature update timestamp must use YYYY-MM-DDTHH:MM+08:00")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError("feature update timestamp is not a valid calendar time") from exc
|
||||
if parsed.utcoffset() != timedelta(hours=8):
|
||||
raise ValueError("feature update timestamp must use Beijing UTC+08:00")
|
||||
return value
|
||||
Reference in New Issue
Block a user