341 lines
16 KiB
Python
341 lines
16 KiB
Python
import hashlib
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
from app.system import kernel_recovery as kr
|
|
from app.ota.manager import OtaManager
|
|
from app.ota.state import write_json, write_last_result
|
|
from app.ota.versioning import SoftwareVersion
|
|
from scripts.ota_worker import UpdateFailed, verify_kernel_capability_retained
|
|
from scripts.axp313a_kernel_health import (
|
|
HealthError,
|
|
check_cpufreq,
|
|
check_driver_pair,
|
|
check_kernel_messages,
|
|
)
|
|
from scripts.render_dual_kernel_boot import BootScriptError, render
|
|
|
|
|
|
def make_recovery(tmp_path: Path, monkeypatch) -> kr.KernelRecovery:
|
|
host = tmp_path / "host"
|
|
data = tmp_path / "data"
|
|
boot = host / "boot"
|
|
boot.mkdir(parents=True)
|
|
files = {
|
|
"Image-matrix-axp313a1": b"candidate-image",
|
|
"sun50i-h616-walnutpi-1b-matrix-axp313a1.dtb": b"candidate-sd-dtb",
|
|
"sun50i-h616-walnutpi-1b-emmc-matrix-axp313a1.dtb": b"candidate-emmc-dtb",
|
|
}
|
|
monkeypatch.setattr(kr, "BOOT_HASHES", {name: hashlib.sha256(value).hexdigest() for name, value in files.items()})
|
|
for name, value in files.items():
|
|
(boot / name).write_bytes(value)
|
|
for name in ("Image", "sun50i-h616-walnutpi-1b.dtb", "sun50i-h616-walnutpi-1b-emmc.dtb",
|
|
"boot.cmd.matrix-original", "boot.scr.matrix-original", "matrix-original.SHA256SUMS"):
|
|
(boot / name).write_bytes(b"original")
|
|
(boot / "boot.cmd").write_text(
|
|
"matrix-kernel-good matrix-kernel-good matrix_kernel_candidate=1",
|
|
encoding="utf-8",
|
|
)
|
|
(boot / "boot.scr").write_bytes(b"matrix_kernel_candidate=1 Image-matrix-axp313a1")
|
|
health = host / "etc/systemd/system/matrix-axp313a-health.service"
|
|
health.parent.mkdir(parents=True)
|
|
health.write_text("ConditionKernelCommandLine=matrix_kernel_candidate=1", encoding="utf-8")
|
|
script = host / "opt/matrix-screen-controller-system/axp313a_kernel_health.py"
|
|
script.parent.mkdir(parents=True)
|
|
script.write_text("health", encoding="utf-8")
|
|
enabled = host / "etc/systemd/system/multi-user.target.wants/matrix-axp313a-health.service"
|
|
enabled.parent.mkdir(parents=True)
|
|
enabled.write_text("enabled", encoding="utf-8")
|
|
modules = host / "lib/modules" / kr.CANDIDATE_RELEASE
|
|
(modules / "kernel").mkdir(parents=True)
|
|
(modules / "modules.dep").write_text("index", encoding="ascii")
|
|
module = modules / "kernel/example.ko"
|
|
module.write_bytes(b"module")
|
|
digest = hashlib.sha256(b"kernel/example.ko\0" + hashlib.sha256(b"module").digest()).hexdigest()
|
|
monkeypatch.setattr(kr, "MODULE_COUNT", 1)
|
|
monkeypatch.setattr(kr, "MODULE_DIGEST", digest)
|
|
release = host / "proc/sys/kernel/osrelease"
|
|
release.parent.mkdir(parents=True)
|
|
release.write_text("6.1.31", encoding="ascii")
|
|
boot_id = host / "proc/sys/kernel/random/boot_id"
|
|
boot_id.parent.mkdir(parents=True)
|
|
boot_id.write_text("old-boot", encoding="ascii")
|
|
return kr.KernelRecovery(data, host_root=host)
|
|
|
|
|
|
def test_candidate_validation_and_single_recovery_attempt(tmp_path, monkeypatch):
|
|
recovery = make_recovery(tmp_path, monkeypatch)
|
|
assert recovery.verify_candidate() is None
|
|
assert recovery.prepare_attempt("1.2.0:time") is True
|
|
assert recovery.prepare_attempt("1.2.0:time") is False
|
|
monkeypatch.setattr(kr.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess(args, 1))
|
|
recovery.arm_and_reboot(reboot=False)
|
|
assert recovery.status()["state"] == "pending"
|
|
with pytest.raises(RuntimeError, match="no unused"):
|
|
recovery.arm_and_reboot(reboot=False)
|
|
recovery.host("/proc/sys/kernel/random/boot_id").write_text("new-boot", encoding="ascii")
|
|
recovery.host("/proc/sys/kernel/osrelease").write_text(kr.CANDIDATE_RELEASE, encoding="ascii")
|
|
policy = recovery.host("/sys/devices/system/cpu/cpufreq/policy0")
|
|
policy.mkdir(parents=True)
|
|
(policy / "scaling_governor").write_text("schedutil", encoding="ascii")
|
|
(policy / "scaling_available_governors").write_text("schedutil performance", encoding="ascii")
|
|
recovery.finalize_after_boot(wait_seconds=1)
|
|
assert recovery.status()["state"] == "succeeded"
|
|
|
|
|
|
def test_recovery_failure_and_damaged_candidate_do_not_retry(tmp_path, monkeypatch):
|
|
recovery = make_recovery(tmp_path, monkeypatch)
|
|
recovery.host("/boot/Image-matrix-axp313a1").write_bytes(b"damaged")
|
|
assert recovery.prepare_attempt("1.2.0:time") is False
|
|
assert recovery.status()["state"] == "requires_package"
|
|
assert recovery.prepare_attempt("1.2.1:later") is False
|
|
|
|
other = make_recovery(tmp_path / "other", monkeypatch)
|
|
assert other.prepare_attempt("1.2.0:time") is True
|
|
monkeypatch.setattr(kr.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess(args, 1))
|
|
other.arm_and_reboot(reboot=False)
|
|
other.host("/proc/sys/kernel/random/boot_id").write_text("new-boot", encoding="ascii")
|
|
other.finalize_after_boot(wait_seconds=1)
|
|
assert other.status()["state"] == "failed"
|
|
assert other.prepare_attempt("1.2.1:later") is False
|
|
|
|
|
|
def test_scheduled_recovery_that_never_runs_becomes_persistent_failure(tmp_path, monkeypatch):
|
|
recovery = make_recovery(tmp_path, monkeypatch)
|
|
assert recovery.prepare_attempt("1.2.0:time") is True
|
|
monkeypatch.setattr(kr.time, "time", lambda: recovery.read_record()["recorded_at"] + 181)
|
|
assert recovery.status()["state"] == "failed"
|
|
assert recovery.read_record()["state"] == "failed"
|
|
assert recovery.prepare_attempt("1.2.1:later") is False
|
|
|
|
|
|
def test_delayed_recovery_task_does_not_reboot(tmp_path, monkeypatch):
|
|
recovery = make_recovery(tmp_path, monkeypatch)
|
|
assert recovery.prepare_attempt("1.2.0:time") is True
|
|
monkeypatch.setattr(kr.time, "time", lambda: recovery.read_record()["recorded_at"] + 181)
|
|
monkeypatch.setattr(kr.subprocess, "run", lambda *args, **kwargs: pytest.fail("late task ran a command"))
|
|
recovery.arm_and_reboot()
|
|
assert recovery.read_record()["state"] == "failed"
|
|
|
|
|
|
def test_ota_rejects_new_cpufreq_loss_and_keeps_preexisting_degradation():
|
|
before = {"release": kr.CANDIDATE_RELEASE, "available": True, "marker": True,
|
|
"policies": {"policy0": {"current": "schedutil", "performance": True}}}
|
|
after = {**before, "policies": {}, "available": False}
|
|
off = {"requested": False, "available": True, "effective": False,
|
|
"current_governors": {"policy0": "schedutil"}}
|
|
with pytest.raises(UpdateFailed, match="lost the validated"):
|
|
verify_kernel_capability_retained(before, after, off, off)
|
|
verify_kernel_capability_retained({**before, "available": False}, after, off, off)
|
|
with pytest.raises(UpdateFailed, match="governors"):
|
|
verify_kernel_capability_retained(before, before, off, {**off, "current_governors": {"policy0": "performance"}})
|
|
|
|
|
|
def test_legacy_worker_completion_schedules_lightweight_recovery(tmp_path, monkeypatch):
|
|
class Display:
|
|
def start_ota_indicator(self, _path):
|
|
pass
|
|
|
|
def stop_ota_indicator(self):
|
|
pass
|
|
|
|
class Recovery:
|
|
def __init__(self):
|
|
self.attempts = []
|
|
|
|
def status(self):
|
|
return {"state": "ready", "reason": ""}
|
|
|
|
def prepare_attempt(self, result_id):
|
|
self.attempts.append(result_id)
|
|
return len(self.attempts) == 1
|
|
|
|
recovery = Recovery()
|
|
manager = OtaManager(
|
|
code_root=tmp_path / "code", data_root=tmp_path / "data", runtime_root=tmp_path / "run",
|
|
software_version=SoftwareVersion.parse("1.2.0"), display=Display(),
|
|
)
|
|
monkeypatch.setattr(manager, "_production_recovery", lambda: recovery)
|
|
calls = []
|
|
monkeypatch.setattr("app.ota.manager.subprocess.run", lambda command, **kwargs: calls.append(command))
|
|
write_json(manager.status_path, {"schema_version": 1, "active": True, "job": {"id": "old-worker"}})
|
|
assert manager.resume_or_start_monitor(manager.schedule_kernel_recovery_after_ota)
|
|
write_last_result(manager.data_root, {"status": "success", "target_version": "1.2.0",
|
|
"installed_at": "2026-09-26T08:00:00Z"})
|
|
write_json(manager.status_path, {"schema_version": 1, "active": False, "job": {"id": "old-worker"}})
|
|
manager._monitor_thread.join(timeout=3)
|
|
assert recovery.attempts == ["1.2.0:2026-09-26T08:00:00Z"]
|
|
assert len(calls) == 1
|
|
assert "systemd-run" in calls[0]
|
|
|
|
|
|
BOOT_SOURCE = '''# DO NOT EDIT THIS FILE
|
|
setenv docker_optimizations "on"
|
|
setenv bootargs "root=/dev/mmcblk1p2"
|
|
if test "${docker_optimizations}" = "on"; then setenv bootargs "${bootargs} cgroup_enable=memory swapaccount=1"; fi
|
|
if mmc dev 1; then
|
|
load ${devtype} ${devnum} ${fdt_addr_r} ${prefix}${fdtfile_emmc}.dtb
|
|
else
|
|
load ${devtype} ${devnum} ${fdt_addr_r} ${prefix}${fdtfile}.dtb
|
|
fi
|
|
if test "${overlay_error}" = "true"; then
|
|
load ${devtype} ${devnum} ${fdt_addr_r} ${prefix}${fdtfile}.dtb
|
|
fi
|
|
load ${devtype} ${devnum} ${kernel_addr_r} ${prefix}Image
|
|
'''
|
|
|
|
|
|
def test_dual_kernel_boot_consumes_health_marker_and_keeps_original_paths():
|
|
managed = render(BOOT_SOURCE)
|
|
|
|
assert "fatrm ${devtype} ${devnum} ${prefix}matrix-kernel-good" in managed
|
|
assert 'setenv matrix_kernel_file "Image-matrix-axp313a1"' in managed
|
|
assert 'setenv matrix_dtb_suffix "-matrix-axp313a1"' in managed
|
|
assert "matrix_kernel_candidate=1 panic=10" in managed
|
|
assert "${prefix}${fdtfile}${matrix_dtb_suffix}.dtb" in managed
|
|
assert "${prefix}${matrix_kernel_file}" in managed
|
|
assert 'setenv matrix_kernel_file "Image"' in managed
|
|
assert 'setenv matrix_dtb_suffix ""' in managed
|
|
|
|
|
|
def test_dual_kernel_boot_refuses_to_stack_management_blocks():
|
|
with pytest.raises(BootScriptError, match="already candidate-managed"):
|
|
render(render(BOOT_SOURCE))
|
|
|
|
|
|
def test_cpufreq_health_requires_policy0_all_cpus_and_performance(tmp_path):
|
|
policy = tmp_path / "cpufreq" / "policy0"
|
|
policy.mkdir(parents=True)
|
|
(policy / "affected_cpus").write_text("0 1 2 3\n", encoding="ascii")
|
|
(policy / "scaling_available_governors").write_text(
|
|
"powersave performance schedutil\n", encoding="ascii"
|
|
)
|
|
deferred = tmp_path / "devices_deferred"
|
|
deferred.write_text("", encoding="utf-8")
|
|
|
|
check_cpufreq(policy.parent, deferred)
|
|
|
|
deferred.write_text("platform cpufreq-dt\n", encoding="utf-8")
|
|
with pytest.raises(HealthError, match="still deferred"):
|
|
check_cpufreq(policy.parent, deferred)
|
|
|
|
|
|
def driver_status(frames: int, misses: int = 0) -> dict:
|
|
return {
|
|
"screen": {
|
|
"driver": "walnutpi-h618-hub75",
|
|
"driver_status": {
|
|
"actual_refresh_rate_hz": 97.5,
|
|
"panel_scan_rate_hz": 97.5,
|
|
"completed_frames": frames,
|
|
"completed_scans": frames,
|
|
"scans_per_frame": 1,
|
|
"deadline_misses": misses,
|
|
"oe_pulse_faults": 0,
|
|
"oe_forced_blanks": 0,
|
|
"driver_error": None,
|
|
"oe_timing_error": None,
|
|
"safeoff_error": None,
|
|
"cpu_affinity": 3,
|
|
"cpu_affinity_active": True,
|
|
"realtime_priority": 50,
|
|
"realtime_priority_active": True,
|
|
"memory_locked": True,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def test_candidate_health_accepts_only_progressing_equal_gpio_counters():
|
|
check_driver_pair(driver_status(1000), driver_status(1200))
|
|
|
|
stalled = driver_status(1000)
|
|
with pytest.raises(HealthError, match="stalled or unequal"):
|
|
check_driver_pair(stalled, stalled)
|
|
|
|
|
|
def test_candidate_health_rejects_axp313a_dcdc_boot_errors():
|
|
check_kernel_messages("axp20x-regulator: regulator probe completed\n")
|
|
with pytest.raises(HealthError, match="regulator logged a boot error"):
|
|
check_kernel_messages(
|
|
"axp20x-regulator axp20x-regulator: Error setting dcdc frequency: -22\n"
|
|
)
|
|
|
|
|
|
def test_kernel_scripts_pin_release_commits_and_preserve_original_boot_files():
|
|
source_root = Path(__file__).resolve().parents[1]
|
|
build = (source_root / "scripts/build_axp313a_kernel.sh").read_text(encoding="utf-8")
|
|
install = (source_root / "scripts/install_axp313a_kernel.sh").read_text(encoding="utf-8")
|
|
capture = (source_root / "scripts/capture_axp313a_state.sh").read_text(encoding="utf-8")
|
|
dts_patch = (source_root / "kernel/patches/0004-walnutpi-axp313a-dts.patch").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
dldo_patch = (
|
|
source_root / "kernel/patches/0005-walnutpi-axp313a-preserve-dldo1.patch"
|
|
).read_text(encoding="utf-8")
|
|
uwe_patch = (
|
|
source_root / "kernel/patches/0006-walnutpi-uwe5622-out-of-tree-build.patch"
|
|
).read_text(encoding="utf-8")
|
|
realtek_patch = (
|
|
source_root / "kernel/patches/0007-walnutpi-enabled-realtek-out-of-tree-build.patch"
|
|
).read_text(encoding="utf-8")
|
|
rtl8192_patch = (
|
|
source_root / "kernel/patches/0008-walnutpi-rtl8192eu-out-of-tree-build.patch"
|
|
).read_text(encoding="utf-8")
|
|
rtl8812_patch = (
|
|
source_root / "kernel/patches/0009-walnutpi-rtl8812au-out-of-tree-build.patch"
|
|
).read_text(encoding="utf-8")
|
|
rtl8812_phydm_patch = (
|
|
source_root / "kernel/patches/0010-walnutpi-rtl8812au-phydm-out-of-tree-build.patch"
|
|
).read_text(encoding="utf-8")
|
|
rtl88x2cs_topdir_patch = (
|
|
source_root / "kernel/patches/0011-walnutpi-rtl88x2cs-topdir-out-of-tree-build.patch"
|
|
).read_text(encoding="utf-8")
|
|
emmc_dts_patch = (
|
|
source_root / "kernel/patches/0012-walnutpi-emmc-axp313a-dts.patch"
|
|
).read_text(encoding="utf-8")
|
|
dcdc_frequency_patch = (
|
|
source_root / "kernel/patches/0013-walnutpi-axp313a-fixed-dcdc-frequency.patch"
|
|
).read_text(encoding="utf-8")
|
|
unit = (source_root / "systemd/matrix-axp313a-health.service").read_text(encoding="utf-8")
|
|
|
|
assert "30ff3fd5cf45417622b447a12e7a947402ffe34d" in build
|
|
assert "75c8cb2f4cb218aaf4ea68cab08d6dbc96eeae15.patch" in build
|
|
assert "60fd7eb89670d2636ac3156881acbd103c6eba6a.patch" in build
|
|
assert "4628b804555773daa982d9578d89fe04fcdde374.patch" in build
|
|
assert "/boot/boot.cmd.matrix-original" in install
|
|
assert "/boot/boot.scr.matrix-original" in install
|
|
assert "KERNEL_RELEASE" in build
|
|
assert "verify_dtb" in install
|
|
assert "dldo1 3300000 3300000 vcc3v3" in install
|
|
assert "cpu_supply" in install and "dcdc2_phandle" in install
|
|
assert "+\t\t\t\tregulator-min-microvolt = <1100000>;" in dts_patch
|
|
assert "reg_dldo1: dldo1" in dldo_patch
|
|
assert "regulator-min-microvolt = <3300000>" in dldo_patch
|
|
assert "0005-walnutpi-axp313a-preserve-dldo1.patch" in build
|
|
assert "0006-walnutpi-uwe5622-out-of-tree-build.patch" in build
|
|
assert "$(srctree)/drivers/net/wireless/uwe5622/unisocwcn" in uwe_patch
|
|
assert "0007-walnutpi-enabled-realtek-out-of-tree-build.patch" in build
|
|
for directory in ("rtl8189fs", "rtl8723ds", "rtl8723du", "rtl8811cu", "rtl88x2cs"):
|
|
assert f"$(srctree)/drivers/net/wireless/{directory}" in realtek_patch
|
|
assert "0008-walnutpi-rtl8192eu-out-of-tree-build.patch" in build
|
|
assert "$(srctree)/drivers/net/wireless/rtl8192eu" in rtl8192_patch
|
|
assert "0009-walnutpi-rtl8812au-out-of-tree-build.patch" in build
|
|
assert "$(srctree)/drivers/net/wireless/rtl8812au" in rtl8812_patch
|
|
assert "0010-walnutpi-rtl8812au-phydm-out-of-tree-build.patch" in build
|
|
assert "$(srctree)/drivers/net/wireless/rtl8812au/hal/phydm" in rtl8812_phydm_patch
|
|
assert "0011-walnutpi-rtl88x2cs-topdir-out-of-tree-build.patch" in build
|
|
assert "$(srctree)/drivers/net/wireless/rtl88x2cs/rtl8822c.mk" in rtl88x2cs_topdir_patch
|
|
assert "0012-walnutpi-emmc-axp313a-dts.patch" in build
|
|
assert "sun50i-h616-walnutpi-1b-emmc.dts" in emmc_dts_patch
|
|
assert "x-powers,axp313a" in emmc_dts_patch
|
|
assert "reg_dldo1: dldo1" in emmc_dts_patch
|
|
assert "0013-walnutpi-axp313a-fixed-dcdc-frequency.patch" in build
|
|
assert "+\t\tif (dcdcfreq != 0)" in dcdc_frequency_patch
|
|
assert "-\t\tif (dcdcfreq != 3000000)" in dcdc_frequency_patch
|
|
assert "/var/lib/matrix-screen-controller" in capture
|
|
assert "state output directory must not exist" in capture
|
|
assert "ConditionKernelCommandLine=matrix_kernel_candidate=1" in unit
|