import hashlib from pathlib import Path import subprocess import pytest from app.system import kernel_recovery as kr from app.ota.manager import OtaManager from app.ota.state import write_json, write_last_result from app.ota.versioning import SoftwareVersion from scripts.ota_worker import UpdateFailed, verify_kernel_capability_retained from scripts.axp313a_kernel_health import ( HealthError, check_cpufreq, check_driver_pair, check_kernel_messages, ) from scripts.render_dual_kernel_boot import BootScriptError, render def make_recovery(tmp_path: Path, monkeypatch) -> kr.KernelRecovery: host = tmp_path / "host" data = tmp_path / "data" boot = host / "boot" boot.mkdir(parents=True) files = { "Image-matrix-axp313a1": b"candidate-image", "sun50i-h616-walnutpi-1b-matrix-axp313a1.dtb": b"candidate-sd-dtb", "sun50i-h616-walnutpi-1b-emmc-matrix-axp313a1.dtb": b"candidate-emmc-dtb", } monkeypatch.setattr(kr, "BOOT_HASHES", {name: hashlib.sha256(value).hexdigest() for name, value in files.items()}) for name, value in files.items(): (boot / name).write_bytes(value) for name in ("Image", "sun50i-h616-walnutpi-1b.dtb", "sun50i-h616-walnutpi-1b-emmc.dtb", "boot.cmd.matrix-original", "boot.scr.matrix-original", "matrix-original.SHA256SUMS"): (boot / name).write_bytes(b"original") (boot / "boot.cmd").write_text( "matrix-kernel-good matrix-kernel-good matrix_kernel_candidate=1", encoding="utf-8", ) (boot / "boot.scr").write_bytes(b"matrix_kernel_candidate=1 Image-matrix-axp313a1") health = host / "etc/systemd/system/matrix-axp313a-health.service" health.parent.mkdir(parents=True) health.write_text("ConditionKernelCommandLine=matrix_kernel_candidate=1", encoding="utf-8") script = host / "opt/matrix-screen-controller-system/axp313a_kernel_health.py" script.parent.mkdir(parents=True) script.write_text("health", encoding="utf-8") enabled = host / "etc/systemd/system/multi-user.target.wants/matrix-axp313a-health.service" enabled.parent.mkdir(parents=True) enabled.write_text("enabled", encoding="utf-8") modules = host / "lib/modules" / kr.CANDIDATE_RELEASE (modules / "kernel").mkdir(parents=True) (modules / "modules.dep").write_text("index", encoding="ascii") module = modules / "kernel/example.ko" module.write_bytes(b"module") digest = hashlib.sha256(b"kernel/example.ko\0" + hashlib.sha256(b"module").digest()).hexdigest() monkeypatch.setattr(kr, "MODULE_COUNT", 1) monkeypatch.setattr(kr, "MODULE_DIGEST", digest) release = host / "proc/sys/kernel/osrelease" release.parent.mkdir(parents=True) release.write_text("6.1.31", encoding="ascii") boot_id = host / "proc/sys/kernel/random/boot_id" boot_id.parent.mkdir(parents=True) boot_id.write_text("old-boot", encoding="ascii") return kr.KernelRecovery(data, host_root=host) def test_candidate_validation_and_single_recovery_attempt(tmp_path, monkeypatch): recovery = make_recovery(tmp_path, monkeypatch) assert recovery.verify_candidate() is None assert recovery.prepare_attempt("1.2.0:time") is True assert recovery.prepare_attempt("1.2.0:time") is False monkeypatch.setattr(kr.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess(args, 1)) recovery.arm_and_reboot(reboot=False) assert recovery.status()["state"] == "pending" with pytest.raises(RuntimeError, match="no unused"): recovery.arm_and_reboot(reboot=False) recovery.host("/proc/sys/kernel/random/boot_id").write_text("new-boot", encoding="ascii") recovery.host("/proc/sys/kernel/osrelease").write_text(kr.CANDIDATE_RELEASE, encoding="ascii") policy = recovery.host("/sys/devices/system/cpu/cpufreq/policy0") policy.mkdir(parents=True) (policy / "scaling_governor").write_text("schedutil", encoding="ascii") (policy / "scaling_available_governors").write_text("schedutil performance", encoding="ascii") recovery.finalize_after_boot(wait_seconds=1) assert recovery.status()["state"] == "succeeded" def test_recovery_failure_and_damaged_candidate_do_not_retry(tmp_path, monkeypatch): recovery = make_recovery(tmp_path, monkeypatch) recovery.host("/boot/Image-matrix-axp313a1").write_bytes(b"damaged") assert recovery.prepare_attempt("1.2.0:time") is False assert recovery.status()["state"] == "requires_package" assert recovery.prepare_attempt("1.2.1:later") is False other = make_recovery(tmp_path / "other", monkeypatch) assert other.prepare_attempt("1.2.0:time") is True monkeypatch.setattr(kr.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess(args, 1)) other.arm_and_reboot(reboot=False) other.host("/proc/sys/kernel/random/boot_id").write_text("new-boot", encoding="ascii") other.finalize_after_boot(wait_seconds=1) assert other.status()["state"] == "failed" assert other.prepare_attempt("1.2.1:later") is False def test_scheduled_recovery_that_never_runs_becomes_persistent_failure(tmp_path, monkeypatch): recovery = make_recovery(tmp_path, monkeypatch) assert recovery.prepare_attempt("1.2.0:time") is True monkeypatch.setattr(kr.time, "time", lambda: recovery.read_record()["recorded_at"] + 181) assert recovery.status()["state"] == "failed" assert recovery.read_record()["state"] == "failed" assert recovery.prepare_attempt("1.2.1:later") is False def test_delayed_recovery_task_does_not_reboot(tmp_path, monkeypatch): recovery = make_recovery(tmp_path, monkeypatch) assert recovery.prepare_attempt("1.2.0:time") is True monkeypatch.setattr(kr.time, "time", lambda: recovery.read_record()["recorded_at"] + 181) monkeypatch.setattr(kr.subprocess, "run", lambda *args, **kwargs: pytest.fail("late task ran a command")) recovery.arm_and_reboot() assert recovery.read_record()["state"] == "failed" def test_ota_rejects_new_cpufreq_loss_and_keeps_preexisting_degradation(): before = {"release": kr.CANDIDATE_RELEASE, "available": True, "marker": True, "policies": {"policy0": {"current": "schedutil", "performance": True}}} after = {**before, "policies": {}, "available": False} off = {"requested": False, "available": True, "effective": False, "current_governors": {"policy0": "schedutil"}} with pytest.raises(UpdateFailed, match="lost the validated"): verify_kernel_capability_retained(before, after, off, off) verify_kernel_capability_retained({**before, "available": False}, after, off, off) with pytest.raises(UpdateFailed, match="governors"): verify_kernel_capability_retained(before, before, off, {**off, "current_governors": {"policy0": "performance"}}) def test_legacy_worker_completion_schedules_lightweight_recovery(tmp_path, monkeypatch): class Display: def start_ota_indicator(self, _path): pass def stop_ota_indicator(self): pass class Recovery: def __init__(self): self.attempts = [] def status(self): return {"state": "ready", "reason": ""} def prepare_attempt(self, result_id): self.attempts.append(result_id) return len(self.attempts) == 1 recovery = Recovery() manager = OtaManager( code_root=tmp_path / "code", data_root=tmp_path / "data", runtime_root=tmp_path / "run", software_version=SoftwareVersion.parse("1.2.0"), display=Display(), ) monkeypatch.setattr(manager, "_production_recovery", lambda: recovery) calls = [] monkeypatch.setattr("app.ota.manager.subprocess.run", lambda command, **kwargs: calls.append(command)) write_json(manager.status_path, {"schema_version": 1, "active": True, "job": {"id": "old-worker"}}) assert manager.resume_or_start_monitor(manager.schedule_kernel_recovery_after_ota) write_last_result(manager.data_root, {"status": "success", "target_version": "1.2.0", "installed_at": "2026-09-26T08:00:00Z"}) write_json(manager.status_path, {"schema_version": 1, "active": False, "job": {"id": "old-worker"}}) manager._monitor_thread.join(timeout=3) assert recovery.attempts == ["1.2.0:2026-09-26T08:00:00Z"] assert len(calls) == 1 assert "systemd-run" in calls[0] BOOT_SOURCE = '''# DO NOT EDIT THIS FILE setenv docker_optimizations "on" setenv bootargs "root=/dev/mmcblk1p2" if test "${docker_optimizations}" = "on"; then setenv bootargs "${bootargs} cgroup_enable=memory swapaccount=1"; fi if mmc dev 1; then load ${devtype} ${devnum} ${fdt_addr_r} ${prefix}${fdtfile_emmc}.dtb else load ${devtype} ${devnum} ${fdt_addr_r} ${prefix}${fdtfile}.dtb fi if test "${overlay_error}" = "true"; then load ${devtype} ${devnum} ${fdt_addr_r} ${prefix}${fdtfile}.dtb fi load ${devtype} ${devnum} ${kernel_addr_r} ${prefix}Image ''' def test_dual_kernel_boot_consumes_health_marker_and_keeps_original_paths(): managed = render(BOOT_SOURCE) assert "fatrm ${devtype} ${devnum} ${prefix}matrix-kernel-good" in managed assert 'setenv matrix_kernel_file "Image-matrix-axp313a1"' in managed assert 'setenv matrix_dtb_suffix "-matrix-axp313a1"' in managed assert "matrix_kernel_candidate=1 panic=10" in managed assert "${prefix}${fdtfile}${matrix_dtb_suffix}.dtb" in managed assert "${prefix}${matrix_kernel_file}" in managed assert 'setenv matrix_kernel_file "Image"' in managed assert 'setenv matrix_dtb_suffix ""' in managed def test_dual_kernel_boot_refuses_to_stack_management_blocks(): with pytest.raises(BootScriptError, match="already candidate-managed"): render(render(BOOT_SOURCE)) def test_cpufreq_health_requires_policy0_all_cpus_and_performance(tmp_path): policy = tmp_path / "cpufreq" / "policy0" policy.mkdir(parents=True) (policy / "affected_cpus").write_text("0 1 2 3\n", encoding="ascii") (policy / "scaling_available_governors").write_text( "powersave performance schedutil\n", encoding="ascii" ) deferred = tmp_path / "devices_deferred" deferred.write_text("", encoding="utf-8") check_cpufreq(policy.parent, deferred) deferred.write_text("platform cpufreq-dt\n", encoding="utf-8") with pytest.raises(HealthError, match="still deferred"): check_cpufreq(policy.parent, deferred) def driver_status(frames: int, misses: int = 0) -> dict: return { "screen": { "driver": "walnutpi-h618-hub75", "driver_status": { "actual_refresh_rate_hz": 97.5, "panel_scan_rate_hz": 97.5, "completed_frames": frames, "completed_scans": frames, "scans_per_frame": 1, "deadline_misses": misses, "oe_pulse_faults": 0, "oe_forced_blanks": 0, "driver_error": None, "oe_timing_error": None, "safeoff_error": None, "cpu_affinity": 3, "cpu_affinity_active": True, "realtime_priority": 50, "realtime_priority_active": True, "memory_locked": True, }, }, } def test_candidate_health_accepts_only_progressing_equal_gpio_counters(): check_driver_pair(driver_status(1000), driver_status(1200)) stalled = driver_status(1000) with pytest.raises(HealthError, match="stalled or unequal"): check_driver_pair(stalled, stalled) def test_candidate_health_rejects_axp313a_dcdc_boot_errors(): check_kernel_messages("axp20x-regulator: regulator probe completed\n") with pytest.raises(HealthError, match="regulator logged a boot error"): check_kernel_messages( "axp20x-regulator axp20x-regulator: Error setting dcdc frequency: -22\n" ) def test_kernel_scripts_pin_release_commits_and_preserve_original_boot_files(): source_root = Path(__file__).resolve().parents[1] build = (source_root / "scripts/build_axp313a_kernel.sh").read_text(encoding="utf-8") install = (source_root / "scripts/install_axp313a_kernel.sh").read_text(encoding="utf-8") capture = (source_root / "scripts/capture_axp313a_state.sh").read_text(encoding="utf-8") dts_patch = (source_root / "kernel/patches/0004-walnutpi-axp313a-dts.patch").read_text( encoding="utf-8" ) dldo_patch = ( source_root / "kernel/patches/0005-walnutpi-axp313a-preserve-dldo1.patch" ).read_text(encoding="utf-8") uwe_patch = ( source_root / "kernel/patches/0006-walnutpi-uwe5622-out-of-tree-build.patch" ).read_text(encoding="utf-8") realtek_patch = ( source_root / "kernel/patches/0007-walnutpi-enabled-realtek-out-of-tree-build.patch" ).read_text(encoding="utf-8") rtl8192_patch = ( source_root / "kernel/patches/0008-walnutpi-rtl8192eu-out-of-tree-build.patch" ).read_text(encoding="utf-8") rtl8812_patch = ( source_root / "kernel/patches/0009-walnutpi-rtl8812au-out-of-tree-build.patch" ).read_text(encoding="utf-8") rtl8812_phydm_patch = ( source_root / "kernel/patches/0010-walnutpi-rtl8812au-phydm-out-of-tree-build.patch" ).read_text(encoding="utf-8") rtl88x2cs_topdir_patch = ( source_root / "kernel/patches/0011-walnutpi-rtl88x2cs-topdir-out-of-tree-build.patch" ).read_text(encoding="utf-8") emmc_dts_patch = ( source_root / "kernel/patches/0012-walnutpi-emmc-axp313a-dts.patch" ).read_text(encoding="utf-8") dcdc_frequency_patch = ( source_root / "kernel/patches/0013-walnutpi-axp313a-fixed-dcdc-frequency.patch" ).read_text(encoding="utf-8") unit = (source_root / "systemd/matrix-axp313a-health.service").read_text(encoding="utf-8") assert "30ff3fd5cf45417622b447a12e7a947402ffe34d" in build assert "75c8cb2f4cb218aaf4ea68cab08d6dbc96eeae15.patch" in build assert "60fd7eb89670d2636ac3156881acbd103c6eba6a.patch" in build assert "4628b804555773daa982d9578d89fe04fcdde374.patch" in build assert "/boot/boot.cmd.matrix-original" in install assert "/boot/boot.scr.matrix-original" in install assert "KERNEL_RELEASE" in build assert "verify_dtb" in install assert "dldo1 3300000 3300000 vcc3v3" in install assert "cpu_supply" in install and "dcdc2_phandle" in install assert "+\t\t\t\tregulator-min-microvolt = <1100000>;" in dts_patch assert "reg_dldo1: dldo1" in dldo_patch assert "regulator-min-microvolt = <3300000>" in dldo_patch assert "0005-walnutpi-axp313a-preserve-dldo1.patch" in build assert "0006-walnutpi-uwe5622-out-of-tree-build.patch" in build assert "$(srctree)/drivers/net/wireless/uwe5622/unisocwcn" in uwe_patch assert "0007-walnutpi-enabled-realtek-out-of-tree-build.patch" in build for directory in ("rtl8189fs", "rtl8723ds", "rtl8723du", "rtl8811cu", "rtl88x2cs"): assert f"$(srctree)/drivers/net/wireless/{directory}" in realtek_patch assert "0008-walnutpi-rtl8192eu-out-of-tree-build.patch" in build assert "$(srctree)/drivers/net/wireless/rtl8192eu" in rtl8192_patch assert "0009-walnutpi-rtl8812au-out-of-tree-build.patch" in build assert "$(srctree)/drivers/net/wireless/rtl8812au" in rtl8812_patch assert "0010-walnutpi-rtl8812au-phydm-out-of-tree-build.patch" in build assert "$(srctree)/drivers/net/wireless/rtl8812au/hal/phydm" in rtl8812_phydm_patch assert "0011-walnutpi-rtl88x2cs-topdir-out-of-tree-build.patch" in build assert "$(srctree)/drivers/net/wireless/rtl88x2cs/rtl8822c.mk" in rtl88x2cs_topdir_patch assert "0012-walnutpi-emmc-axp313a-dts.patch" in build assert "sun50i-h616-walnutpi-1b-emmc.dts" in emmc_dts_patch assert "x-powers,axp313a" in emmc_dts_patch assert "reg_dldo1: dldo1" in emmc_dts_patch assert "0013-walnutpi-axp313a-fixed-dcdc-frequency.patch" in build assert "+\t\tif (dcdcfreq != 0)" in dcdc_frequency_patch assert "-\t\tif (dcdcfreq != 3000000)" in dcdc_frequency_patch assert "/var/lib/matrix-screen-controller" in capture assert "state output directory must not exist" in capture assert "ConditionKernelCommandLine=matrix_kernel_candidate=1" in unit