From ebd13eb569ddfbdfd21caac8bbfb1e1834e394eb Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:29:49 +0300 Subject: [PATCH 1/3] install: write the UBI-only NAND layout for u-boot-xmedia SoCs OpenIPC retired the HiSilicon split NAND layout (1M boot, 1M env, 8M raw kernel, ubi) and the u-boot-hi3516ev200 "universal" build that carried it. hi3516ev200, hi3516ev300, hi3518ev300, hi3516dv200 and gk7205v500/v510/v530 now boot u-boot-xmedia, published per flash type in the OpenIPC/firmware latest release as u-boot--nor.bin and u-boot--nand.bin. The NAND build uses one layout: 0x000000 boot 768K u-boot--nand.bin 0x0C0000 env 256K 0x100000 ubi rest rootfs.ubi. (rootfs volume with the kernel as /boot/fitImage, plus rootfs_data) and its default environment defines mtdids/mtdparts, bootcmd and bootargs for it. firmware: PER_FLASH_TYPE_UBOOT lists those SoCs. asset_name(), firmware_url(), has_firmware(), get_cached_path() and download_firmware() take an optional flash_type ("nor"/"nand", NOR when unknown) and resolve u-boot--.bin for them; other chips ignore it. The universal image is never downloaded for these SoCs. A universal image already in the cache is no longer reported by get_cached_path() (it would shadow the published build), but download_firmware() still falls back to it for NOR when the download fails. gk7205v5x0 NOR shares its cache name with download_v500_donor(), so either source satisfies the other. Callers: install picks the build from --nand; burn gains --nand (NOR by default); restore uses the NAND build for --flash-type nand and warns that "auto" means NOR for these SoCs. agent upload/flash only take the SPL, whose DDR init is the same in both builds, so they keep the NOR default. install: with --nand on those SoCs the plan becomes uboot tftp u (padded to 0xC0000), crc32, nand erase 0x0 0xc0000, nand write 0x0 0xc0000 kernel no-op: the kernel is inside the UBI image rootfs tftp r (rootfs.ubi.), crc32, printenv mtdparts, nand erase.part ubi, nand write.trimffs 0x100000 , nand read + crc32 readback when crc32 is available rootfs-data no-op: the UBI image carries an empty rootfs_data env (if uboot was written) env default -a, restore ethaddr, saveenv reset with no setenv mtdparts/bootcmd/bootargs and no ubi create/write. write.trimffs is required: a plain nand write programs the image's 0xFF pages and UBIFS programming them again breaks ECC (OpenIPC/firmware#2519). erase.part erases the whole partition, so chips over 128 MiB lose every stale block. It runs only when the live mtdparts puts ubi at 0x100000; a different offset aborts before any erase. Without mtdparts, or when the U-Boot lacks erase.part (no "Erasing at" in the reply), the installer erases with `nand erase 0x100000`, which U-Boot runs to the chip end. The reported erase offset must be 0x100000 and the image must fit the reported size. The firmware package member is rootfs.ubi. (never rootfs.ubifs.*); a package without one is rejected with a pointer to openipc.-nand-.tgz. The split layout stays only for `install --nand` on other chips: it is chip-agnostic code the existing hi3516cv300 test drives, and nothing in the tree says those chips moved. Tests: offline FakeNandUBoot console checks the exact hi3516ev300 NAND command sequence, the erase.part fallback, the missing-mtdparts path, the refusal on a foreign ubi offset, an env-only run, the NOR install picking u-boot-hi3516ev300-nor.bin, package member selection, and the per-flash naming for all seven SoCs. Untested on hardware: the full install. The command sequence follows the manual procedure verified on a hi3516ev300 with a defib-burned u-boot-hi3516ev300-nand.bin; the erase.part fallback and the readback CRC step have not run on a camera. --- CLAUDE.md | 9 +- README.md | 28 ++ src/defib/cli/app.py | 42 ++- src/defib/firmware.py | 125 +++++-- src/defib/install/firmware.py | 61 +++- src/defib/install/layout.py | 73 +++++ src/defib/install/orchestrator.py | 255 +++++++++++++-- tests/test_firmware.py | 108 ++++++- tests/test_nand_ubi_install.py | 520 ++++++++++++++++++++++++++++++ 9 files changed, 1155 insertions(+), 66 deletions(-) create mode 100644 tests/test_nand_ubi_install.py diff --git a/CLAUDE.md b/CLAUDE.md index eb97903..3e4563e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -164,7 +164,14 @@ Backends: `spi_flash.c` (fmc100), `spi_flash_hisfc350.c` (V1-era parts), filename-based partition routing, plus temporary static-IP management and U-Boot device discovery. - **Firmware** (`src/defib/firmware.py`) — downloads OpenIPC releases from - GitHub, caches under `XDG_CACHE_HOME`. + GitHub, caches under `XDG_CACHE_HOME`. Most chips use one + `u-boot--universal.bin`; the u-boot-xmedia SoCs in + `PER_FLASH_TYPE_UBOOT` (hi3516ev200/ev300, hi3518ev300, hi3516dv200, + gk7205v500/v510/v530) publish `u-boot--nor.bin` and `-nand.bin` + instead, so callers pass `flash_type` (NOR when unknown). On NAND those SoCs + install the UBI-only layout (`NAND_UBI_LAYOUT` in `install/layout.py`: 768k + boot, 256k env, rest ubi), whose mtdparts/bootcmd/bootargs come from the + U-Boot default env; `NAND_LAYOUT` is the legacy split layout for other chips. - **Capture** (`src/defib/capture/`) — record/replay UART sessions in `.dcap`. - Loose modules worth knowing: `flashdump.py` (dump flash through a U-Boot console), `ubi.py` (extract UBIFS volumes from raw UBI), `uboot_env.py`, diff --git a/README.md b/README.md index 82dcb4d..07d7843 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,34 @@ Requires root for TFTP port 69 and NIC IP assignment. Standard 8/16/32 MiB NOR layouts are selected from U-Boot flash detection; `--nor-size` remains an explicit override. +hi3516ev200, hi3516ev300, hi3518ev300, hi3516dv200 and gk7205v500/v510/v530 +take their U-Boot from OpenIPC/u-boot-xmedia, published once per flash type as +`u-boot--nor.bin` and `u-boot--nand.bin`. `install` picks the build +from `--nand`; `burn` loads the NOR build unless `--nand` is given. On NAND +these SoCs use a UBI-only layout: + +| Offset | Size | Contents | +|--------|------|----------| +| `0x000000` | 768K | boot: `u-boot--nand.bin` | +| `0x0C0000` | 256K | env | +| `0x100000` | rest of chip | ubi: `rootfs.ubi.` (volume `rootfs` with the kernel as `/boot/fitImage`, plus `rootfs_data`) | + +```bash +defib install -c hi3516ev300 --nand \ + --firmware openipc.hi3516ev300-nand-lite.tgz \ + -p /dev/ttyUSB0 --power-cycle +``` + +The NAND U-Boot's default environment defines `mtdparts`, `bootcmd` and +`bootargs` for this layout, so the installer leaves them alone: it writes U-Boot, +erases the `ubi` partition (`nand erase.part ubi`) and writes the UBI image with +`nand write.trimffs`, then resets the environment to the U-Boot defaults +(`env default -a`), restores the camera's `ethaddr` and saves. The `kernel` and +`rootfs-data` stages are no-ops on this layout: both live inside the UBI image. +gk7205v510/v530 NAND packages are published under board `gk7205v500`. +`install --nand` on any other chip still uses the older split layout (raw +kernel partition, `mtdparts` and `bootcmd` set by the installer). + Targets that must bootstrap through a stock U-Boot use an explicit U-Boot variant. For example, HiWatch DS-I203 uses: diff --git a/src/defib/cli/app.py b/src/defib/cli/app.py index ccd066d..8f4190a 100644 --- a/src/defib/cli/app.py +++ b/src/defib/cli/app.py @@ -33,6 +33,7 @@ def burn( chip: str = typer.Option(..., "-c", "--chip", help="Chip model name"), file: str = typer.Option("", "-f", "--file", help="Firmware file (auto-downloads from OpenIPC if omitted)"), port: str = typer.Option("/dev/ttyUSB0", "-p", "--port", help="Serial device (/dev/ttyUSB0), tcp://host:port, rfc2217://host:port, or socket:///path"), + nand: bool = typer.Option(False, "--nand", help="Auto-download the NAND U-Boot build (u-boot--nand.bin) for SoCs published per flash type; the NOR build is the default"), send_break: bool = typer.Option(False, "-b", "--break", help="Send Ctrl-C after upload"), terminal: bool = typer.Option(False, "-t", "--terminal", help="Open serial terminal after upload"), power_cycle: bool = typer.Option(False, "--power-cycle", help="Auto power-cycle via the controller selected by DEFIB_POWER_TYPE (default routeros, needs DEFIB_POE_* env vars)"), @@ -55,14 +56,14 @@ def burn( into MaskROM on its own at power-up, so --power-cycle is all it takes. """ import asyncio - asyncio.run(_burn_async(chip, file, port, send_break, terminal, power_cycle, poe_port_override, output, debug, ddr, usbplug, loader, wait, usb_path)) + asyncio.run(_burn_async(chip, file, port, send_break, terminal, power_cycle, poe_port_override, output, debug, ddr, usbplug, loader, wait, usb_path, flash_type="nand" if nand else "nor")) async def _burn_async( chip: str, file: str, port: str, send_break: bool, terminal: bool, power_cycle: bool, poe_port_override: str, output: str, debug: bool, ddr: str = "", usbplug: str = "", loader: str = "", wait: float = 30.0, - usb_path: str = "", + usb_path: str = "", flash_type: str = "nor", ) -> None: import json as json_mod import logging @@ -92,7 +93,7 @@ async def _burn_async( if not firmware_path: from defib.firmware import has_firmware, download_firmware, get_cached_path - if not has_firmware(chip): + if not has_firmware(chip, flash_type): msg = ( f"No pre-built firmware for '{chip}' on OpenIPC. " f"Specify a local file with -f/--file." @@ -103,7 +104,7 @@ async def _burn_async( console.print(f"[red]{msg}[/red]") raise typer.Exit(1) - cached = get_cached_path(chip) + cached = get_cached_path(chip, flash_type) if cached: firmware_path = str(cached) if output == "human": @@ -121,7 +122,9 @@ def _dl_progress(done: int, total: int) -> None: elif output == "json" and done == total: print(json_mod.dumps({"event": "download_complete", "bytes": total}), flush=True) - path = download_firmware(chip, on_progress=_dl_progress) + path = download_firmware( + chip, on_progress=_dl_progress, flash_type=flash_type, + ) firmware_path = str(path) if output == "human": console.print(f"\n Saved: [cyan]{path.name}[/cyan] ({path.stat().st_size} bytes)") @@ -2498,7 +2501,15 @@ def install( 0, "--nor-size", help="NOR size override in MB; 0 auto-detects from U-Boot", ), - nand: bool = typer.Option(False, "--nand", help="Use NAND flash instead of NOR"), + nand: bool = typer.Option( + False, "--nand", + help=( + "Use NAND flash instead of NOR. hi3516ev200/ev300, hi3518ev300, " + "hi3516dv200 and gk7205v500/v510/v530 get the UBI layout (768k boot, " + "256k env, rest ubi): u-boot--nand.bin plus the package's " + "rootfs.ubi., which carries the kernel." + ), + ), wipe_env: bool = typer.Option( False, "--wipe-env", @@ -2729,12 +2740,25 @@ async def _restore_async( # --- Resolve U-Boot binary --- if not uboot_path: from defib.firmware import get_cached_path, download_firmware, has_firmware - if has_firmware(chip): - cached = get_cached_path(chip) + # SoCs with per-flash-type U-Boot builds need the NAND build to drive + # NAND; anything other than an explicit --flash-type nand gets NOR. + uboot_flash_type = "nand" if flash_type.lower() == "nand" else "nor" + from defib.firmware import uses_per_flash_type_uboot + if ( + flash_type.lower() == "auto" + and uses_per_flash_type_uboot(chip) + and output == "human" + ): + console.print( + f" [yellow]{chip} publishes separate NOR and NAND U-Boot builds; " + "using NOR. Pass --flash-type nand for a NAND camera.[/yellow]" + ) + if has_firmware(chip, uboot_flash_type): + cached = get_cached_path(chip, uboot_flash_type) if not cached: if output == "human": console.print(f" Downloading U-Boot for [cyan]{chip}[/cyan]...") - cached = download_firmware(chip) + cached = download_firmware(chip, flash_type=uboot_flash_type) uboot_path = str(cached) else: console.print(f"[red]No U-Boot for '{chip}'. Specify --uboot.[/red]") diff --git a/src/defib/firmware.py b/src/defib/firmware.py index 6e06bab..8aa897b 100644 --- a/src/defib/firmware.py +++ b/src/defib/firmware.py @@ -5,6 +5,9 @@ - Classic SoCs: ``u-boot-{chip}-universal.bin`` — a bare U-Boot image. - Selected classic board variants may publish a dedicated U-Boot release asset. +- u-boot-xmedia SoCs (hi3516ev200/ev300, hi3518ev300, hi3516dv200, + gk7205v500/v510/v530): ``u-boot-{chip}-{nor|nand}.bin`` — one bare U-Boot per + flash type, because each build carries that flash type's partition layout. - CV6xx SoCs: ``boot-{chip}[-{variant}]-nor.bin`` — a composite image (GSL + DDR tables + U-Boot) that the bootrom expects as a single blob. @@ -32,13 +35,28 @@ "gk7202v300", "gk7205v200", "gk7205v300", "gk7605v100", "hi3516av100", "hi3516av200", "hi3516av300", "hi3516cv100", "hi3516cv200", "hi3516cv300", "hi3516cv500", - "hi3516dv100", "hi3516dv200", "hi3516dv300", - "hi3516ev100", "hi3516ev200", "hi3516ev300", - "hi3518av100", "hi3518cv100", "hi3518ev100", "hi3518ev200", "hi3518ev300", + "hi3516dv100", "hi3516dv300", + "hi3516ev100", + "hi3518av100", "hi3518cv100", "hi3518ev100", "hi3518ev200", "hi3519v101", "hi3520dv200", "hi3536cv100", "hi3536dv100", "t40a", "t40n", "t40xp", } +# SoCs whose OpenIPC U-Boot is built from OpenIPC/u-boot-xmedia and published +# once per flash type as u-boot-{chip}-nor.bin / u-boot-{chip}-nand.bin. The +# NAND build owns the UBI-only layout (768k boot, 256k env, rest ubi) in its +# default environment, so a NOR image must never be installed on NAND or vice +# versa. The old u-boot-{chip}-universal.bin of the HiSilicon members was the +# retired u-boot-hi3516ev200 build with the split NAND layout; it is never +# downloaded for these SoCs again. +PER_FLASH_TYPE_UBOOT: frozenset[str] = frozenset({ + "hi3516ev200", "hi3516ev300", "hi3518ev300", "hi3516dv200", + "gk7205v500", "gk7205v510", "gk7205v530", +}) + +FLASH_TYPES: tuple[str, ...] = ("nor", "nand") +DEFAULT_FLASH_TYPE = "nor" + # CV6xx SoCs publish a composite boot image, not a bare U-Boot, and no # u-boot-{chip}-universal.bin exists for any of them — the URL assumed by # #112 always 404'd, so auto-download could never work and users had to @@ -68,8 +86,7 @@ } # Chip aliases: map chip names to the firmware download name -# e.g. hi3516ev300 profile resolves to hi3516ev200 internally, -# but the firmware binary is named u-boot-hi3516ev300-universal.bin +# e.g. hi3518ev201 has its own profile but boots the hi3518ev200 U-Boot. CHIP_TO_FIRMWARE: dict[str, str] = { "hi3518ev201": "hi3518ev200", "hi3516dv100": "hi3516dv100", @@ -109,14 +126,42 @@ def _strip_variant(chip: str) -> str: return _split_variant(chip)[0] -def asset_name(chip: str) -> str | None: +def normalize_flash_type(flash_type: str | None) -> str: + """Normalize an optional flash type; NOR when the caller does not know.""" + if flash_type is None: + return DEFAULT_FLASH_TYPE + normalized = flash_type.strip().lower() + if normalized not in FLASH_TYPES: + raise ValueError( + f"unknown flash type {flash_type!r}; expected one of: " + + ", ".join(FLASH_TYPES) + ) + return normalized + + +def uses_per_flash_type_uboot(chip: str) -> bool: + """True for SoCs whose U-Boot is published per flash type (nor/nand).""" + base = _strip_variant(chip).lower() + return CHIP_TO_FIRMWARE.get(base, base) in PER_FLASH_TYPE_UBOOT + + +def asset_name(chip: str, flash_type: str | None = None) -> str | None: """Published release filename for a chip, or None if there isn't one. Returns None both for chips OpenIPC doesn't build and for a CV6xx chip named without the variant needed to pick between several board images. + + ``flash_type`` (``"nor"``/``"nand"``) picks the build for SoCs listed in + PER_FLASH_TYPE_UBOOT and defaults to NOR there. Every other chip publishes + one image whatever its flash, so the argument does not change its name. """ + kind = normalize_flash_type(flash_type) base, variant = _split_variant(chip) - name = CHIP_TO_FIRMWARE.get(base, base) + name = CHIP_TO_FIRMWARE.get(base.lower(), base.lower()) + + if name in PER_FLASH_TYPE_UBOOT: + # Board variants (e.g. ``:emmc``) do not change the per-flash build. + return f"u-boot-{name}-{kind}.bin" if name in CV6XX_BOOT_VARIANTS: variants = CV6XX_BOOT_VARIANTS[name] @@ -136,19 +181,22 @@ def asset_name(chip: str) -> str | None: return None -def firmware_url(chip: str) -> str | None: +def firmware_url(chip: str, flash_type: str | None = None) -> str | None: """Get the OpenIPC download URL for a chip, or None if unavailable.""" - name = asset_name(chip) + name = asset_name(chip, flash_type) return f"{OPENIPC_BASE_URL}/{name}" if name else None -def has_firmware(chip: str) -> bool: +def has_firmware(chip: str, flash_type: str | None = None) -> bool: """Check if firmware can be obtained for this chip. True when OpenIPC publishes an image *or* one is already cached — the latter keeps hand-seeded blobs working for chips with no published build. """ - return firmware_url(chip) is not None or get_cached_path(chip) is not None + return ( + firmware_url(chip, flash_type) is not None + or get_cached_path(chip, flash_type) is not None + ) def _legacy_cache_name(chip: str) -> str: @@ -159,23 +207,36 @@ def _legacy_cache_name(chip: str) -> str: return f"u-boot-{CHIP_TO_FIRMWARE.get(base, base)}-universal.bin" -def get_cached_path(chip: str) -> Path | None: +def _cached_file(name: str) -> Path | None: + path = get_cache_dir() / name + if path.exists() and path.stat().st_size > 0: + return path + return None + + +def get_cached_path(chip: str, flash_type: str | None = None) -> Path | None: """Get the path to cached firmware, or None if not cached. A registered classic board variant must never fall back to the chip-wide universal cache entry: doing so could select incompatible DDR init data. + + Per-flash-type SoCs report only their exact ``-nor``/``-nand`` entry, so a + universal image cached from the retired build cannot shadow the published + one. download_firmware() still reads that universal entry for NOR, as a + last resort when the download itself fails. """ - cache_dir = get_cache_dir() base, variant = _split_variant(chip) name = CHIP_TO_FIRMWARE.get(base, base) - candidates: list[str | None] = [asset_name(chip)] - if not (variant is not None and name in CLASSIC_UBOOT_VARIANTS): + candidates: list[str | None] = [asset_name(chip, flash_type)] + if not uses_per_flash_type_uboot(chip) and not ( + variant is not None and name in CLASSIC_UBOOT_VARIANTS + ): candidates.append(_legacy_cache_name(chip)) for candidate in candidates: if not candidate: continue - path = cache_dir / candidate - if path.exists() and path.stat().st_size > 0: + path = _cached_file(candidate) + if path is not None: return path return None @@ -224,12 +285,15 @@ def _unavailable_message(chip: str) -> str: def download_firmware( chip: str, on_progress: Callable[[int, int], None] | None = None, + flash_type: str | None = None, ) -> Path: """Download U-Boot firmware from OpenIPC, with caching. Args: chip: Chip name (e.g., "hi3516ev300"). on_progress: Optional callback(bytes_downloaded, total_bytes). + flash_type: "nor" or "nand" for SoCs that publish one U-Boot per + flash type (NOR when omitted); other chips ignore it. Returns: Path to the downloaded (or cached) firmware file. @@ -238,26 +302,43 @@ def download_firmware( ValueError: If no firmware is available for this chip. ConnectionError: If download fails. """ - url = firmware_url(chip) + url = firmware_url(chip, flash_type) if url is None: # Check the cache before giving up: a chip with no published build may # still have a hand-seeded blob. - cached = get_cached_path(chip) + cached = get_cached_path(chip, flash_type) if cached is not None: logger.info("Using cached firmware: %s", cached) return cached raise ValueError(_unavailable_message(chip)) # Check cache - cached = get_cached_path(chip) + cached = get_cached_path(chip, flash_type) if cached is not None: logger.info("Using cached firmware: %s", cached) return cached # Download - name = asset_name(chip) + name = asset_name(chip, flash_type) assert name is not None # firmware_url() is None otherwise - return _download(url, get_cache_dir() / name, on_progress) + try: + return _download(url, get_cache_dir() / name, on_progress) + except ConnectionError: + # A universal image cached before the per-flash-type split still boots + # a NOR board, so keep it usable offline. It is never downloaded, and + # never used for NAND, whose split layout it carries is retired. + if ( + uses_per_flash_type_uboot(chip) + and normalize_flash_type(flash_type) == "nor" + ): + legacy = _cached_file(_legacy_cache_name(chip)) + if legacy is not None: + logger.warning( + "Download of %s failed; using cached legacy %s", + name, legacy.name, + ) + return legacy + raise def _download( diff --git a/src/defib/install/firmware.py b/src/defib/install/firmware.py index 1395f1b..26dceac 100644 --- a/src/defib/install/firmware.py +++ b/src/defib/install/firmware.py @@ -12,7 +12,12 @@ @dataclass(frozen=True) class FirmwareBundle: - """Kernel and rootfs payloads extracted from one OpenIPC firmware archive.""" + """Kernel and rootfs payloads extracted from one OpenIPC firmware archive. + + For a UBI-only NAND package the kernel lives inside the rootfs image + (``/boot/fitImage`` in the UBIFS volume): ``kernel_name`` is empty and + ``kernel`` is ``b""``. + """ kernel_name: str kernel: bytes @@ -20,12 +25,24 @@ class FirmwareBundle: rootfs: bytes -def load_firmware_bundle(path: str | Path) -> FirmwareBundle: - """Read kernel/rootfs and verify any matching md5sum entries in one pass.""" +def _is_ubi_member(name: str) -> bool: + """``rootfs.ubi.`` (or bare ``rootfs.ubi``), never ``rootfs.ubifs.*``.""" + return name == "rootfs.ubi" or name.startswith("rootfs.ubi.") + + +def load_firmware_bundle(path: str | Path, *, ubi_only: bool = False) -> FirmwareBundle: + """Read kernel/rootfs and verify any matching md5sum entries in one pass. + + ``ubi_only`` selects the UBI-only NAND package + (``openipc.-nand-.tgz``): the payload is the single + ``rootfs.ubi.`` image, which already contains the kernel, and no + ``uImage`` is expected. + """ kernel_name = "" kernel: bytes | None = None rootfs_name = "" rootfs: bytes | None = None + ubi_members: list[str] = [] expected_md5: dict[str, str] = {} with tarfile.open(path, "r:gz") as archive: @@ -34,21 +51,45 @@ def load_firmware_bundle(path: str | Path) -> FirmwareBundle: continue stream = archive.extractfile(member) assert stream is not None - if member.name.endswith(".md5sum"): + name = member.name + if name.endswith(".md5sum"): line = stream.read().decode().strip() if line: - expected_md5[member.name.removesuffix(".md5sum")] = line.split()[0] - elif member.name.startswith("uImage"): - kernel_name = member.name + expected_md5[name.removesuffix(".md5sum")] = line.split()[0] + elif ubi_only: + if _is_ubi_member(name): + ubi_members.append(name) + rootfs_name = name + rootfs = stream.read() + elif name.startswith("uImage"): + kernel_name = name kernel = stream.read() - elif member.name.startswith(("rootfs.squashfs", "rootfs.ubi")): - rootfs_name = member.name + elif name.startswith("rootfs.squashfs") or _is_ubi_member(name): + rootfs_name = name rootfs = stream.read() - if not kernel or not rootfs: + if ubi_only: + if not rootfs: + raise ValueError( + "tarball has no rootfs.ubi. image; this NAND layout needs " + "the OpenIPC NAND package openipc.-nand-.tgz" + ) + if len(ubi_members) > 1: + raise ValueError( + "tarball has more than one rootfs.ubi image: " + ", ".join(ubi_members) + ) + from defib.ubi import is_ubi_image + + if not is_ubi_image(rootfs): + raise ValueError(f"{rootfs_name} is not a UBI image (no UBI# header)") + kernel = b"" + elif not kernel or not rootfs: raise ValueError("tarball missing uImage or rootfs (squashfs/ubi)") + assert kernel is not None and rootfs is not None for name, data in ((kernel_name, kernel), (rootfs_name, rootfs)): + if not name: + continue expected = expected_md5.get(name) if expected is not None and hashlib.md5(data).hexdigest() != expected: raise ValueError(f"MD5 mismatch for {name}") diff --git a/src/defib/install/layout.py b/src/defib/install/layout.py index b6a4a80..ef16af5 100644 --- a/src/defib/install/layout.py +++ b/src/defib/install/layout.py @@ -35,6 +35,10 @@ "rootfs": (0x350000, 0x1800000), } +# Legacy split NAND layout (boot, env, raw kernel, UBI) of the retired +# u-boot-hi3516ev200 build. OpenIPC no longer ships it for the SoCs in +# defib.firmware.PER_FLASH_TYPE_UBOOT, which use NAND_UBI_LAYOUT instead. It is +# kept only for `install --nand` on other chips, whose U-Boot defines no layout. NAND_LAYOUT = { "boot": (0x000000, 0x100000), "env": (0x100000, 0x100000), @@ -42,6 +46,75 @@ "rootfs": (0xA00000, 0x7600000), } +# UBI-only NAND layout of the u-boot-xmedia NAND builds: +# :768k(boot),256k(env),-(ubi) +# The ubi partition runs to the end of the chip and holds one UBI image with +# the rootfs volume (UBIFS, kernel inside as /boot/fitImage) and rootfs_data. +# U-Boot's default environment carries mtdids/mtdparts/bootcmd/bootargs for it. +NAND_UBI_LAYOUT = { + "boot": (0x000000, 0x0C0000), + "env": (0x0C0000, 0x040000), +} +NAND_UBI_OFFSET = 0x100000 + +_MTDPART_RE = re.compile( + r"^(?P-|(?:0x[0-9a-f]+|\d+)[kmg]?)" + r"(?:@(?P(?:0x[0-9a-f]+|\d+)[kmg]?))?" + r"\((?P[^)]*)\)", + re.IGNORECASE, +) + + +def _mtd_size(text: str) -> int: + multiplier = {"k": 1024, "m": 1024**2, "g": 1024**3}.get(text[-1].lower(), 1) + digits = text[:-1] if multiplier != 1 else text + return int(digits, 0) * multiplier + + +def mtdparts_partition_offset(mtdparts: str, name: str) -> int | None: + """Offset of partition ``name`` in a U-Boot/Linux mtdparts string. + + Accepts the value with or without the ``mtdparts=`` prefix and with several + ``;``-separated devices. Returns None if the partition is not listed or the + string cannot be parsed up to it. + """ + value = mtdparts.strip().removeprefix("mtdparts=") + for device in value.split(";"): + _, sep, parts = device.partition(":") + if not sep: + continue + offset = 0 + for part in parts.split(","): + match = _MTDPART_RE.match(part.strip()) + if match is None: + break + if match.group("offset"): + offset = _mtd_size(match.group("offset")) + if match.group("name") == name: + return offset + size = match.group("size") + if size == "-": + break + offset += _mtd_size(size) + return None + + +def parse_nand_erase_range(response: str) -> tuple[int, int] | None: + """Return ``(offset, size)`` from U-Boot's ``nand erase`` banner.""" + match = re.search( + r"offset\s+0x([0-9a-f]+),\s*size\s+0x([0-9a-f]+)", + response, + re.IGNORECASE, + ) + if match is None: + return None + return int(match.group(1), 16), int(match.group(2), 16) + + +def uboot_reports_ok(response: str) -> bool: + """True when a U-Boot nand erase/write printed its final ``OK``.""" + return re.search(r"(?:^|[\s:])OK\s*$", response, re.MULTILINE) is not None + def align_up(value: int, alignment: int) -> int: if value < 0: diff --git a/src/defib/install/orchestrator.py b/src/defib/install/orchestrator.py index 9ad2fa0..069835e 100644 --- a/src/defib/install/orchestrator.py +++ b/src/defib/install/orchestrator.py @@ -16,17 +16,22 @@ from defib.install.firmware import load_firmware_bundle from defib.install.layout import ( NAND_LAYOUT, + NAND_UBI_LAYOUT, + NAND_UBI_OFFSET, NOR8M_LAYOUT, align_up, detect_nor_size_mb, erased_region_crc, + mtdparts_partition_offset, nand_bootargs, nor_layout, nor_mtdparts, + parse_nand_erase_range, parse_uboot_crc32, select_nor_size_mb, set_uboot_env_verified, uboot_flash_command_error, + uboot_reports_ok, uboot_sf_lock_unsupported, verify_spi_environment_crc, ) @@ -52,6 +57,7 @@ async def run_install(request: InstallRequest) -> None: get_cached_path, has_firmware, pad_to_size, + uses_per_flash_type_uboot, ) from defib.flashdump import get_ram_staging_addr, send_command from defib.network.ip_manager import list_interfaces_async, temporary_ip @@ -143,7 +149,13 @@ def warn(message: str) -> None: fail("--wipe-rootfs-data is only supported for NOR installs", exit_code=2) stage_set = set(stages) - needs_tftp = bool(stage_set & {"uboot", "kernel", "rootfs"}) + # u-boot-xmedia SoCs on NAND use the UBI-only layout: no kernel partition, + # one UBI image from 0x100000 to the end of the chip, and a U-Boot whose + # default environment already describes all of it. + ubi_layout = nand and uses_per_flash_type_uboot(chip) + uboot_flash_type = "nand" if nand else "nor" + tftp_stages = {"uboot", "rootfs"} if ubi_layout else {"uboot", "kernel", "rootfs"} + needs_tftp = bool(stage_set & tftp_stages) if debug: logging.basicConfig(level=logging.DEBUG) @@ -176,10 +188,17 @@ def warn(message: str) -> None: # The NOR boot and env partitions are fixed across the standard OpenIPC # 8/16/32 MiB layouts. Kernel/rootfs sizing is selected after ``sf probe`` # reports the actual flash capacity. - if nand: + layout: dict[str, tuple[int, int]] | None + if ubi_layout: + layout = None + flash_cmd = "nand" + flash_label = "NAND (UBI layout: 768k boot, 256k env, ubi)" + b_off, b_sz = NAND_UBI_LAYOUT["boot"] + _, env_sz = NAND_UBI_LAYOUT["env"] + elif nand: layout = NAND_LAYOUT flash_cmd = "nand" - flash_label = "NAND" + flash_label = "NAND (legacy split layout)" b_off, b_sz = layout["boot"] _, env_sz = layout["env"] else: @@ -205,7 +224,7 @@ def warn(message: str) -> None: console.print(f" Stages: [cyan]{', '.join(stages)}[/cyan]") try: - firmware = load_firmware_bundle(firmware_path) + firmware = load_firmware_bundle(firmware_path, ubi_only=ubi_layout) except ValueError as exc: fail(str(exc)) @@ -214,7 +233,7 @@ def warn(message: str) -> None: rootfs_name = firmware.rootfs_name rootfs_data = firmware.rootfs - if nand: + if nand and not ubi_layout: assert layout is not None k_off, k_sz = layout["kernel"] r_off, r_sz = layout["rootfs"] @@ -226,8 +245,12 @@ def warn(message: str) -> None: raise typer.Exit(1) if output == "human": - console.print(f" Kernel: [cyan]{kernel_name}[/cyan] ({len(kernel_data)} bytes)") - console.print(f" Rootfs: [cyan]{rootfs_name}[/cyan] ({len(rootfs_data)} bytes)") + if ubi_layout: + console.print(" Kernel: [cyan]inside the UBI image (/boot/fitImage)[/cyan]") + console.print(f" UBI: [cyan]{rootfs_name}[/cyan] ({len(rootfs_data)} bytes)") + else: + console.print(f" Kernel: [cyan]{kernel_name}[/cyan] ({len(kernel_data)} bytes)") + console.print(f" Rootfs: [cyan]{rootfs_name}[/cyan] ({len(rootfs_data)} bytes)") # --- Step 2: Resolve U-Boot artifact --- if uboot_path: @@ -237,13 +260,13 @@ def warn(message: str) -> None: uboot_raw = source_path.read_bytes() uboot_source_name = source_path.name else: - if not has_firmware(chip): + if not has_firmware(chip, uboot_flash_type): fail(f"No OpenIPC U-Boot for '{chip}'") - cached = get_cached_path(chip) + cached = get_cached_path(chip, uboot_flash_type) if cached is None: if output == "human": console.print(f" Downloading U-Boot for [cyan]{chip}[/cyan]...") - cached = download_firmware(chip) + cached = download_firmware(chip, flash_type=uboot_flash_type) uboot_raw = cached.read_bytes() uboot_source_name = cached.name @@ -326,9 +349,11 @@ def warn(message: str) -> None: if not has_stock_uboot: # Boot-ROM recovery needs a filesystem path for RecoverySession. Registered # vendor-U-Boot migration targets use their bootstrap path instead. - cached = get_cached_path(chip) if not uboot_path else Path(uboot_path) + cached = ( + get_cached_path(chip, uboot_flash_type) if not uboot_path else Path(uboot_path) + ) if cached is None: - cached = download_firmware(chip) + cached = download_firmware(chip, flash_type=uboot_flash_type) session = RecoverySession( chip=chip, firmware_path=str(cached), power_controller=power_controller, poe_port=poe_port, @@ -819,9 +844,15 @@ async def _set_env_verified_or_fail(key: str, value: str) -> None: f"erase block 0x{nor_erase_block:X}" ) - assert layout is not None - k_off, k_sz = layout["kernel"] - r_off, r_sz = layout["rootfs"] + if ubi_layout: + # No raw kernel/rootfs partitions; the UBI image runs to the chip end + # and its size is checked against what `nand erase` reports. + k_off, k_sz = 0, 0 + r_off, r_sz = NAND_UBI_OFFSET, 0 + else: + assert layout is not None + k_off, k_sz = layout["kernel"] + r_off, r_sz = layout["rootfs"] # --- Step 5: Pick a TFTP backend, stage / start, then drive U-Boot --- # @@ -854,7 +885,7 @@ async def _set_env_verified_or_fail(key: str, value: str) -> None: tftp_files: dict[str, bytes] = {} if "uboot" in stage_set: tftp_files[tftp_alias["uboot"]] = uboot_data - if "kernel" in stage_set: + if "kernel" in stage_set and not ubi_layout: tftp_files[tftp_alias["kernel"]] = kernel_data if "rootfs" in stage_set: tftp_files[tftp_alias["rootfs"]] = rootfs_data @@ -1239,16 +1270,167 @@ async def tftp_and_flash( await tftp_and_flash( "U-Boot", tftp_alias["uboot"], uboot_data, b_off, uboot_flash_size ) - if "kernel" in stage_set: + if "kernel" in stage_set and ubi_layout: + if output == "human": + console.print( + "\n [bold]kernel[/bold]: no separate partition on the UBI " + "layout; the kernel ships inside the UBI image as " + "/boot/fitImage (written by the rootfs stage). Nothing to do." + ) + elif "kernel" in stage_set: await tftp_and_flash( "kernel", tftp_alias["kernel"], kernel_data, k_off, k_sz ) - # For NAND, raw UBI images must be written through UBI rather than - # ``nand write`` because bad-block skipping would shift UBIFS data. + async def flash_ubi_image() -> None: + """Write the whole UBI image to the ubi partition (UBI layout). + + UBI tolerates bad blocks being skipped (it identifies PEBs by + their EC/VID headers, not position), so the image goes in raw. + ``write.trimffs`` is mandatory: a plain ``nand write`` would + program the image's 0xFF padding pages, and UBIFS programming + them again later breaks their ECC (OpenIPC/firmware#2519). + """ + if output == "human": + console.print( + f"\n [bold]Flashing UBI image[/bold] → 0x{NAND_UBI_OFFSET:X}" + f" ({len(rootfs_data)} bytes)" + ) + ram_crc = await _verify_tftp_ram( + "rootfs (UBI)", tftp_alias["rootfs"], rootfs_data, + ) + if output == "human": + if ram_crc is None: + console.print( + " TFTP transfer accepted; U-Boot CRC32 unavailable" + ) + else: + console.print(f" TFTP CRC verified: {ram_crc:08X}") + + # Erase the whole ubi partition, so chips larger than 128 MiB + # lose every stale block. `nand erase.part ubi` is preferred but + # only when the live mtdparts provably puts ubi at 0x100000; + # otherwise erase from 0x100000 to the end of the chip (U-Boot's + # `nand erase ` with no size runs to the chip end). + mtd_resp = await _optional_printenv("mtdparts", timeout=5.0) + mtd_value = parse_printenv_value(mtd_resp, "mtdparts") + ubi_off = ( + mtdparts_partition_offset(mtd_value, "ubi") + if mtd_value is not None + else None + ) + if ubi_off is not None and ubi_off != NAND_UBI_OFFSET: + raise RuntimeError( + f"U-Boot mtdparts places ubi at 0x{ubi_off:X}, not " + f"0x{NAND_UBI_OFFSET:X} ({mtd_value!r}); refusing to erase. " + "Is this the u-boot--nand.bin build?" + ) + + erase_resp = "" + erased = False + if ubi_off == NAND_UBI_OFFSET: + _, erase_resp = await _cmd_result( + "nand erase.part ubi", timeout=600.0, allow_failure=True, + ) + if "erasing at" in erase_resp.lower(): + if not uboot_reports_ok(erase_resp) or uboot_flash_command_error( + erase_resp + ): + raise RuntimeError( + "nand erase.part ubi failed: " + f"{erase_resp.strip()[-200:]}" + ) + erased = True + else: + warn( + "`nand erase.part ubi` is unavailable in this U-Boot " + f"({erase_resp.strip()[-80:]!r}); erasing from " + f"0x{NAND_UBI_OFFSET:X} to the end of the chip instead." + ) + if not erased: + erase_resp = await _cmd( + f"nand erase 0x{NAND_UBI_OFFSET:x}", timeout=600.0, + ) + erase_error = uboot_flash_command_error(erase_resp) + if erase_error or not uboot_reports_ok(erase_resp): + raise RuntimeError( + f"nand erase 0x{NAND_UBI_OFFSET:x} (to chip end) failed: " + f"{(erase_error or erase_resp.strip())[-200:]}" + ) + + erase_range = parse_nand_erase_range(erase_resp) + if erase_range is not None: + erase_off, erase_size = erase_range + if erase_off != NAND_UBI_OFFSET: + raise RuntimeError( + f"nand erase reported offset 0x{erase_off:X}, expected " + f"0x{NAND_UBI_OFFSET:X}" + ) + if len(rootfs_data) > erase_size: + raise RuntimeError( + f"UBI image is {len(rootfs_data)} bytes, larger than the " + f"0x{erase_size:X}-byte ubi partition" + ) + if output == "human": + size_note = ( + f"0x{erase_range[1]:X} bytes" if erase_range else "to chip end" + ) + console.print( + f" Erased ubi: 0x{NAND_UBI_OFFSET:X}, {size_note}" + ) + + write_resp = await _cmd( + f"nand write.trimffs 0x{ram_addr:x} 0x{NAND_UBI_OFFSET:x} " + f"0x{len(rootfs_data):x}", + timeout=600.0, + ) + write_error = uboot_flash_command_error(write_resp) + if write_error or not uboot_reports_ok(write_resp): + raise RuntimeError( + "nand write.trimffs failed: " + f"{(write_error or write_resp.strip())[-200:]}" + ) + + # Read the image back through ECC and compare CRCs. Blocks + # skipped as bad on write are skipped identically on read, and + # pages trimffs left erased read back as 0xFF. + if crc32_available: + read_resp = await _cmd( + f"nand read 0x{ram_addr:x} 0x{NAND_UBI_OFFSET:x} " + f"0x{len(rootfs_data):x}", + timeout=600.0, + ) + read_error = uboot_flash_command_error(read_resp) + if read_error or not uboot_reports_ok(read_resp): + raise RuntimeError( + "UBI image readback failed: " + f"{(read_error or read_resp.strip())[-200:]}" + ) + crc_resp = await _cmd( + f"crc32 0x{ram_addr:x} 0x{len(rootfs_data):x}", + timeout=_crc_timeout_for_size(len(rootfs_data)), + ) + flash_crc = parse_uboot_crc32(crc_resp) + expected_crc = zlib.crc32(rootfs_data) & 0xFFFFFFFF + if flash_crc != expected_crc: + got = "none" if flash_crc is None else f"{flash_crc:08X}" + raise RuntimeError( + "UBI image flash verify failed: " + f"expected={expected_crc:08X} got={got}" + ) + if output == "human": + console.print(f" Flash verified: {flash_crc:08X}") + if output == "human": + console.print(" [green]UBI image OK[/green]") + + # Legacy split layout: raw UBI images must be written through UBI + # rather than ``nand write`` because bad-block skipping would shift + # UBIFS data. from defib.ubi import extract_ubifs, is_ubi_image - if "rootfs" in stage_set and nand and is_ubi_image(rootfs_data): + if "rootfs" in stage_set and ubi_layout: + await flash_ubi_image() + elif "rootfs" in stage_set and nand and is_ubi_image(rootfs_data): if output == "human": console.print( f"\n [bold]Flashing rootfs (UBI)[/bold] → 0x{r_off:X}" @@ -1303,6 +1485,13 @@ async def tftp_and_flash( "rootfs", tftp_alias["rootfs"], rootfs_data, r_off, r_sz ) + if "rootfs-data" in stage_set and ubi_layout and output == "human": + if "rootfs" in stage_set: + note = "the UBI image just written carries an empty rootfs_data volume" + else: + note = "the existing rootfs_data UBI volume is left untouched" + console.print(f"\n [bold]rootfs-data[/bold]: {note}. Nothing to do.") + erase_rootfs_data = ( not nand and ( @@ -1440,7 +1629,29 @@ async def tftp_and_flash( console.print(" [green]OpenIPC U-Boot defaults loaded[/green]") # Set up the persistent boot environment. - if "env" in stage_set and nand: + ubi_layout_eth: str | None = None + if "env" in stage_set and ubi_layout: + # The NAND U-Boot's default environment owns mtdids, mtdparts, + # bootcmd and bootargs for the UBI layout; never override them. + # A freshly written U-Boot starts from those defaults; only the + # camera's MAC is carried across. + if output == "human": + console.print( + "\n [bold]Setting boot environment[/bold] (NAND UBI layout)" + ) + if "uboot" in stage_set: + pre_default_resp = await _optional_printenv("ethaddr", timeout=5.0) + ubi_layout_eth = parse_printenv_value(pre_default_resp, "ethaddr") + default_resp = await _cmd("env default -a", timeout=5.0) + default_error = uboot_flash_command_error(default_resp) + if default_error: + raise RuntimeError( + f"env default -a failed ({default_error}): " + f"{default_resp.strip()[-200:]}" + ) + if output == "human": + console.print(" U-Boot default environment loaded") + elif "env" in stage_set and nand: if output == "human": console.print("\n [bold]Setting boot environment[/bold] (NAND)") await _cmd( @@ -1476,7 +1687,7 @@ async def tftp_and_flash( # boot-ROM installs keep the existing generic rescue-MAC behavior. eth_resp = await _optional_printenv("ethaddr", timeout=5.0) current_eth = parse_printenv_value(eth_resp, "ethaddr") - preserved_eth = preserved_stock_env.get("ethaddr") + preserved_eth = preserved_stock_env.get("ethaddr") or ubi_layout_eth selected_eth, eth_source = select_install_ethaddr( current_eth, preserved_eth, diff --git a/tests/test_firmware.py b/tests/test_firmware.py index a3971c6..a8e6a86 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -7,6 +7,7 @@ from defib.firmware import ( AVAILABLE_FIRMWARE, CLASSIC_UBOOT_VARIANTS, + PER_FLASH_TYPE_UBOOT, CV6XX_BOOT_VARIANTS, asset_name, download_firmware, @@ -57,9 +58,12 @@ def test_at_least_20_chips(self): assert len(AVAILABLE_FIRMWARE) >= 20 def test_common_chips_included(self): - for chip in ["hi3516ev200", "hi3516ev300", "gk7205v200", "hi3518ev200"]: + for chip in ["gk7205v200", "hi3518ev200"]: assert chip in AVAILABLE_FIRMWARE + def test_per_flash_type_socs_are_not_universal(self): + assert not AVAILABLE_FIRMWARE & PER_FLASH_TYPE_UBOOT + class TestCacheDir: def test_returns_path(self): @@ -181,7 +185,10 @@ def test_empty_cache_file_is_ignored(self, tmp_path, monkeypatch): class TestClassicChipsUnchanged: def test_universal_naming_preserved(self): - assert asset_name("hi3516ev300") == "u-boot-hi3516ev300-universal.bin" + assert asset_name("hi3516cv300") == "u-boot-hi3516cv300-universal.bin" + + def test_flash_type_does_not_rename_universal_images(self): + assert asset_name("hi3516cv300", "nand") == "u-boot-hi3516cv300-universal.bin" def test_variant_still_ignored_for_classic_socs(self): assert asset_name("hi3516ev300:emmc") == asset_name("hi3516ev300") @@ -255,3 +262,100 @@ def fake_urlopen(req, timeout=None): "https://github.com/OpenIPC/u-boot-xmedia/releases/download/latest/" "u-boot-gk7205v510-nor.bin" ] + + +class TestPerFlashTypeUBoot: + """u-boot-xmedia SoCs publish u-boot--{nor,nand}.bin, not -universal.""" + + SOCS = ( + "hi3516ev200", "hi3516ev300", "hi3518ev300", "hi3516dv200", + "gk7205v500", "gk7205v510", "gk7205v530", + ) + + def test_set_matches_the_published_socs(self): + assert set(self.SOCS) == PER_FLASH_TYPE_UBOOT + + @pytest.mark.parametrize("soc", SOCS) + def test_name_per_flash_type(self, soc): + assert asset_name(soc, "nor") == f"u-boot-{soc}-nor.bin" + assert asset_name(soc, "nand") == f"u-boot-{soc}-nand.bin" + assert firmware_url(soc, "nand") == ( + "https://github.com/OpenIPC/firmware/releases/download/latest/" + f"u-boot-{soc}-nand.bin" + ) + + @pytest.mark.parametrize("soc", SOCS) + def test_nor_is_the_default(self, soc): + assert asset_name(soc) == f"u-boot-{soc}-nor.bin" + assert has_firmware(soc) + + def test_flash_type_is_case_insensitive_and_validated(self): + assert asset_name("hi3516ev300", "NAND") == "u-boot-hi3516ev300-nand.bin" + with pytest.raises(ValueError): + asset_name("hi3516ev300", "emmc") + + def test_variant_suffix_is_ignored(self): + assert asset_name("hi3516ev300:emmc", "nand") == "u-boot-hi3516ev300-nand.bin" + + def test_legacy_universal_cache_is_not_reported(self, tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setattr("sys.platform", "linux") + cache = get_cache_dir() + (cache / "u-boot-hi3516ev300-universal.bin").write_bytes(b"U" * 4096) + assert get_cached_path("hi3516ev300") is None + assert get_cached_path("hi3516ev300", "nand") is None + + nand = cache / "u-boot-hi3516ev300-nand.bin" + nand.write_bytes(b"N" * 4096) + assert get_cached_path("hi3516ev300", "nand") == nand + assert get_cached_path("hi3516ev300", "nor") is None + + def test_downloads_the_flash_type_build_never_universal(self, tmp_path, monkeypatch): + from defib import firmware + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setattr("sys.platform", "linux") + urls: list[str] = [] + + def fake_download(url, dest, on_progress=None): + urls.append(url) + dest.write_bytes(b"X" * 4096) + return dest + + monkeypatch.setattr(firmware, "_download", fake_download) + path = download_firmware("hi3516ev300", flash_type="nand") + assert path.name == "u-boot-hi3516ev300-nand.bin" + path = download_firmware("hi3516ev300") + assert path.name == "u-boot-hi3516ev300-nor.bin" + assert all("universal" not in url for url in urls) + assert [u.rsplit("/", 1)[1] for u in urls] == [ + "u-boot-hi3516ev300-nand.bin", + "u-boot-hi3516ev300-nor.bin", + ] + + def test_cached_universal_is_an_offline_nor_fallback_only(self, tmp_path, monkeypatch): + from defib import firmware + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setattr("sys.platform", "linux") + legacy = get_cache_dir() / "u-boot-hi3516ev300-universal.bin" + legacy.write_bytes(b"U" * 4096) + + def offline(url, dest, on_progress=None): + raise ConnectionError("offline") + + monkeypatch.setattr(firmware, "_download", offline) + assert download_firmware("hi3516ev300") == legacy + with pytest.raises(ConnectionError): + download_firmware("hi3516ev300", flash_type="nand") + + def test_gk7205v5xx_shares_the_v500_donor_cache_entry(self, tmp_path, monkeypatch): + # download_v500_donor() caches u-boot-xmedia's NOR build under the + # same name the OpenIPC/firmware release uses, so either source + # satisfies the other. + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setattr("sys.platform", "linux") + donor = get_cache_dir() / "u-boot-gk7205v510-nor.bin" + donor.write_bytes(b"D" * 4096) + assert get_cached_path("gk7205v510") == donor + assert get_cached_path("gk7205v510", "nand") is None diff --git a/tests/test_nand_ubi_install.py b/tests/test_nand_ubi_install.py new file mode 100644 index 0000000..cabb989 --- /dev/null +++ b/tests/test_nand_ubi_install.py @@ -0,0 +1,520 @@ +"""UBI-only NAND install for the u-boot-xmedia SoCs (hi3516ev300 & co.). + +Layout: 0x000000 boot 768K, 0x0C0000 env 256K, 0x100000 ubi to the chip end. +The NAND U-Boot's default environment owns mtdparts/bootcmd/bootargs, so the +installer must only write U-Boot and the UBI image, and carry the MAC across. +""" + +from __future__ import annotations + +import hashlib +import io +import tarfile +import zlib +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest +import typer + +from defib.firmware import pad_to_size +from defib.install import InstallRequest +from defib.install.firmware import load_firmware_bundle +from defib.install.layout import ( + NAND_UBI_LAYOUT, + NAND_UBI_OFFSET, + mtdparts_partition_offset, + parse_nand_erase_range, + uboot_reports_ok, +) +from defib.recovery.events import RecoveryResult +from defib.transport.base import Transport, TransportTimeout + +RAM = 0x42000000 +UBI_IMAGE = b"UBI#" + b"\x01" * 0x1FFFC + b"\xff" * 0x20000 # two 128 KiB PEBs +FACTORY_MAC = "00:12:34:56:78:9a" +XMEDIA_MTDPARTS = "mtdparts=hinand:768k(boot),256k(env),-(ubi)" + + +def _write_tar(path: Path, members: dict[str, bytes]) -> None: + with tarfile.open(path, "w:gz") as archive: + for name, payload in members.items(): + info = tarfile.TarInfo(name) + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + + +def _nand_package(path: Path, board: str = "hi3516ev300") -> Path: + """Shape of openipc.-nand-.tgz: no uImage, no squashfs.""" + fit = b"F" * 4096 + ubifs = b"\x31\x18\x10\x06" + b"\x00" * 4092 + members = { + f"fitImage.{board}": fit, + f"fitImage.{board}.md5sum": f"{hashlib.md5(fit).hexdigest()} fitImage.{board}\n".encode(), + f"rootfs.ubifs.{board}": ubifs, + f"rootfs.ubifs.{board}.md5sum": ( + f"{hashlib.md5(ubifs).hexdigest()} rootfs.ubifs.{board}\n".encode() + ), + f"rootfs.ubi.{board}": UBI_IMAGE, + f"rootfs.ubi.{board}.md5sum": ( + f"{hashlib.md5(UBI_IMAGE).hexdigest()} rootfs.ubi.{board}\n".encode() + ), + } + _write_tar(path, members) + return path + + +class ShellTransport(Transport): + def __init__(self) -> None: + self.rx = bytearray() + self.closed = False + + async def read(self, size: int, timeout: float | None = None) -> bytes: + if not self.rx: + raise TransportTimeout("no data") + data = bytes(self.rx[:size]) + del self.rx[:size] + return data + + async def write(self, data: bytes) -> None: + if b"\x03" in data: + self.rx.extend(b"OpenIPC # ") + + async def flush_input(self) -> None: + self.rx.clear() + + async def flush_output(self) -> None: + return None + + async def bytes_waiting(self) -> int: + return len(self.rx) + + async def close(self) -> None: + self.closed = True + + +class FakeNandUBoot: + """Just enough of a u-boot-xmedia NAND console to drive run_install.""" + + def __init__( + self, + *, + mtdparts: str | None = XMEDIA_MTDPARTS, + erase_part_supported: bool = True, + ) -> None: + self.commands: list[str] = [] + self.tftp_files: dict[str, bytes] = {} + self.ram = b"" + self.flash: dict[int, bytes] = {} + self.env: dict[str, str] = {"ethaddr": FACTORY_MAC} + if mtdparts is not None: + self.env["mtdparts"] = mtdparts + self.erase_part_supported = erase_part_supported + + def reply(self, command: str) -> str: + self.commands.append(command) + prompt = "\nOpenIPC # " + words = command.split() + if command == "nand info": + return "Device 0: nand0, sector size 128 KiB\n Page size 2048 b" + prompt + if words[0] in ("tftpboot", "tftp"): + self.ram = self.tftp_files[words[-1]] + return f"Bytes transferred = {len(self.ram)} (hex)" + prompt + if words[0] == "crc32": + size = int(words[2], 16) + crc = zlib.crc32(self.ram[:size]) & 0xFFFFFFFF + return f"crc32 for {words[1]} ... ==> {crc:08x}" + prompt + if words[0] == "printenv": + key = words[1] + if key in self.env: + return f"{key}={self.env[key]}" + prompt + return f'## Error: "{key}" not defined' + prompt + if words[0] == "setenv": + self.env[words[1]] = " ".join(words[2:]) + return prompt + if command == "env default -a": + self.env = {"mtdparts": XMEDIA_MTDPARTS, "bootcmd": "default"} + return "## Resetting to default environment" + prompt + if command == "nand erase.part ubi": + if not self.erase_part_supported: + return "Usage:\nnand - NAND sub-system" + prompt + return ( + "\nNAND erase.part: device 0 offset 0x100000, size 0x7f00000\n" + "Erasing at 0x7fe0000 -- 100% complete.\nOK" + prompt + ) + if command == "nand erase 0x100000": + return ( + "\nNAND erase: device 0 offset 0x100000, size 0x7f00000\n" + "Erasing at 0x7fe0000 -- 100% complete.\nOK" + prompt + ) + if words[:2] == ["nand", "erase"]: + return "Erasing at 0x0 -- 100% complete.\nOK" + prompt + if words[0] == "nand" and words[1].startswith("write"): + offset, size = int(words[3], 16), int(words[4], 16) + self.flash[offset] = self.ram[:size] + return f" {size} bytes written: OK" + prompt + if words[:2] == ["nand", "read"]: + offset, size = int(words[3], 16), int(words[4], 16) + self.ram = self.flash[offset][:size].ljust(size, b"\xff") + return f" {size} bytes read: OK" + prompt + if command == "saveenv": + return "Saving Environment to NAND...\nWriting to NAND... OK" + prompt + if command == "reset": + return "resetting ..." + return prompt + + +def _patch_install(monkeypatch, device: FakeNandUBoot) -> dict[str, object]: + import defib.flashdump + import defib.network.ip_manager + import defib.network.tftp_server + import defib.recovery.session + import defib.transport.serial_platform + + transport = ShellTransport() + seen: dict[str, object] = {"transport": transport} + + class FakeRecoverySession: + def __init__(self, *args, **kwargs) -> None: + seen["burned"] = kwargs.get("firmware_path") + + async def run(self, transport_obj, **kwargs): + assert transport_obj is transport + return RecoveryResult(success=True) + + async def fake_create_transport(port: str): + return transport + + @asynccontextmanager + async def fake_temporary_ip(interface: str, ip: str, netmask: str): + yield + + class FakeTFTPTransport: + def close(self) -> None: + pass + + class FakeTFTPProtocol: + def __init__(self, files): + self._files = files + + def set_max_blocksize(self, blocksize: int) -> None: + raise AssertionError("no retry expected") + + async def fake_start_tftp_server(*, files, bind_addr, port, done_count): + seen["done_count"] = done_count + device.tftp_files = dict(files) + return FakeTFTPTransport(), FakeTFTPProtocol(device.tftp_files) + + async def fake_send_command(transport_obj, command: str, timeout: float = 0.0, **kw): + assert transport_obj is transport + return device.reply(command) + + monkeypatch.setattr(defib.flashdump, "send_command", fake_send_command) + monkeypatch.setattr(defib.recovery.session, "RecoverySession", FakeRecoverySession) + monkeypatch.setattr( + defib.transport.serial_platform, "create_transport", fake_create_transport + ) + monkeypatch.setattr( + defib.transport.serial_platform, "normalize_port_name", lambda port: port + ) + monkeypatch.setattr(defib.network.ip_manager, "temporary_ip", fake_temporary_ip) + monkeypatch.setattr( + defib.network.tftp_server, "start_tftp_server", fake_start_tftp_server + ) + return seen + + +def _seed_cache(monkeypatch, tmp_path: Path) -> Path: + """Cache holding both builds plus a stale universal image.""" + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr("sys.platform", "linux") + from defib.firmware import get_cache_dir + + cache = get_cache_dir() + (cache / "u-boot-hi3516ev300-nor.bin").write_bytes(b"R" * 0x30000) + (cache / "u-boot-hi3516ev300-nand.bin").write_bytes(b"N" * 0x40000) + (cache / "u-boot-hi3516ev300-universal.bin").write_bytes(b"X" * 0x30000) + return cache + + +# The exact console sequence a full hi3516ev300 NAND install sends (default +# stage plan: uboot, kernel, rootfs, rootfs-data, env, reset). +EXPECTED_HI3516EV300_NAND_SEQUENCE = [ + "nand info", + "setenv ipaddr 192.168.1.20", + "setenv serverip 192.168.1.10", + # uboot: u-boot-hi3516ev300-nand.bin, padded host-side to 0xC0000 + f"tftpboot 0x{RAM:x} u", + f"crc32 0x{RAM:x} 0xc0000", + "nand erase 0x0 0xc0000", + f"nand write 0x{RAM:x} 0x0 0xc0000", + # kernel and rootfs-data: no-ops, the UBI image carries both + # rootfs: the whole rootfs.ubi. + f"tftpboot 0x{RAM:x} r", + f"crc32 0x{RAM:x} 0x40000", + "printenv mtdparts", + "nand erase.part ubi", + f"nand write.trimffs 0x{RAM:x} 0x100000 0x40000", + f"nand read 0x{RAM:x} 0x100000 0x40000", + f"crc32 0x{RAM:x} 0x40000", + # env: start from the new U-Boot's defaults, keep the camera's MAC + "printenv ethaddr", + "env default -a", + "printenv ethaddr", + f"setenv ethaddr {FACTORY_MAC}", + "saveenv", + "reset", +] + + +@pytest.mark.asyncio +async def test_hi3516ev300_nand_install_command_sequence(monkeypatch, tmp_path): + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNandUBoot() + seen = _patch_install(monkeypatch, device) + + await run_install( + InstallRequest( + chip="hi3516ev300", + firmware_path=str(_nand_package(tmp_path / "openipc.hi3516ev300-nand-lite.tgz")), + port="COM15", + nic="eth0", + nand=True, + output="json", + ) + ) + + assert device.commands == EXPECTED_HI3516EV300_NAND_SEQUENCE + joined = "\n".join(device.commands) + for forbidden in ( + "setenv mtdparts", "setenv mtdids", "setenv bootcmd", "setenv bootargs", + "ubi create", "ubi write", "ubi part", + ): + assert forbidden not in joined + + # Burned and installed the NAND build, never the stale universal image. + assert Path(str(seen["burned"])).name == "u-boot-hi3516ev300-nand.bin" + assert seen["done_count"] == 2 + assert set(device.tftp_files) == {"u", "r"} + assert device.tftp_files["u"] == pad_to_size(b"N" * 0x40000, 0xC0000) + assert device.tftp_files["r"] == UBI_IMAGE + assert device.flash[0x100000] == UBI_IMAGE + assert device.env["ethaddr"] == FACTORY_MAC + assert seen["transport"].closed is True # type: ignore[attr-defined] + + +@pytest.mark.asyncio +async def test_nand_install_falls_back_to_erase_to_chip_end(monkeypatch, tmp_path): + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNandUBoot(erase_part_supported=False) + _patch_install(monkeypatch, device) + + await run_install( + InstallRequest( + chip="hi3516ev300", + firmware_path=str(_nand_package(tmp_path / "fw.tgz")), + port="COM15", + nic="eth0", + nand=True, + stages=("rootfs",), + output="json", + ) + ) + + erase_part = device.commands.index("nand erase.part ubi") + fallback = device.commands.index("nand erase 0x100000") + write = device.commands.index(f"nand write.trimffs 0x{RAM:x} 0x100000 0x40000") + assert erase_part < fallback < write + + +@pytest.mark.asyncio +async def test_nand_install_without_mtdparts_erases_by_offset(monkeypatch, tmp_path): + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNandUBoot(mtdparts=None) + _patch_install(monkeypatch, device) + + await run_install( + InstallRequest( + chip="hi3516ev300", + firmware_path=str(_nand_package(tmp_path / "fw.tgz")), + port="COM15", + nic="eth0", + nand=True, + stages=("rootfs",), + output="json", + ) + ) + + assert "nand erase.part ubi" not in device.commands + assert "nand erase 0x100000" in device.commands + + +@pytest.mark.asyncio +async def test_nand_install_refuses_a_foreign_ubi_offset(monkeypatch, tmp_path): + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNandUBoot( + mtdparts="mtdparts=hinand:1024k(boot),1024k(env),8192k(kernel),-(ubi)" + ) + _patch_install(monkeypatch, device) + + with pytest.raises(typer.Exit): + await run_install( + InstallRequest( + chip="hi3516ev300", + firmware_path=str(_nand_package(tmp_path / "fw.tgz")), + port="COM15", + nic="eth0", + nand=True, + stages=("rootfs",), + output="json", + ) + ) + + assert not any(c.startswith("nand erase") for c in device.commands) + assert not any(c.startswith("nand write") for c in device.commands) + + +@pytest.mark.asyncio +async def test_env_only_stage_keeps_the_existing_environment(monkeypatch, tmp_path): + """Without a new U-Boot there is no `env default -a`; the MAC stays.""" + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNandUBoot() + _patch_install(monkeypatch, device) + + await run_install( + InstallRequest( + chip="hi3516ev300", + firmware_path=str(_nand_package(tmp_path / "fw.tgz")), + port="COM15", + nand=True, + stages=("env",), + output="json", + ) + ) + + assert device.commands == ["nand info", "printenv ethaddr", "saveenv"] + + +@pytest.mark.asyncio +async def test_hi3516ev300_nor_install_uses_the_nor_build(monkeypatch, tmp_path): + import defib.flashdump + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNandUBoot() + seen = _patch_install(monkeypatch, device) + nor_firmware = tmp_path / "openipc.hi3516ev300-nor-lite.tgz" + _write_tar( + nor_firmware, + {"uImage.hi3516ev300": b"K" * 1024, "rootfs.squashfs.hi3516ev300": b"R" * 2048}, + ) + + async def nor_send_command(transport_obj, command: str, timeout: float = 0.0, **kw): + if command == "sf probe 0": + device.commands.append(command) + return 'Spi(cs1): Block:64KB Chip:8MB Name:"XT25F64B"\nOpenIPC # ' + if command.startswith("sf read"): + device.commands.append(command) + device.ram = device.tftp_files["u"] + return "SF: done\nOpenIPC # " + if command.startswith("sf "): + device.commands.append(command) + return "SF: done\nOpenIPC # " + return device.reply(command) + + monkeypatch.setattr(defib.flashdump, "send_command", nor_send_command) + + await run_install( + InstallRequest( + chip="hi3516ev300", + firmware_path=str(nor_firmware), + port="COM15", + nic="eth0", + stages=("uboot",), + output="json", + ) + ) + + assert Path(str(seen["burned"])).name == "u-boot-hi3516ev300-nor.bin" + assert device.tftp_files["u"] == pad_to_size(b"R" * 0x30000, 0x40000) + assert "sf erase 0x0 0x40000" in device.commands + + +def test_nand_package_selects_rootfs_ubi_not_ubifs(tmp_path): + bundle = load_firmware_bundle(_nand_package(tmp_path / "fw.tgz"), ubi_only=True) + assert bundle.rootfs_name == "rootfs.ubi.hi3516ev300" + assert bundle.rootfs == UBI_IMAGE + assert bundle.kernel == b"" + + +def test_nand_package_for_gk7205v510_uses_gk7205v500_board(tmp_path): + bundle = load_firmware_bundle( + _nand_package(tmp_path / "fw.tgz", board="gk7205v500"), ubi_only=True + ) + assert bundle.rootfs_name == "rootfs.ubi.gk7205v500" + + +def test_split_layout_package_is_rejected_for_ubi_layout(tmp_path): + old = tmp_path / "old.tgz" + _write_tar(old, {"uImage.hi3516ev300": b"K" * 64, "rootfs.squashfs.hi3516ev300": b"R" * 64}) + with pytest.raises(ValueError, match="rootfs.ubi"): + load_firmware_bundle(old, ubi_only=True) + + +def test_ubi_md5_mismatch_is_rejected(tmp_path): + bad = tmp_path / "bad.tgz" + _write_tar( + bad, + { + "rootfs.ubi.hi3516ev300": UBI_IMAGE, + "rootfs.ubi.hi3516ev300.md5sum": b"0" * 32 + b" rootfs.ubi.hi3516ev300\n", + }, + ) + with pytest.raises(ValueError, match="MD5 mismatch"): + load_firmware_bundle(bad, ubi_only=True) + + +class TestLayoutHelpers: + def test_ubi_layout_constants(self): + assert NAND_UBI_LAYOUT["boot"] == (0x0, 0xC0000) + assert NAND_UBI_LAYOUT["env"] == (0xC0000, 0x40000) + assert NAND_UBI_OFFSET == 0x100000 + + @pytest.mark.parametrize( + "value", + [ + "mtdparts=hinand:768k(boot),256k(env),-(ubi)", + "hinand:768k(boot),256k(env),-(ubi)", + "nand:768k(boot),256k(env),-(ubi)", + "spi:1m(x);nand:0xc0000(boot),0x40000(env),-(ubi)", + ], + ) + def test_ubi_offset_from_mtdparts(self, value): + assert mtdparts_partition_offset(value, "ubi") == 0x100000 + + def test_split_layout_ubi_offset(self): + value = "hinand:1024k(boot),1024k(env),8192k(kernel),-(ubi)" + assert mtdparts_partition_offset(value, "ubi") == 0xA00000 + + def test_missing_partition(self): + assert mtdparts_partition_offset("hinand:768k(boot),-(rest)", "ubi") is None + + def test_erase_range_and_ok(self): + resp = ( + "NAND erase.part: device 0 offset 0x100000, size 0x7f00000\n" + "Erasing at 0x7fe0000 -- 100% complete.\nOK\nOpenIPC # " + ) + assert parse_nand_erase_range(resp) == (0x100000, 0x7F00000) + assert uboot_reports_ok(resp) + assert uboot_reports_ok(" 262144 bytes written: OK\nOpenIPC # ") + assert not uboot_reports_ok("Usage:\nnand - NAND sub-system\nOpenIPC # ") From dc883e2635bed09b1dc72f52ac1498bebfe9ed6c Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:34:25 +0300 Subject: [PATCH 2/3] web: load the u-boot-xmedia -nor build, with the SPL cut at its payload The seven u-boot-xmedia SoCs (hi3516ev200/ev300/dv200, hi3518ev300, gk7205v500/v510/v530) publish their U-Boot per flash type now. The -universal image the Web UI looked for is the retired u-boot-hi3516ev200 build, or absent. A recovery loads U-Boot into RAM and needs no flash layout, so the Web UI takes the -nor build (fwAssetName), as `defib burn` does without --nand. A stale -universal asset of these SoCs is ignored rather than taken in its place. Those images put their compressed payload earlier than the reference SPL the profiles describe (gzip at 0x4400 against FILELEN 0x6000). The Web UI sent the profile's length, and bytes past the payload land in SRAM the bootrom uses for its own state. detectSplSize ports HiSiliconStandard._detect_spl_size: it finds the LZMA or gzip header from 0x4000, rounds down to 1K and caps at SRAMLIMIT where a profile has one. On the real u-boot-hi3516ev300-nand.bin it gives 0x4400, the same as defib's Python. Web tests: 92 pass. --- web/index.html | 5 +++-- web/protocol.js | 53 ++++++++++++++++++++++++++++++++++++++++---- web/protocol.test.js | 46 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/web/index.html b/web/index.html index d3a7ca0..75ab066 100644 --- a/web/index.html +++ b/web/index.html @@ -297,7 +297,8 @@

Serial Console

const ddrAddr = parseInt(profile.ADDRESS[0], 16); const splAddr = parseInt(profile.ADDRESS[1], 16); const ubootAddr = parseInt(profile.ADDRESS[2], 16); - const splMax = parseInt(profile.FILELEN[1], 16); + const splMax = detectSplSize(firmware, parseInt(profile.FILELEN[1], 16), + profile.SRAMLIMIT ? parseInt(profile.SRAMLIMIT, 16) : null); const ddrStep = new Uint8Array(profile.DDRSTEP0); // DDR step @@ -554,7 +555,7 @@

Serial Console

} catch (e) { log(`Could not read release metadata (${e.message}) — downloading without checksum verification`, 'warn'); } - const url = meta ? meta.url : `${FW_DIRECT_BASE}/u-boot-${name}-universal.bin`; + const url = meta ? meta.url : `${FW_DIRECT_BASE}/${fwAssetName(name)}`; if (meta && !meta.sha256) { log('Release metadata has no SHA-256 for this asset — falling back to a size check', 'warn'); } diff --git a/web/protocol.js b/web/protocol.js index e47596d..d0e9fce 100644 --- a/web/protocol.js +++ b/web/protocol.js @@ -190,8 +190,20 @@ const FW_RELEASE_API = const FW_DIRECT_BASE = 'https://github.com/OpenIPC/firmware/releases/download/latest'; -// Asset name → chip, e.g. u-boot-hi3516ev300-universal.bin → hi3516ev300 -const FW_ASSET_RE = /^u-boot-(.+)-universal\.bin$/; +// Asset name → chip, e.g. u-boot-gk7205v300-universal.bin → gk7205v300. +// The u-boot-xmedia SoCs publish per flash type instead (-nor.bin, -nand.bin); +// a recovery loads U-Boot into RAM and needs no flash layout, so it takes the +// -nor one, as `defib burn` does without --nand. +const FW_ASSET_RE = /^u-boot-(.+)-(universal|nor)\.bin$/; +const XMEDIA_SOCS = new Set([ + 'hi3516ev200', 'hi3516ev300', 'hi3518ev300', 'hi3516dv200', + 'gk7205v500', 'gk7205v510', 'gk7205v530', +]); + +/** The published U-Boot a firmware name resolves to. */ +function fwAssetName(name) { + return XMEDIA_SOCS.has(name) ? `u-boot-${name}-nor.bin` : `u-boot-${name}-universal.bin`; +} // Chip aliases: chips whose U-Boot binary is published under another name. const CHIP_FW_ALIAS = { @@ -209,7 +221,7 @@ function fwNameForChip(chip) { /** * Build chip → {name, url, size, sha256} from a GitHub release API response. - * Only u-boot-*-universal.bin assets are considered. `digest` is present on + * Only each chip's own U-Boot asset is considered (fwAssetName). `digest` is present on * modern GitHub responses as "sha256:"; it may be missing on older ones, * in which case sha256 is null and the caller falls back to a size check. */ @@ -217,7 +229,9 @@ function parseReleaseAssets(release) { const out = new Map(); for (const asset of (release && release.assets) || []) { const m = FW_ASSET_RE.exec(asset.name || ''); - if (!m) continue; + // Only the asset this chip is meant to load: a stale -universal image of + // an xmedia SoC must not stand in for its -nor one. + if (!m || asset.name !== fwAssetName(m[1])) continue; out.set(m[1], { name: asset.name, url: `${FW_DIRECT_BASE}/${asset.name}`, @@ -228,6 +242,36 @@ function parseReleaseAssets(release) { return out; } +/** + * The SPL length to upload: the mini-boot's code, up to where its compressed + * U-Boot payload starts. Bytes past that boundary land in SRAM the bootrom + * uses for its own state, and writing them corrupts it, so a build smaller + * than the profile's reference SPL must not be padded to it. Same search as + * HiSiliconStandard._detect_spl_size in defib: an LZMA header (0x5D and a + * power-of-two dictionary of 64K..16M) or a gzip one (1f 8b 08) from 0x4000, + * rounded down to 1K, capped at sramLimit when the profile sets one. Without + * either, the profile's length stands. + */ +function detectSplSize(firmware, profileMax, sramLimit = null) { + const end = Math.min(firmware.length, 0x10000); + for (let i = 0x4000; i < end; i++) { + const b = firmware[i]; + let found = false; + if (b === 0x5d && i + 4 < firmware.length) { + const ds = (firmware[i + 1] | (firmware[i + 2] << 8) | (firmware[i + 3] << 16) + | (firmware[i + 4] << 24)) >>> 0; + found = ds >= 0x10000 && ds <= 0x1000000 && (ds & (ds - 1)) === 0; + } else if (b === 0x1f && firmware[i + 1] === 0x8b && firmware[i + 2] === 0x08) { + found = true; + } + if (found) { + const at = i & ~0x3ff; + return sramLimit !== null && at > sramLimit ? sramLimit : at; + } + } + return profileMax; +} + /** "sha256:abc..." → "abc..." (lowercase hex), else null. */ function parseDigest(digest) { const m = /^sha256:([0-9a-f]{64})$/i.exec(String(digest || '')); @@ -343,6 +387,7 @@ if (typeof module !== 'undefined' && module.exports) { FRAME_BLAST_SOCS, needsFrameBlast, FW_RELEASE_API, FW_DIRECT_BASE, FW_ASSET_RE, FW_PROXIES, CHIP_FW_ALIAS, fwNameForChip, parseReleaseAssets, parseDigest, + XMEDIA_SOCS, fwAssetName, detectSplSize, fwSourceUrls, bytesToHex, verifyFirmwareBytes, PROXY_WINDOW_SECONDS, proxySignatureMessage, hmacSha256Hex, buildOpenIpcProxyUrl, diff --git a/web/protocol.test.js b/web/protocol.test.js index c883727..e2506cb 100644 --- a/web/protocol.test.js +++ b/web/protocol.test.js @@ -594,3 +594,49 @@ describe('fwSourceUrls with the OpenIPC worker', () => { assert.equal(sources.length, 1 + FW_PROXIES.length); }); }); + +describe('detectSplSize', () => { + const { detectSplSize } = require('./protocol.js'); + it('stops the SPL where a gzip payload starts, rounded down to 1K', () => { + const fw = new Uint8Array(0x8000); + fw.set([0x1f, 0x8b, 0x08], 0x4412); + assert.equal(detectSplSize(fw, 0x6000), 0x4400); + }); + it('recognises an LZMA header by its dictionary size', () => { + const fw = new Uint8Array(0x8000); + fw.set([0x5d, 0x00, 0x00, 0x80, 0x00], 0x4800); // 8 MiB dictionary + assert.equal(detectSplSize(fw, 0x6000), 0x4800); + const notLzma = new Uint8Array(0x8000); + notLzma.set([0x5d, 0x01, 0x02, 0x03, 0x04], 0x4800); + assert.equal(detectSplSize(notLzma, 0x6000), 0x6000); + }); + it('keeps the profile length when there is no payload header', () => { + assert.equal(detectSplSize(new Uint8Array(0x8000), 0x6000), 0x6000); + }); + it('caps the boundary at the SRAM limit', () => { + const fw = new Uint8Array(0x8000); + fw.set([0x1f, 0x8b, 0x08], 0x4400); + assert.equal(detectSplSize(fw, 0x6000, 0x3b00), 0x3b00); + }); +}); + +describe('fwAssetName', () => { + const { fwAssetName, parseReleaseAssets } = require('./protocol.js'); + it('names the -nor build for u-boot-xmedia SoCs, -universal otherwise', () => { + assert.equal(fwAssetName('hi3516ev300'), 'u-boot-hi3516ev300-nor.bin'); + assert.equal(fwAssetName('gk7205v510'), 'u-boot-gk7205v510-nor.bin'); + assert.equal(fwAssetName('gk7205v300'), 'u-boot-gk7205v300-universal.bin'); + }); + it('takes each chip\'s own asset and ignores a stale -universal one', () => { + const idx = parseReleaseAssets({ assets: [ + { name: 'u-boot-hi3516ev300-universal.bin', size: 1, digest: null }, + { name: 'u-boot-hi3516ev300-nor.bin', size: 2, digest: null }, + { name: 'u-boot-hi3516ev300-nand.bin', size: 3, digest: null }, + { name: 'u-boot-gk7205v300-universal.bin', size: 4, digest: null }, + { name: 'u-boot-gk7205v300-nor.bin', size: 5, digest: null }, + ] }); + assert.equal(idx.get('hi3516ev300').name, 'u-boot-hi3516ev300-nor.bin'); + assert.equal(idx.get('gk7205v300').name, 'u-boot-gk7205v300-universal.bin'); + assert.equal(idx.size, 2); + }); +}); From dcf35e8f9a0ecb5308adb0adfd6759b5bdceb717 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:42:08 +0300 Subject: [PATCH 3/3] install: match the NAND image to the board, and ubi to the NAND device Review on #147. - load_firmware_bundle checks that rootfs.ubi. is built for the chip being installed, through ubi_nand_boards: the chip itself, plus gk7205v500 for gk7205v510/v530, which share that build. The image carries its kernel, so one built for another board would put that board's system on this camera's NAND. - mtdparts_partition_offset(..., nand_only=True) looks for `ubi` only on NAND devices (an mtd-id with "nand" in it). A `ubi` on a SPI device listed first can no longer pass, or fail, the erase guard for the NAND one. - The Web UI cuts the SPL at its payload only for the u-boot-xmedia images. Its profiles carry no SRAMLIMIT, so on other chips, where a payload can sit past the SRAM window (hi3520dv200, 0x4400 against 0x3B00), it keeps sending the profile length as it always did. --- src/defib/install/firmware.py | 13 ++++++++++++- src/defib/install/layout.py | 22 +++++++++++++++++++--- src/defib/install/orchestrator.py | 9 +++++++-- tests/test_nand_ubi_install.py | 20 +++++++++++++++++++- web/index.html | 12 ++++++++---- 5 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/defib/install/firmware.py b/src/defib/install/firmware.py index 26dceac..b99a2e4 100644 --- a/src/defib/install/firmware.py +++ b/src/defib/install/firmware.py @@ -30,7 +30,9 @@ def _is_ubi_member(name: str) -> bool: return name == "rootfs.ubi" or name.startswith("rootfs.ubi.") -def load_firmware_bundle(path: str | Path, *, ubi_only: bool = False) -> FirmwareBundle: +def load_firmware_bundle( + path: str | Path, *, ubi_only: bool = False, boards: set[str] | None = None +) -> FirmwareBundle: """Read kernel/rootfs and verify any matching md5sum entries in one pass. ``ubi_only`` selects the UBI-only NAND package @@ -82,6 +84,15 @@ def load_firmware_bundle(path: str | Path, *, ubi_only: bool = False) -> Firmwar if not is_ubi_image(rootfs): raise ValueError(f"{rootfs_name} is not a UBI image (no UBI# header)") + # The image carries its kernel, so one built for another board would + # put that board's kernel and rootfs on this camera's NAND. + board = rootfs_name.rsplit("/", 1)[-1].removeprefix("rootfs.ubi.") + if boards is not None and board not in boards: + raise ValueError( + f"{rootfs_name} is built for {board}, not " + + " or ".join(sorted(boards)) + + "; refusing to write another board's NAND image" + ) kernel = b"" elif not kernel or not rootfs: raise ValueError("tarball missing uImage or rootfs (squashfs/ubi)") diff --git a/src/defib/install/layout.py b/src/defib/install/layout.py index ef16af5..3a9bdd4 100644 --- a/src/defib/install/layout.py +++ b/src/defib/install/layout.py @@ -57,6 +57,16 @@ } NAND_UBI_OFFSET = 0x100000 +# The build a SoC's NAND package is published under, where it is not the SoC +# itself: the GK7205V500 family shares one NAND build. +_UBI_NAND_BOARD = {"gk7205v510": "gk7205v500", "gk7205v530": "gk7205v500"} + + +def ubi_nand_boards(chip: str) -> set[str]: + """The ``rootfs.ubi.`` suffixes a NAND package for ``chip`` may carry.""" + base = chip.partition(":")[0] + return {base, _UBI_NAND_BOARD.get(base, base)} + _MTDPART_RE = re.compile( r"^(?P-|(?:0x[0-9a-f]+|\d+)[kmg]?)" r"(?:@(?P(?:0x[0-9a-f]+|\d+)[kmg]?))?" @@ -71,18 +81,24 @@ def _mtd_size(text: str) -> int: return int(digits, 0) * multiplier -def mtdparts_partition_offset(mtdparts: str, name: str) -> int | None: +def mtdparts_partition_offset( + mtdparts: str, name: str, *, nand_only: bool = False +) -> int | None: """Offset of partition ``name`` in a U-Boot/Linux mtdparts string. Accepts the value with or without the ``mtdparts=`` prefix and with several ``;``-separated devices. Returns None if the partition is not listed or the - string cannot be parsed up to it. + string cannot be parsed up to it. ``nand_only`` looks only at NAND devices + (an mtd-id with ``nand`` in it: ``hinand``, ``nand``), so a partition of the + same name on a SPI NOR device cannot stand in for the NAND one. """ value = mtdparts.strip().removeprefix("mtdparts=") for device in value.split(";"): - _, sep, parts = device.partition(":") + mtd_id, sep, parts = device.partition(":") if not sep: continue + if nand_only and "nand" not in mtd_id.strip(): + continue offset = 0 for part in parts.split(","): match = _MTDPART_RE.match(part.strip()) diff --git a/src/defib/install/orchestrator.py b/src/defib/install/orchestrator.py index 069835e..5a7c493 100644 --- a/src/defib/install/orchestrator.py +++ b/src/defib/install/orchestrator.py @@ -28,6 +28,7 @@ nor_mtdparts, parse_nand_erase_range, parse_uboot_crc32, + ubi_nand_boards, select_nor_size_mb, set_uboot_env_verified, uboot_flash_command_error, @@ -224,7 +225,11 @@ def warn(message: str) -> None: console.print(f" Stages: [cyan]{', '.join(stages)}[/cyan]") try: - firmware = load_firmware_bundle(firmware_path, ubi_only=ubi_layout) + firmware = load_firmware_bundle( + firmware_path, + ubi_only=ubi_layout, + boards=ubi_nand_boards(chip) if ubi_layout else None, + ) except ValueError as exc: fail(str(exc)) @@ -1315,7 +1320,7 @@ async def flash_ubi_image() -> None: mtd_resp = await _optional_printenv("mtdparts", timeout=5.0) mtd_value = parse_printenv_value(mtd_resp, "mtdparts") ubi_off = ( - mtdparts_partition_offset(mtd_value, "ubi") + mtdparts_partition_offset(mtd_value, "ubi", nand_only=True) if mtd_value is not None else None ) diff --git a/tests/test_nand_ubi_install.py b/tests/test_nand_ubi_install.py index cabb989..ad8db33 100644 --- a/tests/test_nand_ubi_install.py +++ b/tests/test_nand_ubi_install.py @@ -24,6 +24,7 @@ NAND_UBI_LAYOUT, NAND_UBI_OFFSET, mtdparts_partition_offset, + ubi_nand_boards, parse_nand_erase_range, uboot_reports_ok, ) @@ -459,11 +460,22 @@ def test_nand_package_selects_rootfs_ubi_not_ubifs(tmp_path): def test_nand_package_for_gk7205v510_uses_gk7205v500_board(tmp_path): bundle = load_firmware_bundle( - _nand_package(tmp_path / "fw.tgz", board="gk7205v500"), ubi_only=True + _nand_package(tmp_path / "fw.tgz", board="gk7205v500"), + ubi_only=True, + boards=ubi_nand_boards("gk7205v510"), ) assert bundle.rootfs_name == "rootfs.ubi.gk7205v500" +def test_nand_package_for_another_board_is_rejected(tmp_path): + with pytest.raises(ValueError, match="built for hi3516ev200"): + load_firmware_bundle( + _nand_package(tmp_path / "fw.tgz", board="hi3516ev200"), + ubi_only=True, + boards=ubi_nand_boards("hi3516ev300"), + ) + + def test_split_layout_package_is_rejected_for_ubi_layout(tmp_path): old = tmp_path / "old.tgz" _write_tar(old, {"uImage.hi3516ev300": b"K" * 64, "rootfs.squashfs.hi3516ev300": b"R" * 64}) @@ -506,6 +518,12 @@ def test_split_layout_ubi_offset(self): value = "hinand:1024k(boot),1024k(env),8192k(kernel),-(ubi)" assert mtdparts_partition_offset(value, "ubi") == 0xA00000 + def test_ubi_on_a_spi_device_is_not_the_nand_one(self): + value = "sfc:256k(boot),-(ubi);hinand:768k(boot),256k(env),-(ubi)" + assert mtdparts_partition_offset(value, "ubi") == 0x40000 + assert mtdparts_partition_offset(value, "ubi", nand_only=True) == 0x100000 + assert mtdparts_partition_offset("sfc:256k(boot),-(ubi)", "ubi", nand_only=True) is None + def test_missing_partition(self): assert mtdparts_partition_offset("hinand:768k(boot),-(rest)", "ubi") is None diff --git a/web/index.html b/web/index.html index 75ab066..5e437d3 100644 --- a/web/index.html +++ b/web/index.html @@ -293,12 +293,16 @@

Serial Console

} } -async function standardSendFirmware(firmware, profile) { +async function standardSendFirmware(firmware, profile, chip) { const ddrAddr = parseInt(profile.ADDRESS[0], 16); const splAddr = parseInt(profile.ADDRESS[1], 16); const ubootAddr = parseInt(profile.ADDRESS[2], 16); - const splMax = detectSplSize(firmware, parseInt(profile.FILELEN[1], 16), - profile.SRAMLIMIT ? parseInt(profile.SRAMLIMIT, 16) : null); + // Only the u-boot-xmedia images are cut at their payload: the profiles here + // carry no SRAMLIMIT, and on other chips the profile length is what the + // Web UI has always sent safely. + const profileMax = parseInt(profile.FILELEN[1], 16); + const splMax = XMEDIA_SOCS.has(fwNameForChip(chip)) + ? detectSplSize(firmware, profileMax) : profileMax; const ddrStep = new Uint8Array(profile.DDRSTEP0); // DDR step @@ -788,7 +792,7 @@

Serial Console

if (!profile) { log('No profile for chip: ' + chip, 'error'); showResult(false, 'Unknown chip'); return; } if (!await standardHandshake()) { showResult(false, 'Handshake failed'); return; } completeStage('handshake'); - success = await standardSendFirmware(firmwareData, profile); + success = await standardSendFirmware(firmwareData, profile, chip); } if (success && sendBreak) {