Decode every standard NMEA 2000 PGN, checked against real frames

README.md

@@ -907,6 +907,11 @@ scale snapping, are adapted from CSS Electronics'
 (MIT). The OEM checksum algorithms in `core/checksums.py` follow
 [commaai/opendbc](https://github.com/commaai/opendbc) (MIT).
 
+The NMEA 2000 definitions in `canlab/core/data/n2k_pgns.json` are distilled
+from [CANboat](https://github.com/canboat/canboat) (Apache License 2.0, Kees
+Verruijt) by `tools/build_n2k_table.py`; the licence and the changes made are
+in `canlab/core/data/CANBOAT-NOTICE.txt`.
+
 The SNIFFER tab and its notch, the capture splitter and the GVRET protocol
 follow [SavvyCAN](https://github.com/collin80/SavvyCAN) (MIT), whose sniffer
 window and Bisector are the originals and whose source documents the GVRET

canlab.spec

@@ -20,6 +20,8 @@ a = Analysis(
         (str(ROOT / "canlab.png"),             "canlab"             ),
         (str(ROOT / "assets"),                 "canlab/assets"      ),
         (str(ROOT / "sample_data"),            "canlab/sample_data" ),
+        # the NMEA 2000 table distilled from canboat, and its licence notice
+        (str(ROOT / "core" / "data"),          "canlab/core/data"   ),
         *pil_datas,
     ],
     hiddenimports=[

canlab/core/data/CANBOAT-NOTICE.txt

@@ -0,0 +1,40 @@
+canlab/core/data/n2k_pgns.json is derived from docs/canboat.json in
+CANboat, https://github.com/canboat/canboat
+
+CANboat version v8.2.1
+
+(C) 2009-2026, Kees Verruijt, Harlingen, The Netherlands.
+For more information see https://github.com/canboat/canboat
+
+This file is part of CANboat.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Changes: kept each standard PGN's name, frame type and fixed-position
+fields, and the lookup tables those fields use; dropped proprietary
+PGNs and fields whose position depends on earlier variable-length data.

canlab/core/j1939.py

@@ -69,7 +69,7 @@ def parse_j1939_id(arb_id: int) -> dict:
         da  = ps
 
     if is_nmea2000(pgn):
-        _name, single = _N2K_NAMES.get(pgn, (f"PGN {pgn}", True))
+        single = n2k_single_frame(pgn)
         return {
             "priority": priority,
             "pgn":      pgn,
@@ -100,7 +100,15 @@ def parse_j1939_id(arb_id: int) -> dict:
 def pgn_name(pgn: int) -> str:
     """The name of a PGN in whichever table owns it, or a placeholder."""
     if is_nmea2000(pgn):
-        return _N2K_NAMES.get(pgn, (f"PGN {pgn}", True))[0]
+        if pgn in _N2K_NAMES:
+            return _N2K_NAMES[pgn][0]
+        from canlab.core import n2k_db
+        found = n2k_db.name(pgn)
+        if found:
+            return found
+        if 130816 <= pgn <= 131071 or pgn == 126720:
+            return "Proprietary fast packet (manufacturer defined)"
+        return f"PGN {pgn}"
     entry = PGNS.get(pgn)
     if entry is not None:
         return entry.name
@@ -321,25 +329,51 @@ def _decode_satellites_in_view(data: bytes) -> dict:
 }
 
 
-def decode_n2k(pgn: int, data: bytes) -> dict:
+def n2k_single_frame(pgn: int) -> bool:
+    """Whether an NMEA 2000 PGN fits one frame; unknown ones are assumed to."""
+    if pgn in _N2K_NAMES:
+        return _N2K_NAMES[pgn][1]
+    from canlab.core import n2k_db
+    fast = n2k_db.is_fast_packet(pgn)
+    return True if fast is None else not fast
+
+
+def decode_n2k(pgn: int, data: bytes, reassembled: bool | None = None) -> dict:
     """Decode an NMEA 2000 PGN into {field: (value, unit)}.
 
-    Single-frame PGNs decode from their one frame. Fast-packet PGNs decode
-    only from a reassembled buffer longer than a frame; see _N2K_DECODERS.
+    The hand-written decoders, checked against frames from a real marine
+    recording, come first. Every other standard PGN is decoded from the table
+    distilled from canboat (core/n2k_db.py); on that same recording the two
+    agree on every value they share.
+
+    A fast-packet PGN is decoded only from a reassembled message: its single
+    frames carry a sequence byte and a length byte, so reading one alone
+    shifts every field. ``reassembled`` says which one ``data`` is; left as
+    None, a buffer longer than a frame is taken to be reassembled.
     """
+    data = bytes(data)
+    if reassembled is None:
+        reassembled = len(data) > 8
     decoder = _N2K_DECODERS.get(pgn)
     if decoder is not None:
-        return decoder(bytes(data)) if len(data) > 8 else {}
+        return decoder(data) if reassembled else {}
     fields = _N2K_FIELDS.get(pgn)
-    if not fields:
+    if fields:
+        out = {}
+        for name, (start, length, scale, offset, unit, signed) in fields.items():
+            raw = _n2k_value(data, start, length, signed)
+            if raw is None:
+                continue
+            # Rounded to the field's own resolution. A fixed six places turned
+            # a rate of turn of -0.0001047 rad/s into -0.000105 and cut a
+            # 1e-7 degree position to ten centimetres.
+            from canlab.core.n2k_db import _places
+            out[name] = (round(raw * scale + offset, _places(scale)), unit)
+        return out
+    if not n2k_single_frame(pgn) and not reassembled:
         return {}
-    out = {}
-    for name, (start, length, scale, offset, unit, signed) in fields.items():
-        raw = _n2k_value(data, start, length, signed)
-        if raw is None:
-            continue
-        out[name] = (round(raw * scale + offset, 6), unit)
-    return out
+    from canlab.core import n2k_db
+    return n2k_db.decode(pgn, data)
 
 
 # FMI (Failure Mode Identifier) short names — SAE J1939-73 Appendix A.
@@ -390,7 +424,7 @@ def decode_dm1(data: bytes) -> dict:
     return {"lamps": lamps, "dtcs": dtcs}
 
 
-def decode_pgn(pgn: int, data: bytes) -> dict:
+def decode_pgn(pgn: int, data: bytes, reassembled: bool | None = None) -> dict:
     """
     Decode the parameters of one message. Returns {name: (value, unit)}.
 
@@ -404,7 +438,7 @@ def decode_pgn(pgn: int, data: bytes) -> dict:
     PGNs by decode_n2k.
     """
     if is_nmea2000(pgn):
-        return decode_n2k(pgn, data)
+        return decode_n2k(pgn, data, reassembled)
     if pgn == 0xFECA:
         return decode_dm1(data)
     entry = PGNS.get(pgn)

canlab/core/multiframe.py

@@ -61,8 +61,17 @@
 MAX_FAST_PACKET_BYTES = 223
 
 #: NMEA 2000 PGNs the name table marks as multi-frame.
-FAST_PACKET_PGNS = frozenset(pgn for pgn, (_name, single) in _N2K_NAMES.items()
-                             if not single)
+def _fast_packet_pgns() -> frozenset:
+    """Every NMEA 2000 PGN sent as a fast packet: the hand-written table's,
+    and every other standard one canboat lists, so a message is reassembled
+    whether or not CanLab can decode it."""
+    from canlab.core import n2k_db
+    hand = {pgn for pgn, (_name, single) in _N2K_NAMES.items() if not single}
+    single = {pgn for pgn, (_name, s) in _N2K_NAMES.items() if s}
+    return frozenset(hand | (n2k_db.fast_packet_pgns() - single))
+
+
+FAST_PACKET_PGNS = _fast_packet_pgns()
 
 
 @dataclass
@@ -476,7 +485,7 @@ def summarize(messages) -> list[dict]:
             "bytes": last.size, "frames": last.frames,
             "first_t": items[0].t_start, "last_t": last.t_end,
             "last_data_hex": last.data.hex(" ").upper(),
-            "decoded": decode_pgn(pgn, last.data),
+            "decoded": decode_pgn(pgn, last.data, reassembled=True),
         })
     rows.sort(key=lambda r: (-r["count"], r["pgn"], r["sa"]))
     return rows

canlab/core/n2k_db.py

@@ -0,0 +1,133 @@
+"""Decode any standard NMEA 2000 PGN from the table distilled from canboat.
+
+The table (core/data/n2k_pgns.json, built by tools/build_n2k_table.py from
+canboat, Apache License 2.0) gives each field's position, width, resolution,
+sign, unit and valid range. The range is what makes the decode honest: an
+eight-bit field is valid to 252, and 253, 254 and 255 mean reserved, error
+and not available, as they do throughout NMEA 2000. A value outside its
+range is therefore never shown as a reading.
+
+The hand-written decoders in core/j1939.py are checked against real frames
+and take precedence; this covers the rest. Where both exist, the tests
+require them to agree field by field on the real marine recording.
+
+Temperatures are converted from kelvin to degrees Celsius so they read the
+same as the hand-written ones. Angles stay in radians, as NMEA 2000 sends
+them and as the rest of CanLab reports them.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import json
+from functools import lru_cache
+from pathlib import Path
+
+DATA = Path(__file__).resolve().parent / "data" / "n2k_pgns.json"
+
+
+@lru_cache(maxsize=1)
+def table() -> dict:
+    try:
+        return json.loads(DATA.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {"pgns": {}, "lookups": {}}
+
+
+def definition(pgn: int) -> dict | None:
+    return table()["pgns"].get(str(pgn))
+
+
+def name(pgn: int) -> str | None:
+    d = definition(pgn)
+    return d["name"] if d else None
+
+
+def is_fast_packet(pgn: int) -> bool | None:
+    d = definition(pgn)
+    return d["fast"] if d else None
+
+
+def fast_packet_pgns() -> frozenset:
+    return frozenset(int(p) for p, d in table()["pgns"].items() if d["fast"])
+
+
+def _places(resolution: float) -> int:
+    """Decimal places worth keeping for a field of this resolution.
+
+    A fixed six places cut a 1e-16 degree longitude to six centimetres and
+    padded a 0.1 kelvin temperature with float noise; this keeps two places
+    beyond the resolution, up to what a double can hold.
+    """
+    import math
+    if resolution <= 0:
+        return 6
+    return int(min(12, max(0, math.ceil(-math.log10(resolution)) + 2)))
+
+
+def _raw(data: bytes, bit: int, bits: int) -> int | None:
+    if bit + bits > len(data) * 8:
+        return None
+    return (int.from_bytes(bytes(data), "little") >> bit) & ((1 << bits) - 1)
+
+
+def decode(pgn: int, data: bytes) -> dict:
+    """{field: (value, unit)} for a standard PGN, or {} if it is not in the table.
+
+    Not-available and reserved values are left out; an error code is the
+    string "error".
+    """
+    d = definition(pgn)
+    if d is None:
+        return {}
+    lookups = table()["lookups"]
+    out: dict = {}
+    for (fname, bit, bits, res, signed, unit, kind, lo, hi, offset, lookup) in d["fields"]:
+        if fname in ("SID", "Reserved"):
+            continue                     # a sequence number, not a reading
+        raw = _raw(data, bit, bits)
+        if raw is None:
+            continue
+        if kind == "STRING_FIX":
+            text = bytes(data)[bit // 8:(bit + bits) // 8]
+            text = text.rstrip(b"\xff\x00 @").decode("ascii", "replace")
+            if text:
+                out[fname] = (text, "")
+            continue
+        all_ones = (1 << bits) - 1
+        if signed:
+            top = (1 << (bits - 1)) - 1
+            if raw == top:
+                continue                 # not available
+            if raw == top - 1 and bits >= 4:
+                out[fname] = ("error", "")
+                continue
+            if raw > top:
+                raw -= 1 << bits
+        else:
+            if raw == all_ones and bits >= 2:
+                continue
+            if raw == all_ones - 1 and bits >= 4:
+                out[fname] = ("error", "")
+                continue
+        if kind in ("LOOKUP", "INDIRECT_LOOKUP") and lookup:
+            label = lookups.get(lookup, {}).get(str(raw))
+            if label is not None:
+                out[fname] = (label, "")
+            elif hi is None or raw <= hi:
+                out[fname] = (f"unknown ({raw})", "")
+            continue
+        value = raw * res + (offset or 0)
+        if hi is not None and value > hi * (1 + 1e-9) + 1e-9:
+            continue                     # a reserved code
+        if lo is not None and value < lo * (1 + 1e-9) - 1e-9 and not signed:
+            continue
+        if kind == "DATE":
+            out[fname] = ((_dt.date(1970, 1, 1) + _dt.timedelta(days=int(value))).isoformat(), "")
+            continue
+        if kind in ("MMSI",):
+            out[fname] = (f"{int(raw):09d}", "")
+            continue
+        if unit == "K":
+            value, unit = value - 273.15, "°C"
+        out[fname] = (round(float(value), _places(res)), unit)
+    return out

pyproject.toml

@@ -51,7 +51,7 @@ canlab-cli = "canlab.cli:main"
 include = ["canlab*"]
 
 [tool.setuptools.package-data]
-canlab = ["canlab.png", "assets/*.svg", "sample_data/*.csv", "examples/capture-kit/*"]
+canlab = ["canlab.png", "assets/*.svg", "sample_data/*.csv", "examples/capture-kit/*", "core/data/*"]
 
 [tool.ruff]
 line-length = 120