From c9b77b3651833df2b46a3a385e8a089a029fce01 Mon Sep 17 00:00:00 2001 From: Sherin Joseph Roy Date: Thu, 24 Sep 2026 00:19:25 +0530 Subject: [PATCH] fix: J1939 decoded wrong values from wrong bytes and wrong messages The PGN table produced confident wrong numbers. Checked against SAE J1939-71 and against the 145,534-frame truck log in the corpus: - Coolant temperature was read from byte 6 of EFL/P1, which is half of the crankcase pressure. It lives in ET1, which was not decoded at all. - 0xF005 was labelled ETC1; it is ETC2, and its three fields were at the wrong bytes. EBC2's wheel speeds were filed under 0xFE68 and ERC1 under 0xF006. - VEP battery voltages were read from the current bytes, barometric pressure from cab temperature's low byte, oil level from the blow-by byte. - Cruise control active and enable were whole-byte reads of byte 1, the parking brake and axle switch byte: "Cruise Control Active = 195". - Only all-ones was treated as "not available", so J1939 error codes (0xFE, 0xFExx) and reserved ranges were decoded as readings. - Most source address names did not match the preferred address table: 0x21 was "Brakes" (it is the body controller; brakes are 0x0B). - The scan kept the first sender of each PGN and dropped the rest, so DM1 from nine ECUs was one row with one ECU's count. The table now lives in core/j1939_db.py as bit fields: 26 decoded PGNs with 152 parameters, 30 more named, the preferred address table, two-bit states, error and reserved ranges, park as a gear, ASCII ranges. The scan is a row per PGN and sender. On the truck log the corrected layouts agree with each other: the brakes' front axle speed and the engine's wheel speed differ by 0.33 km/h over 1,957 pairs (r 0.9997); absolute inlet pressure minus boost is the barometer; VD and HRVD give the same 464,671 km; lifetime distance over fuel matches the ECU's reported economy to 0.5%. Those checks are now in the stress run, and 22 unit tests pin the layouts with frames copied from the truck. --- canlab/core/j1939.py | 236 +++++--------- canlab/core/j1939_db.py | 466 +++++++++++++++++++++++++++ tests/real_data/acceptance_stress.py | 92 +++++- tests/test_j1939_spns.py | 231 +++++++++++++ 4 files changed, 857 insertions(+), 168 deletions(-) create mode 100644 canlab/core/j1939_db.py create mode 100644 tests/test_j1939_spns.py diff --git a/canlab/core/j1939.py b/canlab/core/j1939.py index 7106350..9460880 100644 --- a/canlab/core/j1939.py +++ b/canlab/core/j1939.py @@ -21,130 +21,17 @@ For PDU1 (PF < 240): PGN = (DP<<16) | (PF<<8) [destination = PS byte] """ -# ── PGN → (name, SPNs) ──────────────────────────────────────────────────────── -# SPN entry: (name, start_byte, length_bytes, scale, offset, unit) -# start_byte is 0-based within the 8-byte data field - -_PGN_DB: dict[int, dict] = { - # Electronic Engine Controller 1 - 0xF004: { - "name": "EEC1 — Electronic Engine Controller 1", - "spns": { - 190: ("Engine Speed", 3, 2, 0.125, 0, "rpm"), - 512: ("Driver's Demand Engine", 1, 1, 1.0, -125, "%"), - 513: ("Actual Engine Torque", 2, 1, 1.0, -125, "%"), - 899: ("Engine Torque Mode", 0, 1, 1.0, 0, ""), - }, - }, - # Vehicle Speed / Cruise Control - 0xFEF1: { - "name": "CCVS — Cruise Control/Vehicle Speed", - "spns": { - 84: ("Wheel-Based Vehicle Speed", 1, 2, 1/256, 0, "km/h"), - 595: ("Cruise Control Active", 0, 1, 1.0, 0, ""), - 596: ("Cruise Control Enable", 0, 1, 1.0, 0, ""), - }, - }, - # Fuel Economy (Liquid) - 0xFEF2: { - "name": "LFE — Fuel Economy", - "spns": { - 183: ("Fuel Rate", 0, 2, 0.05, 0, "L/h"), - 184: ("Instantaneous Fuel Econ.", 2, 2, 1/512, 0, "km/L"), - 185: ("Average Fuel Economy", 4, 2, 1/512, 0, "km/L"), - }, - }, - # Engine Fluid Level / Pressure 1 - 0xFEEF: { - "name": "EFL/P1 — Engine Fluid Level/Pressure 1", - "spns": { - 94: ("Fuel Delivery Pressure", 0, 1, 4.0, 0, "kPa"), - 22: ("Engine Oil Level", 1, 1, 0.4, 0, "%"), - 100: ("Engine Oil Pressure", 3, 1, 4.0, 0, "kPa"), - 110: ("Engine Coolant Temp", 5, 1, 1.0, -40,"°C"), - }, - }, - # Transport protocol (J1939-21). The frames themselves are envelopes; the - # reassembler in core/multiframe.py turns them back into the PGN they carry. - 0xEC00: {"name": "TP.CM - Transport Protocol Connection Management", "spns": {}}, - 0xEB00: {"name": "TP.DT - Transport Protocol Data Transfer", "spns": {}}, - # Two configuration messages that only ever travel as BAM broadcasts. - 0xFEE3: {"name": "EC1 - Engine Configuration 1", "spns": {}}, - 0xFEE1: {"name": "RC - Retarder Configuration", "spns": {}}, - # Ambient Conditions - 0xFEF5: { - "name": "AMB — Ambient Conditions", - "spns": { - 171: ("Ambient Air Temperature", 3, 2, 0.03125, -273, "°C"), - 108: ("Barometric Pressure", 1, 1, 0.5, 0, "kPa"), - }, - }, - # Vehicle Electrical Power - 0xFEF7: { - "name": "VEP — Vehicle Electrical Power", - "spns": { - 158: ("Key Switch Battery Voltage", 0, 2, 0.05, 0, "V"), - 168: ("Battery Potential", 2, 2, 0.05, 0, "V"), - }, - }, - # Transmission - 0xF005: { - "name": "ETC1 — Electronic Transmission Controller 1", - "spns": { - 522: ("Transmission Selected Gear", 3, 1, 1.0, -125, ""), - 523: ("Transmission Actual Gear", 4, 1, 1.0, -125, ""), - 524: ("Transmission Current Range", 5, 2, 1.0, 0, ""), - }, - }, - # Axle / Drive Wheel Speed - 0xFE68: { - "name": "AWSS — Axle/Drive Wheel Speed", - "spns": { - 904: ("Front Axle Speed", 0, 2, 1/256, 0, "km/h"), - 905: ("Relative Speed FA1", 2, 1, 1/16, 0, "km/h"), - }, - }, - # DM1 — Active DTCs - 0xFECA: { - "name": "DM1 — Active Diagnostic Trouble Codes", - "spns": {}, # complex encoding handled separately - }, - # Engine Hours / Revolutions - 0xFEE5: { - "name": "HOURS — Engine Hours, Revolutions", - "spns": { - 247: ("Total Engine Hours", 0, 4, 0.05, 0, "h"), - 249: ("Total Engine Revs", 4, 4, 1000.0,0, "rev"), - }, - }, - # Retarder - 0xF006: { - "name": "ERC1 — Electronic Retarder Controller 1", - "spns": { - 520: ("Retarder Torque Mode", 0, 1, 1.0, 0, ""), - 521: ("Retarder Actual Retarding Pct",2, 1, 1.0, 0, "%"), - }, - }, -} +# ── the J1939 tables ───────────────────────────────────────────────────────── +# Layouts, names and source addresses live in core/j1939_db.py, where each one +# can be read against the specification and each is pinned by a test. -# Source Address → ECU type -_SA_NAMES: dict[int, str] = { - 0x00: "Engine #1", - 0x01: "Engine #2", - 0x03: "Transmission", - 0x10: "Exhaust/Emission Control", - 0x11: "Exhaust/Emission Ctrl #2", - 0x17: "Fuel System", - 0x21: "Brakes — System Controller", - 0x28: "Instrument Cluster #1", - 0x29: "Trip Recorder", - 0x2A: "Vehicle Management System", - 0x2C: "Cab Display #1", - 0x33: "Body Controller", - 0x3D: "Retarder — Exhaust Engine #1", - 0xF0: "Off-Board Diagnostic Tool", - 0xFF: "Global (broadcast)", -} +from canlab.core.j1939_db import ( # noqa: E402 + ERROR, NOT_AVAILABLE, PGNS, SOURCE_ADDRESSES, decode_spn, proprietary_name, +) + +#: Kept under its old name: other modules and the MCP tools look PGNs up here. +_PGN_DB = PGNS +_SA_NAMES = SOURCE_ADDRESSES # ── Public API ──────────────────────────────────────────────────────────────── @@ -164,8 +51,8 @@ def parse_j1939_id(arb_id: int) -> dict: "pgn": 0xFEF1, "sa": 0x00, "da": 0xFF, # 0xFF = broadcast - "sa_name": "Engine #1", - "pgn_name": "CCVS — Cruise Control/Vehicle Speed", + "sa_name": "Engine #1 (0x00)", + "pgn_name": "CCVS - Cruise Control/Vehicle Speed", } """ priority = (arb_id >> 26) & 0x07 @@ -174,10 +61,10 @@ def parse_j1939_id(arb_id: int) -> dict: ps = (arb_id >> 8) & 0xFF sa = arb_id & 0xFF - if pf >= 0xF0: # PDU2 — PS is group extension + if pf >= 0xF0: # PDU2: PS is group extension pgn = (dp << 16) | (pf << 8) | ps da = 0xFF - else: # PDU1 — PS is destination address + else: # PDU1: PS is destination address pgn = (dp << 16) | (pf << 8) da = ps @@ -200,7 +87,7 @@ def parse_j1939_id(arb_id: int) -> dict: "pgn": pgn, "sa": sa, "da": da, - "sa_name": _SA_NAMES.get(sa, f"SA 0x{sa:02X}"), + "sa_name": sa_name(sa), "pgn_name": pgn_name(pgn), "protocol": "J1939", "single_frame": True, @@ -214,7 +101,16 @@ def pgn_name(pgn: int) -> str: """The name of a PGN in whichever table owns it, or a placeholder.""" if is_nmea2000(pgn): return _N2K_NAMES.get(pgn, (f"PGN {pgn}", True))[0] - return _PGN_DB.get(pgn, {}).get("name", f"PGN 0x{pgn:04X}") + entry = PGNS.get(pgn) + if entry is not None: + return entry.name + return proprietary_name(pgn) or f"PGN 0x{pgn:04X}" + + +def sa_name(sa: int) -> str: + """The J1939 preferred name for a source address, or its number.""" + name = SOURCE_ADDRESSES.get(sa) + return f"{name} (0x{sa:02X})" if name else f"SA 0x{sa:02X}" # ── NMEA 2000 ──────────────────────────────────────────────────────────────── @@ -496,59 +392,77 @@ def decode_dm1(data: bytes) -> dict: def decode_pgn(pgn: int, data: bytes) -> dict: """ - Decode SPN values from `data` (8 bytes) for the given PGN. + Decode the parameters of one message. Returns {name: (value, unit)}. - Returns {spn_name: (value, unit)} or {} if PGN unknown / data too short. - DM1 (0xFECA) is decoded into structured DTCs via decode_dm1. + A measured value is a number. A discrete field (a switch, a mode) is its + label. A parameter the sender reports as an error is the string "error", + so a failed sensor is visible rather than read as a reading. A parameter + the sender marks "not available", or that falls in a reserved range, is + left out, because none of those is a value. + + DM1 (0xFECA) is decoded into structured DTCs via decode_dm1, and NMEA 2000 + PGNs by decode_n2k. """ if is_nmea2000(pgn): return decode_n2k(pgn, data) if pgn == 0xFECA: return decode_dm1(data) - info = _PGN_DB.get(pgn) - if not info or not info.get("spns"): + entry = PGNS.get(pgn) + if entry is None or not entry.spns: return {} - result = {} - for spn_id, (name, start, length, scale, offset, unit) in info["spns"].items(): - end = start + length - if end > len(data): - continue - raw_bytes = data[start:end] - raw_int = int.from_bytes(raw_bytes, "little") - # 0xFF...F = not available indicator - if raw_bytes == b"\xFF" * length: + for spec in entry.spns: + value = decode_spn(spec, data) + if value is NOT_AVAILABLE: continue - value = raw_int * scale + offset - result[name] = (round(value, 4), unit) - + if isinstance(value, (int, float)): + value = round(float(value), 4) + result[spec.name] = (value, spec.unit if value != ERROR else "") return result +def decode_spns(pgn: int, data: bytes) -> list[dict]: + """The same decode with the SPN numbers and the raw status kept, for + tools that need to cite the parameter rather than show it.""" + entry = PGNS.get(pgn) + if entry is None or is_nmea2000(pgn): + return [] + out = [] + for spec in entry.spns: + value = decode_spn(spec, data) + status = ("not available" if value is NOT_AVAILABLE + else "error" if value == ERROR else "ok") + out.append({"spn": spec.spn, "name": spec.name, "unit": spec.unit, + "value": None if status != "ok" else + (round(float(value), 4) if isinstance(value, (int, float)) else value), + "status": status}) + return out + + def scan_for_j1939(df) -> list[dict]: """ - Scan a frames DataFrame for J1939 messages (extended 29-bit IDs inferred from - ID values > 0x7FF when stored as hex strings). + One row per parameter group and sender in a frames DataFrame. + + A PGN sent by several ECUs (CCVS from the engine and the body controller, + DM1 from nine modules on the truck log) is a row per sender, each with its + own frame count. Keying by PGN alone kept the first sender and dropped the + rest, so a scan understated both who was talking and how much. - Returns list of: - {"id_hex", "priority", "pgn", "pgn_name", "sa", "sa_name", "frame_count"} - sorted by pgn. + Returns dicts with "id_hex", "priority", "pgn", "pgn_name", "sa", + "sa_name", "protocol", "single_frame", "frame_count". """ + if df is None or df.empty or "ID" not in df.columns: + return [] + counts = df["ID"].value_counts() results = [] - seen = set() - for can_id in df["ID"].unique(): + for can_id, count in counts.items(): try: arb_id = int(can_id, 16) except (ValueError, TypeError): continue if arb_id <= 0x7FF: - continue # standard 11-bit ID — not J1939 + continue # standard 11-bit ID, not J1939 parsed = parse_j1939_id(arb_id) - key = parsed["pgn"] - if key in seen: - continue - seen.add(key) - count = int((df["ID"] == can_id).sum()) results.append({ "id_hex": can_id, "priority": parsed["priority"], @@ -558,7 +472,7 @@ def scan_for_j1939(df) -> list[dict]: "sa_name": parsed["sa_name"], "protocol": parsed["protocol"], "single_frame": parsed["single_frame"], - "frame_count": count, + "frame_count": int(count), }) # Named messages first, then by how much of the bus they are. Sorting by @@ -566,5 +480,5 @@ def scan_for_j1939(df) -> list[dict]: # capture, so the visible rows all read "no idea" while the twenty named # ones sat below the fold. results.sort(key=lambda x: (x["pgn_name"].startswith("PGN "), - -x["frame_count"])) + -x["frame_count"], x["pgn"], x["sa"])) return results diff --git a/canlab/core/j1939_db.py b/canlab/core/j1939_db.py new file mode 100644 index 0000000..24905a3 --- /dev/null +++ b/canlab/core/j1939_db.py @@ -0,0 +1,466 @@ +"""J1939 parameter groups, their parameters, and the preferred source addresses. + +Layouts follow SAE J1939-71 as it is published: byte and bit positions are +counted from the first data byte and its least significant bit, and multi-byte +values are little-endian. A parameter is written down here only when its +position, resolution and offset are known; a PGN whose layout is not known is +listed by name alone, so the scan can say what a message is without inventing +what it contains. + +Every decoded message in the 145,534-frame truck log in the acceptance corpus +is checked for plausibility and against the others it should agree with: the +front axle speed in EBC2 against the wheel-based speed in CCVS, the coolant +temperature of a running diesel, a battery voltage, barometric pressure near +sea level, distances and hours that only grow. + +The table this replaces had coolant temperature in the wrong message, ETC1 and +the wheel speeds and ERC1 filed under other PGNs, the battery voltages read +from the current bytes, and every switch decoded as a whole byte. Those are +the reasons for the tests that now pin each layout. +""" +from __future__ import annotations + +from typing import NamedTuple + + +class Spn(NamedTuple): + """One suspect parameter number, as a bit field in a parameter group.""" + + spn: int + name: str + bit: int # first bit: byte index * 8 + bit within byte + bits: int + scale: float = 1.0 + offset: float = 0.0 + unit: str = "" + states: dict | None = None # discrete fields: raw value -> label + ascii: bool = False + special: dict | None = None # measured values with named codes (gear 251 = park) + + +class Pgn(NamedTuple): + acronym: str + title: str + spns: tuple = () + + @property + def name(self) -> str: + return f"{self.acronym} - {self.title}" if self.acronym else self.title + + +# ── builders, so each line reads like the specification ────────────────────── + +def _v(spn, name, byte, nbytes, scale=1.0, offset=0.0, unit="", special=None): + """A measured value occupying whole bytes, starting at data byte `byte` (1-based).""" + return Spn(spn, name, (byte - 1) * 8, nbytes * 8, scale, offset, unit, special=special) + + +def _s(spn, name, byte, bit, states=None, bits=2): + """A discrete field at data byte `byte` (1-based), starting at bit `bit` (1-based).""" + return Spn(spn, name, (byte - 1) * 8 + (bit - 1), bits, states=states or SWITCH) + + +def _a(spn, name, byte, nbytes): + return Spn(spn, name, (byte - 1) * 8, nbytes * 8, ascii=True) + + +#: The two-bit state every J1939 switch uses. 10 is an error and 11 is "not +#: available"; the decoder handles those for every discrete field. +SWITCH = {0: "off", 1: "on"} +ACTIVE = {0: "not active", 1: "active"} + +TORQUE_MODE = { + 0: "low idle governor / no request", 1: "accelerator pedal / operator selection", + 2: "cruise control", 3: "PTO governor", 4: "road speed governor", + 5: "ASR control", 6: "transmission control", 7: "ABS control", + 8: "torque limiting", 9: "high speed governor", 10: "braking system", + 11: "remote accelerator", +} +OVERRIDE_MODE = {0: "override disabled", 1: "speed control", 2: "torque control", + 3: "speed/torque limit control"} +CRUISE_STATES = {0: "off/disabled", 1: "hold", 2: "accelerate", 3: "decelerate", + 4: "resume", 5: "set", 6: "accelerator override"} + +#: A transmission gear: negative is reverse, 0 is neutral, 251 is park. +GEAR_CODES = {251: "park"} + +PCT = "%" +RPM = "rpm" +KPA = "kPa" +DEGC = "°C" +KMH = "km/h" + + +PGNS: dict[int, Pgn] = { + 0x0000: Pgn("TSC1", "Torque/Speed Control 1", ( + _s(695, "Engine Override Control Mode", 1, 1, OVERRIDE_MODE), + _s(696, "Engine Requested Speed Control Conditions", 1, 3, + {0: "transient optimized, driveline disengaged", + 1: "stability optimized, driveline disengaged", + 2: "stability optimized, driveline engaged, condition 1", + 3: "stability optimized, driveline engaged, condition 2"}), + _s(897, "Override Control Mode Priority", 1, 5, + {0: "highest", 1: "high", 2: "medium", 3: "low"}), + _v(898, "Engine Requested Speed/Speed Limit", 2, 2, 0.125, 0, RPM), + _v(518, "Engine Requested Torque/Torque Limit", 4, 1, 1, -125, PCT), + )), + 0xF000: Pgn("ERC1", "Electronic Retarder Controller 1", ( + _s(900, "Retarder Torque Mode", 1, 1, TORQUE_MODE, bits=4), + _s(571, "Retarder Enable - Brake Assist Switch", 1, 5), + _s(572, "Retarder Enable - Shift Assist Switch", 1, 7), + _v(520, "Actual Retarder - Percent Torque", 2, 1, 1, -125, PCT), + _v(1085, "Intended Retarder Percent Torque", 3, 1, 1, -125, PCT), + _v(1480, "Source Address of Controlling Device for Retarder Control", 5, 1), + _v(1715, "Drivers Demand Retarder - Percent Torque", 6, 1, 1, -125, PCT), + _v(1716, "Retarder Selection, Non-engine", 7, 1, 0.4, 0, PCT), + _v(1717, "Actual Maximum Available Retarder - Percent Torque", 8, 1, 1, -125, PCT), + )), + 0xF001: Pgn("EBC1", "Electronic Brake Controller 1", ( + _s(561, "ASR Engine Control Active", 1, 1, ACTIVE), + _s(562, "ASR Brake Control Active", 1, 3, ACTIVE), + _s(563, "Anti-Lock Braking (ABS) Active", 1, 5, ACTIVE), + _s(1121, "EBS Brake Switch", 1, 7), + _v(521, "Brake Pedal Position", 2, 1, 0.4, 0, PCT), + _s(575, "ABS Off-road Switch", 3, 1), + _s(576, "ASR Off-road Switch", 3, 3), + _s(577, "ASR Hill Holder Switch", 3, 5), + _s(1238, "Traction Control Override Switch", 3, 7), + _s(972, "Accelerator Interlock Switch", 4, 1), + _s(971, "Engine Derate Switch", 4, 3), + _s(970, "Engine Auxiliary Shutdown Switch", 4, 5), + _s(969, "Remote Accelerator Enable Switch", 4, 7), + _v(973, "Engine Retarder Selection", 5, 1, 0.4, 0, PCT), + _s(1243, "ABS Fully Operational", 6, 1, {0: "not fully operational", 1: "fully operational"}), + _s(1439, "EBS Red Warning Signal", 6, 3), + _s(1438, "ABS/EBS Amber Warning Signal", 6, 5), + _s(1793, "ATC/ASR Information Signal", 6, 7), + _v(1481, "Source Address of Controlling Device for Brake Control", 7, 1), + )), + 0xF002: Pgn("ETC1", "Electronic Transmission Controller 1", ( + _s(560, "Transmission Driveline Engaged", 1, 1, {0: "disengaged", 1: "engaged"}), + _s(573, "Transmission Torque Converter Lockup Engaged", 1, 3, + {0: "disengaged", 1: "engaged"}), + _s(574, "Transmission Shift In Process", 1, 5, + {0: "not in process", 1: "in process"}), + _v(191, "Transmission Output Shaft Speed", 2, 2, 0.125, 0, RPM), + _v(522, "Percent Clutch Slip", 4, 1, 0.4, 0, PCT), + _s(606, "Engine Momentary Overspeed Enable", 5, 1, + {0: "disabled", 1: "enabled"}), + _s(607, "Progressive Shift Disable", 5, 3, {0: "not disabled", 1: "disabled"}), + _v(161, "Transmission Input Shaft Speed", 6, 2, 0.125, 0, RPM), + _v(1482, "Source Address of Controlling Device for Transmission Control", 8, 1), + )), + 0xF003: Pgn("EEC2", "Electronic Engine Controller 2", ( + _s(558, "Accelerator Pedal 1 Low Idle Switch", 1, 1, + {0: "not in low idle", 1: "in low idle"}), + _s(559, "Accelerator Pedal Kickdown Switch", 1, 3, + {0: "kickdown passive", 1: "kickdown active"}), + _v(91, "Accelerator Pedal Position 1", 2, 1, 0.4, 0, PCT), + _v(92, "Engine Percent Load At Current Speed", 3, 1, 1, 0, PCT), + _v(974, "Remote Accelerator Pedal Position", 4, 1, 0.4, 0, PCT), + _v(29, "Accelerator Pedal Position 2", 5, 1, 0.4, 0, PCT), + )), + 0xF004: Pgn("EEC1", "Electronic Engine Controller 1", ( + _s(899, "Engine Torque Mode", 1, 1, TORQUE_MODE, bits=4), + _v(512, "Driver's Demand Engine - Percent Torque", 2, 1, 1, -125, PCT), + _v(513, "Actual Engine - Percent Torque", 3, 1, 1, -125, PCT), + _v(190, "Engine Speed", 4, 2, 0.125, 0, RPM), + _v(1483, "Source Address of Controlling Device for Engine Control", 6, 1), + _v(2432, "Engine Demand - Percent Torque", 8, 1, 1, -125, PCT), + )), + 0xF005: Pgn("ETC2", "Electronic Transmission Controller 2", ( + _v(524, "Transmission Selected Gear", 1, 1, 1, -125, special=GEAR_CODES), + _v(526, "Transmission Actual Gear Ratio", 2, 2, 0.001), + _v(523, "Transmission Current Gear", 4, 1, 1, -125, special=GEAR_CODES), + _a(162, "Transmission Requested Range", 5, 2), + _a(163, "Transmission Current Range", 7, 2), + )), + 0xFEBF: Pgn("EBC2", "Wheel Speed Information", ( + _v(904, "Front Axle Speed", 1, 2, 1 / 256, 0, KMH), + _v(905, "Relative Speed; Front Axle, Left Wheel", 3, 1, 1 / 16, -7.8125, KMH), + _v(906, "Relative Speed; Front Axle, Right Wheel", 4, 1, 1 / 16, -7.8125, KMH), + _v(907, "Relative Speed; Rear Axle #1, Left Wheel", 5, 1, 1 / 16, -7.8125, KMH), + _v(908, "Relative Speed; Rear Axle #1, Right Wheel", 6, 1, 1 / 16, -7.8125, KMH), + _v(909, "Relative Speed; Rear Axle #2, Left Wheel", 7, 1, 1 / 16, -7.8125, KMH), + _v(910, "Relative Speed; Rear Axle #2, Right Wheel", 8, 1, 1 / 16, -7.8125, KMH), + )), + 0xFEC1: Pgn("HRVD", "High Resolution Vehicle Distance", ( + _v(917, "High Resolution Total Vehicle Distance", 1, 4, 0.005, 0, "km"), + _v(918, "High Resolution Trip Distance", 5, 4, 0.005, 0, "km"), + )), + 0xFEDF: Pgn("EEC3", "Electronic Engine Controller 3", ( + _v(514, "Nominal Friction - Percent Torque", 1, 1, 1, -125, PCT), + _v(515, "Engine's Desired Operating Speed", 2, 2, 0.125, 0, RPM), + _v(519, "Engine's Desired Operating Speed Asymmetry Adjustment", 4, 1), + )), + 0xFEE0: Pgn("VD", "Vehicle Distance", ( + _v(244, "Trip Distance", 1, 4, 0.125, 0, "km"), + _v(245, "Total Vehicle Distance", 5, 4, 0.125, 0, "km"), + )), + 0xFEE5: Pgn("HOURS", "Engine Hours, Revolutions", ( + _v(247, "Engine Total Hours of Operation", 1, 4, 0.05, 0, "h"), + _v(249, "Engine Total Revolutions", 5, 4, 1000, 0, "r"), + )), + 0xFEE6: Pgn("TD", "Time/Date", ( + _v(959, "Seconds", 1, 1, 0.25, 0, "s"), + _v(960, "Minutes", 2, 1, 1, 0, "min"), + _v(961, "Hours", 3, 1, 1, 0, "h"), + _v(963, "Month", 4, 1, 1, 0, ""), + _v(962, "Day", 5, 1, 0.25, 0, "d"), + _v(964, "Year", 6, 1, 1, 1985, ""), + _v(1601, "Local Minute Offset", 7, 1, 1, -125, "min"), + _v(1602, "Local Hour Offset", 8, 1, 1, -125, "h"), + )), + 0xFEE7: Pgn("VH", "Vehicle Hours", ( + _v(246, "Total Vehicle Hours", 1, 4, 0.05, 0, "h"), + _v(248, "Total Power Takeoff Hours", 5, 4, 0.05, 0, "h"), + )), + 0xFEE9: Pgn("LFC", "Fuel Consumption (Liquid)", ( + _v(182, "Engine Trip Fuel", 1, 4, 0.5, 0, "L"), + _v(250, "Engine Total Fuel Used", 5, 4, 0.5, 0, "L"), + )), + 0xFEEE: Pgn("ET1", "Engine Temperature 1", ( + _v(110, "Engine Coolant Temperature", 1, 1, 1, -40, DEGC), + _v(174, "Engine Fuel Temperature 1", 2, 1, 1, -40, DEGC), + _v(175, "Engine Oil Temperature 1", 3, 2, 0.03125, -273, DEGC), + _v(176, "Engine Turbocharger Oil Temperature", 5, 2, 0.03125, -273, DEGC), + _v(52, "Engine Intercooler Temperature", 7, 1, 1, -40, DEGC), + _v(1134, "Engine Intercooler Thermostat Opening", 8, 1, 0.4, 0, PCT), + )), + 0xFEEF: Pgn("EFL/P1", "Engine Fluid Level/Pressure 1", ( + _v(94, "Engine Fuel Delivery Pressure", 1, 1, 4, 0, KPA), + _v(22, "Engine Extended Crankcase Blow-by Pressure", 2, 1, 0.05, 0, KPA), + _v(98, "Engine Oil Level", 3, 1, 0.4, 0, PCT), + _v(100, "Engine Oil Pressure", 4, 1, 4, 0, KPA), + _v(101, "Engine Crankcase Pressure", 5, 2, 1 / 128, -250, KPA), + _v(109, "Engine Coolant Pressure", 7, 1, 2, 0, KPA), + _v(111, "Engine Coolant Level", 8, 1, 0.4, 0, PCT), + )), + 0xFEF1: Pgn("CCVS", "Cruise Control/Vehicle Speed", ( + _s(69, "Two Speed Axle Switch", 1, 1, {0: "low speed range", 1: "high speed range"}), + _s(70, "Parking Brake Switch", 1, 3, {0: "not set", 1: "set"}), + _s(1633, "Cruise Control Pause Switch", 1, 5), + _v(84, "Wheel-Based Vehicle Speed", 2, 2, 1 / 256, 0, KMH), + _s(595, "Cruise Control Active", 4, 1, {0: "off", 1: "active"}), + _s(596, "Cruise Control Enable Switch", 4, 3, {0: "disabled", 1: "enabled"}), + _s(597, "Brake Switch", 4, 5, {0: "released", 1: "depressed"}), + _s(598, "Clutch Switch", 4, 7, {0: "released", 1: "depressed"}), + _s(599, "Cruise Control Set Switch", 5, 1), + _s(600, "Cruise Control Coast (Decelerate) Switch", 5, 3), + _s(601, "Cruise Control Resume Switch", 5, 5), + _s(602, "Cruise Control Accelerate Switch", 5, 7), + _v(86, "Cruise Control Set Speed", 6, 1, 1, 0, KMH), + _s(527, "Cruise Control States", 7, 6, CRUISE_STATES, bits=3), + _s(968, "Engine Idle Increment Switch", 8, 1), + _s(967, "Engine Idle Decrement Switch", 8, 3), + _s(966, "Engine Test Mode Switch", 8, 5), + _s(1237, "Engine Shutdown Override Switch", 8, 7), + )), + 0xFEF2: Pgn("LFE", "Fuel Economy (Liquid)", ( + _v(183, "Engine Fuel Rate", 1, 2, 0.05, 0, "L/h"), + _v(184, "Engine Instantaneous Fuel Economy", 3, 2, 1 / 512, 0, "km/L"), + _v(185, "Engine Average Fuel Economy", 5, 2, 1 / 512, 0, "km/L"), + _v(51, "Engine Throttle Valve 1 Position", 7, 1, 0.4, 0, PCT), + _v(3673, "Engine Throttle Valve 2 Position", 8, 1, 0.4, 0, PCT), + )), + 0xFEF5: Pgn("AMB", "Ambient Conditions", ( + _v(108, "Barometric Pressure", 1, 1, 0.5, 0, KPA), + _v(170, "Cab Interior Temperature", 2, 2, 0.03125, -273, DEGC), + _v(171, "Ambient Air Temperature", 4, 2, 0.03125, -273, DEGC), + _v(172, "Engine Air Inlet Temperature", 6, 1, 1, -40, DEGC), + _v(79, "Road Surface Temperature", 7, 2, 0.03125, -273, DEGC), + )), + 0xFEF6: Pgn("IC1", "Inlet/Exhaust Conditions 1", ( + _v(81, "Engine Diesel Particulate Filter Inlet Pressure", 1, 1, 0.5, 0, KPA), + _v(102, "Engine Intake Manifold #1 Pressure", 2, 1, 2, 0, KPA), + _v(105, "Engine Intake Manifold 1 Temperature", 3, 1, 1, -40, DEGC), + _v(106, "Engine Air Inlet Pressure", 4, 1, 2, 0, KPA), + _v(107, "Engine Air Filter 1 Differential Pressure", 5, 1, 0.05, 0, KPA), + _v(173, "Engine Exhaust Gas Temperature", 6, 2, 0.03125, -273, DEGC), + _v(112, "Engine Coolant Filter Differential Pressure", 8, 1, 0.5, 0, KPA), + )), + 0xFEF7: Pgn("VEP1", "Vehicle Electrical Power 1", ( + _v(114, "Net Battery Current", 1, 1, 1, -125, "A"), + _v(115, "Alternator Current", 2, 1, 1, 0, "A"), + _v(167, "Charging System Potential (Voltage)", 3, 2, 0.05, 0, "V"), + _v(168, "Battery Potential / Power Input 1", 5, 2, 0.05, 0, "V"), + _v(158, "Keyswitch Battery Potential", 7, 2, 0.05, 0, "V"), + )), + 0xFEFC: Pgn("DD", "Dash Display", ( + _v(80, "Washer Fluid Level", 1, 1, 0.4, 0, PCT), + _v(96, "Fuel Level 1", 2, 1, 0.4, 0, PCT), + _v(95, "Engine Fuel Filter Differential Pressure", 3, 1, 2, 0, KPA), + _v(99, "Engine Oil Filter Differential Pressure", 4, 1, 0.5, 0, KPA), + _v(169, "Cargo Ambient Temperature", 5, 2, 0.03125, -273, DEGC), + _v(38, "Fuel Level 2", 7, 1, 0.4, 0, PCT), + )), + 0xFEF3: Pgn("VP1", "Vehicle Position 1", ( + _v(584, "Latitude", 1, 4, 1e-7, -210, "deg"), + _v(585, "Longitude", 5, 4, 1e-7, -210, "deg"), + )), + 0xFEF0: Pgn("PTO", "Power Takeoff Information", ( + _v(90, "Power Takeoff Oil Temperature", 1, 1, 1, -40, DEGC), + _v(186, "Power Takeoff Speed", 2, 2, 0.125, 0, RPM), + _v(187, "Power Takeoff Set Speed", 4, 2, 0.125, 0, RPM), + )), + 0xFE6C: Pgn("TCO1", "Tachograph", ( + _v(1624, "Tachograph Vehicle Speed", 7, 2, 1 / 256, 0, KMH), + )), + + # Named, not decoded: their layouts are either variable (identification + # strings, diagnostic lists), carried by the transport protocol, or not + # documented well enough here to decode without guessing. + 0xE000: Pgn("CM1", "Cab Message 1"), + 0xE800: Pgn("ACKM", "Acknowledgment"), + 0xEA00: Pgn("RQST", "Request"), + 0xEB00: Pgn("TP.DT", "Transport Protocol Data Transfer"), + 0xEC00: Pgn("TP.CM", "Transport Protocol Connection Management"), + 0xEE00: Pgn("AC", "Address Claimed"), + 0xEF00: Pgn("PropA", "Proprietary A"), + 0xF00A: Pgn("EGF1", "Engine Gas Flow Rate"), + 0xF00E: Pgn("AT1IG1", "Aftertreatment 1 Intake Gas 1"), + 0xF00F: Pgn("AT1OG1", "Aftertreatment 1 Outlet Gas 1"), + 0xFE4F: Pgn("VDC1", "Vehicle Dynamic Stability Control 1"), + 0xFEBD: Pgn("FD", "Fan Drive"), + 0xFEC3: Pgn("ETC5", "Electronic Transmission Controller 5"), + 0xFECA: Pgn("DM1", "Active Diagnostic Trouble Codes"), + 0xFECB: Pgn("DM2", "Previously Active Diagnostic Trouble Codes"), + 0xFECC: Pgn("DM3", "Diagnostic Data Clear/Reset of Previously Active DTCs"), + 0xFEDA: Pgn("SOFT", "Software Identification"), + 0xFEDB: Pgn("EFL/P2", "Engine Fluid Level/Pressure 2"), + 0xFEDC: Pgn("IO", "Idle Operation"), + 0xFEDD: Pgn("TC", "Turbocharger"), + 0xFEE1: Pgn("RC", "Retarder Configuration"), + 0xFEE3: Pgn("EC1", "Engine Configuration 1"), + 0xFEE4: Pgn("SHUTDN", "Shutdown"), + 0xFEEB: Pgn("CI", "Component Identification"), + 0xFEEC: Pgn("VI", "Vehicle Identification"), + 0xFEFA: Pgn("B", "Brakes"), + 0xFEFF: Pgn("WFI", "Water in Fuel Indicator"), + 0xFE92: Pgn("EI", "Engine Information"), + 0xFE69: Pgn("ET3", "Engine Temperature 3"), + 0xFD7C: Pgn("DPFC1", "Diesel Particulate Filter Control 1"), +} + + +def proprietary_name(pgn: int) -> str | None: + """Proprietary B covers a whole range; its contents are the manufacturer's.""" + if 0xFF00 <= pgn <= 0xFFFF: + return "PropB - Proprietary B (manufacturer defined)" + if pgn == 0x1EF00: + return "PropA2 - Proprietary A2" + return None + + +#: SAE J1939 preferred source addresses (the industry-group-independent +#: table). Addresses 128 to 247 are assigned dynamically or by industry group +#: and have no fixed meaning, so they are reported by number. +SOURCE_ADDRESSES: dict[int, str] = { + 0: "Engine #1", 1: "Engine #2", 2: "Turbocharger", 3: "Transmission #1", + 4: "Transmission #2", 5: "Shift Console - Primary", 6: "Shift Console - Secondary", + 7: "Power TakeOff - (Main or Rear)", 8: "Axle - Steering", 9: "Axle - Drive #1", + 10: "Axle - Drive #2", 11: "Brakes - System Controller", 12: "Brakes - Steer Axle", + 13: "Brakes - Drive Axle #1", 14: "Brakes - Drive Axle #2", 15: "Retarder - Engine", + 16: "Retarder - Driveline", 17: "Cruise Control", 18: "Fuel System", + 19: "Steering Controller", 20: "Suspension - Steer Axle", + 21: "Suspension - Drive Axle #1", 22: "Suspension - Drive Axle #2", + 23: "Instrument Cluster #1", 24: "Trip Recorder", + 25: "Passenger-Operator Climate Control #1", + 26: "Alternator/Electrical Charging System", 27: "Aerodynamic Control", + 28: "Vehicle Navigation", 29: "Vehicle Security", 30: "Electrical System", + 31: "Starter System", 32: "Tractor-Trailer Bridge #1", 33: "Body Controller", + 34: "Auxiliary Valve Control or Engine Air System Valve Control", + 35: "Hitch Control", 36: "Power TakeOff (Front or Secondary)", + 37: "Off Vehicle Gateway", 38: "Virtual Terminal (in cab)", + 39: "Management Computer #1", 40: "Cab Display #1", + 41: "Retarder, Exhaust, Engine #1", 42: "Headway Controller", + 43: "On-Board Diagnostic Unit", 44: "Retarder, Exhaust, Engine #2", + 45: "Endurance Braking System", 46: "Hydraulic Pump Controller", + 47: "Suspension - System Controller #1", 48: "Pneumatic - System Controller", + 49: "Cab Controller - Primary", 50: "Cab Controller - Secondary", + 51: "Tire Pressure Controller", 52: "Ignition Control Module #1", + 53: "Ignition Control Module #2", 54: "Seat Control #1", + 55: "Lighting - Operator Controls", 56: "Rear Axle Steering Controller #1", + 57: "Water Pump Controller", 58: "Passenger-Operator Climate Control #2", + 59: "Transmission Display - Primary", 60: "Transmission Display - Secondary", + 61: "Exhaust Emission Controller", 62: "Vehicle Dynamic Stability Controller", + 63: "Oil Sensor", 64: "Suspension - System Controller #2", + 65: "Information System Controller #1", 66: "Ramp Control", + 67: "Clutch/Converter Unit", 68: "Auxiliary Heater #1", 69: "Auxiliary Heater #2", + 70: "Engine Valve Controller", 71: "Chassis Controller #1", + 72: "Chassis Controller #2", 73: "Propulsion Battery Charger", + 74: "Communications Unit, Cellular", 75: "Communications Unit, Satellite", + 76: "Communications Unit, Radio", 77: "Steering Column Unit", + 78: "Fan Drive Controller", 79: "Seat Control #2", 80: "Parking Brake Controller", + 81: "Aftertreatment #1 System Gas Intake", 82: "Aftertreatment #1 System Gas Outlet", + 83: "Safety Restraint System", 84: "Cab Climate Control", + 85: "Aftertreatment #2 System Gas Intake", 86: "Aftertreatment #2 System Gas Outlet", + 248: "File Server / Printer", 249: "Off Board Diagnostic-Service Tool #1", + 250: "Off Board Diagnostic-Service Tool #2", 251: "On-Board Data Logger", + 252: "Reserved for Experimental Use", 253: "Reserved for OEM", + 254: "Null Address", 255: "Global", +} + + +# ── decoding one field ─────────────────────────────────────────────────────── + +NOT_AVAILABLE = object() +ERROR = "error" + + +def extract(data: bytes, bit: int, bits: int) -> int | None: + """Little-endian bit field; None when the frame is too short.""" + if bit + bits > len(data) * 8: + return None + value = int.from_bytes(bytes(data), "little") + return (value >> bit) & ((1 << bits) - 1) + + +def classify(raw: int, bits: int) -> str: + """"valid", "error", "reserved" or "na", by the J1939-71 ranges. + + Whole bytes, words and double words reserve their top codes: for one byte, + 251 to 253 are reserved, 254 is an error and 255 is not available, and the + wider ranges follow the same pattern in their top byte. A two-bit field + uses 10 for error and 11 for not available; wider discrete fields use their + two highest codes the same way. + """ + if bits in (8, 16, 32): + top = raw >> (bits - 8) + if top <= 0xFA: + return "valid" + if top == 0xFE: + return "error" + if top == 0xFF: + return "na" + return "reserved" + full = (1 << bits) - 1 + if raw == full: + return "na" + if bits >= 2 and raw == full - 1: + return "error" + return "valid" + + +def decode_spn(spec: Spn, data: bytes): + """The value, a state label, ERROR, or NOT_AVAILABLE for one parameter.""" + raw = extract(data, spec.bit, spec.bits) + if raw is None: + return NOT_AVAILABLE + if spec.ascii: + text = bytes(data)[spec.bit // 8:(spec.bit + spec.bits) // 8] + if all(b == 0xFF for b in text) or all(b == 0 for b in text): + return NOT_AVAILABLE + return text.decode("ascii", "replace").strip("\x00 ") + if spec.special and raw in spec.special: + return spec.special[raw] + if spec.states is not None and raw in spec.states: + return spec.states[raw] + kind = classify(raw, spec.bits) + if kind == "na" or kind == "reserved": + return NOT_AVAILABLE + if kind == "error": + return ERROR + if spec.states is not None: + return f"state {raw}" + return raw * spec.scale + spec.offset diff --git a/tests/real_data/acceptance_stress.py b/tests/real_data/acceptance_stress.py index a043dbb..ae84968 100644 --- a/tests/real_data/acceptance_stress.py +++ b/tests/real_data/acceptance_stress.py @@ -157,13 +157,17 @@ def scan(): df = load(BIG) with timed("pgn scan") as t: hits = scan_for_j1939(df) - named = [h for h in hits if not h["pgn_name"].startswith("PGN ")] + # one row per sender: a PGN from four ECUs is four rows + pgns = {h["pgn"] for h in hits} + named = {h["pgn"] for h in hits if not h["pgn_name"].startswith("PGN ")} j1939 = [h for h in hits if h["protocol"] == "J1939"] - FACTS["pgns"] = {"total": len(hits), "named": len(named)} - return (len(hits) > 90 and len(named) >= 8 and len(j1939) == len(hits) - and t.seconds < 2), \ - (f"{len(hits)} PGNs, {len(named)} named, all J1939 " - f"(unlike the marine log), in {t.seconds:.2f} s") + frames = sum(h["frame_count"] for h in hits) + FACTS["pgns"] = {"senders": len(hits), "total": len(pgns), "named": len(named)} + return (len(pgns) > 90 and len(named) >= 8 and len(j1939) == len(hits) + and frames == len(df) and t.seconds < 2), \ + (f"{len(pgns)} PGNs from {len(hits)} PGN-sender pairs, {len(named)} named, " + f"all J1939 (unlike the marine log); counts cover all {frames:,} frames; " + f"{t.seconds:.2f} s") check("every identifier decodes as a J1939 PGN", scan) def engine_speed(): @@ -178,7 +182,8 @@ def engine_speed(): for _, row in eec1.iterrows(): data = bytes(int(row[f"B{i}"]) for i in range(8)) out = decode_pgn(0xF004, data) - if "Engine Speed" in out: + # a sensor error is reported as "error", never as a number + if "Engine Speed" in out and isinstance(out["Engine Speed"][0], float): values.append(out["Engine Speed"][0]) lo, hi = min(values), max(values) mean = sum(values) / len(values) @@ -219,6 +224,79 @@ def dm1(): f"codes, lamps {lamps}") check("active fault codes decode", dm1) + def series(cid, pgn, name): + import numpy as np + g = load(BIG) + g = g[g["ID"] == cid] + t, v = [], [] + for r in g.itertuples(): + data = bytes(int(getattr(r, f"B{i}")) for i in range(int(r.DLC))) + out = decode_pgn(pgn, data) + if name in out and isinstance(out[name][0], float): + t.append(r.Timestamp) + v.append(out[name][0]) + return np.array(t), np.array(v) + + def two_ecus_agree(): + """The brakes (EBC2, SA 0x0B) and the engine (CCVS, SA 0x00) each report + road speed. Wrong layouts cannot agree with each other by accident.""" + import numpy as np + ta, va = series("18FEBF0B", 0xFEBF, "Front Axle Speed") + tc, vc = series("18FEF100", 0xFEF1, "Wheel-Based Vehicle Speed") + if len(ta) < 100 or len(tc) < 100: + return None, "EBC2 or CCVS missing" + other = np.interp(ta, tc, vc) + diff = np.abs(va - other) + r = float(np.corrcoef(va, other)[0, 1]) + FACTS["speed_agreement"] = {"pairs": len(ta), "mean_abs_kmh": float(diff.mean()), "r": r} + return diff.mean() < 1.0 and r > 0.999, ( + f"{len(ta)} pairs, mean difference {diff.mean():.2f} km/h, r {r:.5f}, " + f"over {va.min():.0f} to {va.max():.0f} km/h") + check("two ECUs report the same road speed", two_ecus_agree) + + def engine_readings_are_physical(): + """A running diesel's temperatures, pressures and supply, each from the + message J1939-71 puts it in.""" + import numpy as np + checks = { + "coolant": ("18FEEE00", 0xFEEE, "Engine Coolant Temperature", 60, 110), + "oil temperature": ("18FEEE00", 0xFEEE, "Engine Oil Temperature 1", 60, 130), + "oil pressure": ("18FEEF00", 0xFEEF, "Engine Oil Pressure", 100, 700), + "barometer": ("18FEF500", 0xFEF5, "Barometric Pressure", 70, 110), + "battery": ("18FEF700", 0xFEF7, "Battery Potential / Power Input 1", 11, 30), + "fuel level": ("18FEFC17", 0xFEFC, "Fuel Level 1", 0, 100), + } + ok, out = [], [] + for label, (cid, pgn, name, lo, hi) in checks.items(): + _, v = series(cid, pgn, name) + if len(v) == 0: + return False, f"no {label} decoded" + ok.append(lo <= v.min() and v.max() <= hi) + out.append(f"{label} {v.min():.4g}..{v.max():.4g}") + _, boost = series("18FEF600", 0xFEF6, "Engine Intake Manifold #1 Pressure") + _, inlet = series("18FEF600", 0xFEF6, "Engine Air Inlet Pressure") + gauge_to_absolute = float(np.median(inlet - boost)) + ok.append(90 <= gauge_to_absolute <= 110) + out.append(f"absolute inlet minus boost {gauge_to_absolute:.0f} kPa, the barometer") + return all(ok), "; ".join(out) + check("engine readings come from the right messages and are physical", + engine_readings_are_physical) + + def lifetime_counters_agree(): + """Distance from VD and from HRVD, and distance over fuel against the + ECU's own average economy.""" + _, vd = series("18FEE000", 0xFEE0, "Total Vehicle Distance") + _, hr = series("18FEC100", 0xFEC1, "High Resolution Total Vehicle Distance") + _, fuel = series("18FEE900", 0xFEE9, "Engine Total Fuel Used") + _, avg = series("18FEF200", 0xFEF2, "Engine Average Fuel Economy") + if not (len(vd) and len(hr) and len(fuel) and len(avg)): + return None, "a lifetime counter is missing" + economy = vd[-1] / fuel[-1] + ok = abs(vd[-1] - hr[-1]) < 1.0 and abs(economy - avg[-1]) / avg[-1] < 0.05 + return ok, (f"VD {vd[-1]:,.1f} km, HRVD {hr[-1]:,.1f} km; " + f"{economy:.3f} km/L lifetime against {avg[-1]:.3f} reported") + check("lifetime distance and fuel agree with each other", lifetime_counters_agree) + # ── 3. the detectors, over whole logs ──────────────────────────────────────── diff --git a/tests/test_j1939_spns.py b/tests/test_j1939_spns.py new file mode 100644 index 0000000..8a9f65b --- /dev/null +++ b/tests/test_j1939_spns.py @@ -0,0 +1,231 @@ +"""J1939 parameter layouts, pinned against frames from a real truck. + +Every frame below is copied verbatim from the 145,534-frame J1939 log in the +acceptance corpus (a CANedge recording from CSS Electronics). The expected +values are what SAE J1939-71 says those bytes mean, and they agree with each +other the way a running truck's readings must: two ECUs report the same road +speed, the absolute inlet pressure is the boost pressure plus the barometer. + +The table these replace put coolant temperature in the wrong message, filed +ETC1, the wheel speeds and ERC1 under other PGNs, read the battery voltages +from the current bytes, and decoded every switch as a whole byte. Nothing +tested a layout, which is how all of that survived. +""" +import pandas as pd +import pytest + +from canlab.core.j1939 import ( + decode_pgn, decode_spns, parse_j1939_id, pgn_name, sa_name, scan_for_j1939, +) +from canlab.core.j1939_db import PGNS, classify + + +def frame(hexstr: str) -> bytes: + return bytes.fromhex(hexstr) + + +# ── verbatim frames from the truck ─────────────────────────────────────────── + +def test_engine_temperatures_come_from_et1(): + out = decode_pgn(0xFEEE, frame("86 4C 20 2F FF FF 49 FF")) + assert out["Engine Coolant Temperature"] == (94.0, "°C") + assert out["Engine Fuel Temperature 1"] == (36.0, "°C") + assert out["Engine Oil Temperature 1"] == (104.0, "°C") + assert out["Engine Intercooler Temperature"] == (33.0, "°C") + assert "Engine Turbocharger Oil Temperature" not in out # FF FF: not fitted + + +def test_coolant_temperature_is_not_read_from_efl_p1(): + """The old table read byte 6 of EFL/P1 as coolant temperature. That byte is + half of the crankcase pressure.""" + out = decode_pgn(0xFEEF, frame("B1 FF FF 49 FF FF FF FA")) + assert "Engine Coolant Temperature" not in out + assert out["Engine Fuel Delivery Pressure"] == (708.0, "kPa") + assert out["Engine Oil Pressure"] == (292.0, "kPa") + assert out["Engine Coolant Level"] == (100.0, "%") + assert "Engine Oil Level" not in out and "Engine Crankcase Pressure" not in out + + +def test_ambient_conditions_from_the_engine_and_the_body_controller(): + engine = decode_pgn(0xFEF5, frame("CA FF FF FF FF 49 FF FF")) + assert engine["Barometric Pressure"] == (101.0, "kPa") + assert engine["Engine Air Inlet Temperature"] == (33.0, "°C") + body = decode_pgn(0xFEF5, frame("FF FF FF A0 22 FF FF FF")) + assert body == {"Ambient Air Temperature": (4.0, "°C")} # January + + +def test_battery_voltage_is_read_from_its_own_bytes(): + out = decode_pgn(0xFEF7, frame("FF FF FF FF 1A 01 1A 01")) + assert out == {"Battery Potential / Power Input 1": (14.1, "V"), + "Keyswitch Battery Potential": (14.1, "V")} + + +def test_ccvs_switches_are_two_bit_fields_in_their_own_bytes(): + """The old table decoded Cruise Control Active from byte 1, which holds the + parking brake and axle switches, and reported it as 195.""" + out = decode_pgn(0xFEF1, frame("C3 40 47 00 00 00 00 30")) + assert out["Wheel-Based Vehicle Speed"] == (71.25, "km/h") + assert out["Parking Brake Switch"] == ("not set", "") + assert out["Cruise Control Active"] == ("off", "") + assert out["Brake Switch"] == ("released", "") + assert out["Clutch Switch"] == ("released", "") + assert out["Cruise Control States"] == ("off/disabled", "") + assert "Two Speed Axle Switch" not in out # 11: not available + + +def test_two_ecus_agree_on_road_speed(): + """EBC2 from the brakes and CCVS from the engine, a moment apart.""" + brakes = decode_pgn(0xFEBF, frame("D4 46 7E 7C 76 77 FF FF")) + engine = decode_pgn(0xFEF1, frame("C3 40 47 00 00 00 00 30")) + assert brakes["Front Axle Speed"][0] == pytest.approx(70.828, abs=1e-3) + assert abs(brakes["Front Axle Speed"][0] - engine["Wheel-Based Vehicle Speed"][0]) < 1.0 + assert brakes["Relative Speed; Front Axle, Left Wheel"] == (0.0625, "km/h") + assert brakes["Relative Speed; Rear Axle #1, Left Wheel"] == (-0.4375, "km/h") + assert "Relative Speed; Rear Axle #2, Left Wheel" not in brakes + + +def test_absolute_inlet_pressure_is_boost_plus_the_barometer(): + out = decode_pgn(0xFEF6, frame("FF 2B 49 5E FF C6 46 FF")) + boost = out["Engine Intake Manifold #1 Pressure"][0] + inlet = out["Engine Air Inlet Pressure"][0] + assert (boost, inlet) == (86.0, 188.0) + assert inlet - boost == pytest.approx(101, abs=2) # AMB said 101 kPa + assert out["Engine Exhaust Gas Temperature"] == (293.1875, "°C") + + +def test_engine_controller_1(): + out = decode_pgn(0xF004, frame("21 B7 B7 40 27 00 F4 B7")) + assert out["Engine Speed"] == (1256.0, "rpm") + assert out["Engine Torque Mode"] == ("accelerator pedal / operator selection", "") + assert out["Driver's Demand Engine - Percent Torque"] == (58.0, "%") + assert out["Actual Engine - Percent Torque"] == (58.0, "%") + + +def test_the_retarder_names_itself_as_the_controlling_device(): + out = decode_pgn(0xF000, frame("00 7D FF FF 0F 7D FF FF")) + assert out["Actual Retarder - Percent Torque"] == (0.0, "%") + assert out["Source Address of Controlling Device for Retarder Control"][0] == 15.0 + assert parse_j1939_id(0x18F0000F)["sa_name"] == "Retarder - Engine (0x0F)" + + +def test_lifetime_counters(): + hours = decode_pgn(0xFEE5, frame("96 9F 02 00 C9 E6 08 00")) + assert hours["Engine Total Hours of Operation"] == (8596.3, "h") + assert hours["Engine Total Revolutions"] == (583369000.0, "r") + mean_rpm = hours["Engine Total Revolutions"][0] / hours["Engine Total Hours of Operation"][0] / 60 + assert 900 < mean_rpm < 1400 # a working diesel's life + dist = decode_pgn(0xFEE0, frame("E9 B8 38 00 E9 B8 38 00")) + assert dist["Total Vehicle Distance"] == (464669.125, "km") + + +def test_time_date_and_position_from_the_telematics_unit(): + td = decode_pgn(0xFEE6, frame("C4 30 09 01 34 23 83 7E")) + assert (td["Year"][0], td["Month"][0], td["Day"][0]) == (2020.0, 1.0, 13.0) + assert (td["Hours"][0], td["Minutes"][0], td["Seconds"][0]) == (9.0, 48.0, 49.0) + vp = decode_pgn(0xFEF3, frame("8C 1A 5A 95 E1 8C DC 4F")) + assert vp["Latitude"][0] == pytest.approx(40.5710, abs=1e-4) + assert vp["Longitude"][0] == pytest.approx(-76.0146, abs=1e-4) + assert parse_j1939_id(0x18FEE64A)["sa_name"] == "Communications Unit, Cellular (0x4A)" + + +def test_torque_speed_control_from_the_brakes(): + out = decode_pgn(0x0000, frame("FC FF FA FA FF FF FF FF")) + assert out["Engine Override Control Mode"] == ("override disabled", "") + assert out["Override Control Mode Priority"] == ("low", "") + assert out["Engine Requested Speed/Speed Limit"] == (8031.875, "rpm") + + +# ── the J1939-71 value ranges ──────────────────────────────────────────────── + +def test_an_error_code_is_reported_as_an_error_not_a_reading(): + out = decode_pgn(0xFEEE, frame("FE FF FF FE FF FF FF FF")) # 0xFE, 0xFExx + assert out["Engine Coolant Temperature"] == ("error", "") + assert out["Engine Oil Temperature 1"] == ("error", "") + assert classify(0xFE, 8) == "error" and classify(0xFE12, 16) == "error" + assert classify(0xFE000001, 32) == "error" + + +def test_reserved_and_not_available_codes_are_not_values(): + for raw in (0xFB, 0xFC, 0xFD, 0xFF): + assert "Engine Coolant Temperature" not in decode_pgn(0xFEEE, bytes([raw]) + b"\xff" * 7) + assert classify(0xFAFF, 16) == "valid" and classify(0xFB00, 16) == "reserved" + assert classify(0xFF00, 16) == "na" + + +def test_two_bit_states(): + # CCVS byte 4: cruise active 01, enable 10 (error), brake 11 (n/a), clutch 00 + out = decode_pgn(0xFEF1, frame("FF FF FF 39 FF FF FF FF")) + assert out["Cruise Control Active"] == ("active", "") + assert out["Cruise Control Enable Switch"] == ("error", "") + assert "Brake Switch" not in out + assert out["Clutch Switch"] == ("released", "") + + +def test_gear_park_and_range_text(): + out = decode_pgn(0xF005, frame("FB E8 03 7C 44 20 44 20")) + assert out["Transmission Selected Gear"] == ("park", "") + assert out["Transmission Actual Gear Ratio"] == (1.0, "") + assert out["Transmission Current Gear"] == (-1.0, "") # one reverse gear + assert out["Transmission Current Range"] == ("D", "") + + +def test_a_short_frame_decodes_what_it_holds(): + assert decode_pgn(0xFEEE, bytes([0x82])) == {"Engine Coolant Temperature": (90.0, "°C")} + + +def test_decode_spns_keeps_numbers_and_status(): + rows = {r["spn"]: r for r in decode_spns(0xFEEE, frame("FE 4C FF FF FF FF 49 FF"))} + assert rows[110]["status"] == "error" and rows[110]["value"] is None + assert rows[174]["status"] == "ok" and rows[174]["value"] == 36.0 + assert rows[175]["status"] == "not available" + + +# ── the tables themselves ──────────────────────────────────────────────────── + +def test_parameter_groups_carry_their_j1939_71_numbers(): + by_acronym = {p.acronym: pgn for pgn, p in PGNS.items()} + assert by_acronym["ETC1"] == 0xF002 and by_acronym["ETC2"] == 0xF005 + assert by_acronym["ERC1"] == 0xF000 and by_acronym["EBC1"] == 0xF001 + assert by_acronym["EBC2"] == 0xFEBF and by_acronym["ET1"] == 0xFEEE + assert by_acronym["VEP1"] == 0xFEF7 and by_acronym["CCVS"] == 0xFEF1 + + +def test_no_two_parameters_overlap_in_a_message(): + for pgn, p in PGNS.items(): + taken = set() + for spec in p.spns: + bits = set(range(spec.bit, spec.bit + spec.bits)) + assert not bits & taken, f"{p.acronym}: SPN {spec.spn} overlaps" + assert max(bits) < 64, f"{p.acronym}: SPN {spec.spn} runs past 8 bytes" + taken |= bits + + +def test_source_addresses_use_the_preferred_address_table(): + assert sa_name(0x00) == "Engine #1 (0x00)" + assert sa_name(0x0B) == "Brakes - System Controller (0x0B)" + assert sa_name(0x17) == "Instrument Cluster #1 (0x17)" + assert sa_name(0x21) == "Body Controller (0x21)" + assert sa_name(0x3D) == "Exhaust Emission Controller (0x3D)" + assert sa_name(0xF9) == "Off Board Diagnostic-Service Tool #1 (0xF9)" + assert sa_name(0x90) == "SA 0x90" # dynamic range + + +def test_names_for_proprietary_and_unknown_groups(): + assert pgn_name(0xFF21).startswith("PropB") + assert pgn_name(0xF004).startswith("EEC1") + assert pgn_name(0xFD41) == "PGN 0xFD41" + + +def test_the_scan_lists_every_sender_with_its_own_count(): + """Keyed by PGN alone, the scan kept the first sender and dropped the rest.""" + rows = ([{"ID": "18FEF100", "Timestamp": i * 0.1} for i in range(10)] + + [{"ID": "18FEF121", "Timestamp": i * 0.1} for i in range(4)] + + [{"ID": "18FECA3D", "Timestamp": i} for i in range(3)] + + [{"ID": "18FECA0B", "Timestamp": i} for i in range(2)] + + [{"ID": "0A6", "Timestamp": 0.0}]) + scan = scan_for_j1939(pd.DataFrame(rows)) + got = {(r["pgn"], r["sa"]): r["frame_count"] for r in scan} + assert got == {(0xFEF1, 0x00): 10, (0xFEF1, 0x21): 4, + (0xFECA, 0x3D): 3, (0xFECA, 0x0B): 2} + assert sum(got.values()) == 19 # every 29-bit frame + assert scan[0]["pgn_name"].startswith("CCVS")