Skip to content

Fix: J1939 decoded wrong values from wrong bytes and wrong messages #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Sherin-SEF-AI merged 1 commit into from
Sep 23, 2026
Merged

Fix: J1939 decoded wrong values from wrong bytes and wrong messages #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 75 additions & 161 deletions canlab/core/j1939.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,130 +21,17 @@
For PDU1 (PF < 240): PGN = (DP<<16) | (PF<<8) [destination = PS byte]
"""

# ── PGN → (name, SPNs) ────────────────────────────────────────────────────────
# SPN entry: (name, start_byte, length_bytes, scale, offset, unit)
# start_byte is 0-based within the 8-byte data field

_PGN_DB: dict[int, dict] = {
# Electronic Engine Controller 1
0xF004: {
"name": "EEC1 — Electronic Engine Controller 1",
"spns": {
190: ("Engine Speed", 3, 2, 0.125, 0, "rpm"),
512: ("Driver's Demand Engine", 1, 1, 1.0, -125, "%"),
513: ("Actual Engine Torque", 2, 1, 1.0, -125, "%"),
899: ("Engine Torque Mode", 0, 1, 1.0, 0, ""),
},
},
# Vehicle Speed / Cruise Control
0xFEF1: {
"name": "CCVS — Cruise Control/Vehicle Speed",
"spns": {
84: ("Wheel-Based Vehicle Speed", 1, 2, 1/256, 0, "km/h"),
595: ("Cruise Control Active", 0, 1, 1.0, 0, ""),
596: ("Cruise Control Enable", 0, 1, 1.0, 0, ""),
},
},
# Fuel Economy (Liquid)
0xFEF2: {
"name": "LFE — Fuel Economy",
"spns": {
183: ("Fuel Rate", 0, 2, 0.05, 0, "L/h"),
184: ("Instantaneous Fuel Econ.", 2, 2, 1/512, 0, "km/L"),
185: ("Average Fuel Economy", 4, 2, 1/512, 0, "km/L"),
},
},
# Engine Fluid Level / Pressure 1
0xFEEF: {
"name": "EFL/P1 — Engine Fluid Level/Pressure 1",
"spns": {
94: ("Fuel Delivery Pressure", 0, 1, 4.0, 0, "kPa"),
22: ("Engine Oil Level", 1, 1, 0.4, 0, "%"),
100: ("Engine Oil Pressure", 3, 1, 4.0, 0, "kPa"),
110: ("Engine Coolant Temp", 5, 1, 1.0, -40,"°C"),
},
},
# Transport protocol (J1939-21). The frames themselves are envelopes; the
# reassembler in core/multiframe.py turns them back into the PGN they carry.
0xEC00: {"name": "TP.CM - Transport Protocol Connection Management", "spns": {}},
0xEB00: {"name": "TP.DT - Transport Protocol Data Transfer", "spns": {}},
# Two configuration messages that only ever travel as BAM broadcasts.
0xFEE3: {"name": "EC1 - Engine Configuration 1", "spns": {}},
0xFEE1: {"name": "RC - Retarder Configuration", "spns": {}},
# Ambient Conditions
0xFEF5: {
"name": "AMB — Ambient Conditions",
"spns": {
171: ("Ambient Air Temperature", 3, 2, 0.03125, -273, "°C"),
108: ("Barometric Pressure", 1, 1, 0.5, 0, "kPa"),
},
},
# Vehicle Electrical Power
0xFEF7: {
"name": "VEP — Vehicle Electrical Power",
"spns": {
158: ("Key Switch Battery Voltage", 0, 2, 0.05, 0, "V"),
168: ("Battery Potential", 2, 2, 0.05, 0, "V"),
},
},
# Transmission
0xF005: {
"name": "ETC1 — Electronic Transmission Controller 1",
"spns": {
522: ("Transmission Selected Gear", 3, 1, 1.0, -125, ""),
523: ("Transmission Actual Gear", 4, 1, 1.0, -125, ""),
524: ("Transmission Current Range", 5, 2, 1.0, 0, ""),
},
},
# Axle / Drive Wheel Speed
0xFE68: {
"name": "AWSS — Axle/Drive Wheel Speed",
"spns": {
904: ("Front Axle Speed", 0, 2, 1/256, 0, "km/h"),
905: ("Relative Speed FA1", 2, 1, 1/16, 0, "km/h"),
},
},
# DM1 — Active DTCs
0xFECA: {
"name": "DM1 — Active Diagnostic Trouble Codes",
"spns": {}, # complex encoding handled separately
},
# Engine Hours / Revolutions
0xFEE5: {
"name": "HOURS — Engine Hours, Revolutions",
"spns": {
247: ("Total Engine Hours", 0, 4, 0.05, 0, "h"),
249: ("Total Engine Revs", 4, 4, 1000.0,0, "rev"),
},
},
# Retarder
0xF006: {
"name": "ERC1 — Electronic Retarder Controller 1",
"spns": {
520: ("Retarder Torque Mode", 0, 1, 1.0, 0, ""),
521: ("Retarder Actual Retarding Pct",2, 1, 1.0, 0, "%"),
},
},
}
# ── the J1939 tables ─────────────────────────────────────────────────────────
# Layouts, names and source addresses live in core/j1939_db.py, where each one
# can be read against the specification and each is pinned by a test.

# Source Address → ECU type
_SA_NAMES: dict[int, str] = {
0x00: "Engine #1",
0x01: "Engine #2",
0x03: "Transmission",
0x10: "Exhaust/Emission Control",
0x11: "Exhaust/Emission Ctrl #2",
0x17: "Fuel System",
0x21: "Brakes — System Controller",
0x28: "Instrument Cluster #1",
0x29: "Trip Recorder",
0x2A: "Vehicle Management System",
0x2C: "Cab Display #1",
0x33: "Body Controller",
0x3D: "Retarder — Exhaust Engine #1",
0xF0: "Off-Board Diagnostic Tool",
0xFF: "Global (broadcast)",
}
from canlab.core.j1939_db import ( # noqa: E402
ERROR, NOT_AVAILABLE, PGNS, SOURCE_ADDRESSES, decode_spn, proprietary_name,
)

#: Kept under its old name: other modules and the MCP tools look PGNs up here.
_PGN_DB = PGNS
_SA_NAMES = SOURCE_ADDRESSES


# ── Public API ────────────────────────────────────────────────────────────────
Expand All @@ -164,8 +51,8 @@ def parse_j1939_id(arb_id: int) -> dict:
"pgn": 0xFEF1,
"sa": 0x00,
"da": 0xFF, # 0xFF = broadcast
"sa_name": "Engine #1",
"pgn_name": "CCVS — Cruise Control/Vehicle Speed",
"sa_name": "Engine #1 (0x00)",
"pgn_name": "CCVS - Cruise Control/Vehicle Speed",
}
"""
priority = (arb_id >> 26) & 0x07
Expand All @@ -174,10 +61,10 @@ def parse_j1939_id(arb_id: int) -> dict:
ps = (arb_id >> 8) & 0xFF
sa = arb_id & 0xFF

if pf >= 0xF0: # PDU2 — PS is group extension
if pf >= 0xF0: # PDU2: PS is group extension
pgn = (dp << 16) | (pf << 8) | ps
da = 0xFF
else: # PDU1 — PS is destination address
else: # PDU1: PS is destination address
pgn = (dp << 16) | (pf << 8)
da = ps

Expand All @@ -200,7 +87,7 @@ def parse_j1939_id(arb_id: int) -> dict:
"pgn": pgn,
"sa": sa,
"da": da,
"sa_name": _SA_NAMES.get(sa, f"SA 0x{sa:02X}"),
"sa_name": sa_name(sa),
"pgn_name": pgn_name(pgn),
"protocol": "J1939",
"single_frame": True,
Expand All @@ -214,7 +101,16 @@ def pgn_name(pgn: int) -> str:
"""The name of a PGN in whichever table owns it, or a placeholder."""
if is_nmea2000(pgn):
return _N2K_NAMES.get(pgn, (f"PGN {pgn}", True))[0]
return _PGN_DB.get(pgn, {}).get("name", f"PGN 0x{pgn:04X}")
entry = PGNS.get(pgn)
if entry is not None:
return entry.name
return proprietary_name(pgn) or f"PGN 0x{pgn:04X}"


def sa_name(sa: int) -> str:
"""The J1939 preferred name for a source address, or its number."""
name = SOURCE_ADDRESSES.get(sa)
return f"{name} (0x{sa:02X})" if name else f"SA 0x{sa:02X}"


# ── NMEA 2000 ────────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -496,59 +392,77 @@ def decode_dm1(data: bytes) -> dict:

def decode_pgn(pgn: int, data: bytes) -> dict:
"""
Decode SPN values from `data` (8 bytes) for the given PGN.
Decode the parameters of one message. Returns {name: (value, unit)}.

Returns {spn_name: (value, unit)} or {} if PGN unknown / data too short.
DM1 (0xFECA) is decoded into structured DTCs via decode_dm1.
A measured value is a number. A discrete field (a switch, a mode) is its
label. A parameter the sender reports as an error is the string "error",
so a failed sensor is visible rather than read as a reading. A parameter
the sender marks "not available", or that falls in a reserved range, is
left out, because none of those is a value.

DM1 (0xFECA) is decoded into structured DTCs via decode_dm1, and NMEA 2000
PGNs by decode_n2k.
"""
if is_nmea2000(pgn):
return decode_n2k(pgn, data)
if pgn == 0xFECA:
return decode_dm1(data)
info = _PGN_DB.get(pgn)
if not info or not info.get("spns"):
entry = PGNS.get(pgn)
if entry is None or not entry.spns:
return {}

result = {}
for spn_id, (name, start, length, scale, offset, unit) in info["spns"].items():
end = start + length
if end > len(data):
continue
raw_bytes = data[start:end]
raw_int = int.from_bytes(raw_bytes, "little")
# 0xFF...F = not available indicator
if raw_bytes == b"\xFF" * length:
for spec in entry.spns:
value = decode_spn(spec, data)
if value is NOT_AVAILABLE:
continue
value = raw_int * scale + offset
result[name] = (round(value, 4), unit)

if isinstance(value, (int, float)):
value = round(float(value), 4)
result[spec.name] = (value, spec.unit if value != ERROR else "")
return result


def decode_spns(pgn: int, data: bytes) -> list[dict]:
"""The same decode with the SPN numbers and the raw status kept, for
tools that need to cite the parameter rather than show it."""
entry = PGNS.get(pgn)
if entry is None or is_nmea2000(pgn):
return []
out = []
for spec in entry.spns:
value = decode_spn(spec, data)
status = ("not available" if value is NOT_AVAILABLE
else "error" if value == ERROR else "ok")
out.append({"spn": spec.spn, "name": spec.name, "unit": spec.unit,
"value": None if status != "ok" else
(round(float(value), 4) if isinstance(value, (int, float)) else value),
"status": status})
return out


def scan_for_j1939(df) -> list[dict]:
"""
Scan a frames DataFrame for J1939 messages (extended 29-bit IDs inferred from
ID values > 0x7FF when stored as hex strings).
One row per parameter group and sender in a frames DataFrame.

A PGN sent by several ECUs (CCVS from the engine and the body controller,
DM1 from nine modules on the truck log) is a row per sender, each with its
own frame count. Keying by PGN alone kept the first sender and dropped the
rest, so a scan understated both who was talking and how much.

Returns list of:
{"id_hex", "priority", "pgn", "pgn_name", "sa", "sa_name", "frame_count"}
sorted by pgn.
Returns dicts with "id_hex", "priority", "pgn", "pgn_name", "sa",
"sa_name", "protocol", "single_frame", "frame_count".
"""
if df is None or df.empty or "ID" not in df.columns:
return []
counts = df["ID"].value_counts()
results = []
seen = set()
for can_id in df["ID"].unique():
for can_id, count in counts.items():
try:
arb_id = int(can_id, 16)
except (ValueError, TypeError):
continue
if arb_id <= 0x7FF:
continue # standard 11-bit ID — not J1939
continue # standard 11-bit ID, not J1939
parsed = parse_j1939_id(arb_id)
key = parsed["pgn"]
if key in seen:
continue
seen.add(key)
count = int((df["ID"] == can_id).sum())
results.append({
"id_hex": can_id,
"priority": parsed["priority"],
Expand All @@ -558,13 +472,13 @@ def scan_for_j1939(df) -> list[dict]:
"sa_name": parsed["sa_name"],
"protocol": parsed["protocol"],
"single_frame": parsed["single_frame"],
"frame_count": count,
"frame_count": int(count),
})

# Named messages first, then by how much of the bus they are. Sorting by
# PGN put the four unnamed low-numbered messages at the top of a marine
# capture, so the visible rows all read "no idea" while the twenty named
# ones sat below the fold.
results.sort(key=lambda x: (x["pgn_name"].startswith("PGN "),
-x["frame_count"]))
-x["frame_count"], x["pgn"], x["sa"]))
return results
Loading
Loading