From fc7afa24b81b5e41e515ab3b013f3f5f3859b8d9 Mon Sep 17 00:00:00 2001 From: Rob Dobson Date: Mon, 9 Nov 2026 18:09:51 +0100 Subject: [PATCH 1/2] Decode poll data as consumers get it: pad a short poll; read field values in one place A data subscriber (registerForDeviceData) and the bus's latest value get each poll result at its actual length. The decoders read whole records of the record's size (resp.b plus the timestamp), as the poll store keeps them, so a poll with a variable read length (an LSM6DS FIFO read, r{...}) decoded to nothing unless the consumer padded it. RaftMicroPy didn't, nor did RaftROS's example; RaftROS's AutoPub did. - DeviceTypeRecords::decodePollData pads a last record cut short with zeros, as the poll store does, and decodes; decodeNewestPollRecord keeps only the newest record and returns how many the poll held. Static overloads take the decode function and sizes, and working space, for a consumer on the bus task. - AttrFieldDesc::getValue reads one element of a field from a decoded record and applies what is still to be scaled; elemCount() and isIntegral() go with it. DeviceManager, RaftMicroPy and RaftROS each had their own copy. - The decoders are unchanged: test_runtime_decoder already compares the three decoder kinds on truncated buffers. - New Linux test test_poll_data_decode: for every built-in record, a short last record decodes as the zero-padded data does; a real LSM6DS3TR-C FIFO poll gives every sample and the newest; getValue reads every field and every element of array attributes as its struct bytes. Co-Authored-By: Claude Opus 5.5 --- .../core/DeviceTypes/DeviceTypeRecord.h | 58 +++++ .../core/DeviceTypes/DeviceTypeRecords.cpp | 86 +++++++ .../core/DeviceTypes/DeviceTypeRecords.h | 50 ++++ linux_unit_tests/Makefile | 5 +- linux_unit_tests/README.md | 1 + linux_unit_tests/test_poll_data_decode.cpp | 224 ++++++++++++++++++ 6 files changed, 422 insertions(+), 2 deletions(-) create mode 100644 linux_unit_tests/test_poll_data_decode.cpp diff --git a/components/core/DeviceTypes/DeviceTypeRecord.h b/components/core/DeviceTypes/DeviceTypeRecord.h index 5b0e0d7..726898a 100644 --- a/components/core/DeviceTypes/DeviceTypeRecord.h +++ b/components/core/DeviceTypes/DeviceTypeRecord.h @@ -53,6 +53,64 @@ struct AttrFieldDesc float divisor; // Divisor from resp.a[].d (1.0 = no division) float addend; // Addend from resp.a[].a (applied after division) uint16_t count; // Elements in the field (>1 for array attributes; 0 treated as 1) + + // Reading a value from a decoded record. Every on-device consumer (DeviceManager named values, RaftMicroPy, + // RaftROS) uses these so that they all give the same value for the same field + + /// @brief Number of elements (1 for a scalar attribute) + uint16_t elemCount() const { return count ? count : 1; } + + /// @brief Bytes per element in the decoded struct + /// @note the type is the struct field's C type except for the few wider fields: int64 and uint64 are described + /// as Int32 and Uint32 and double as Float (no RaftCore record has a double; BLEBTHome has a uint64). + /// Such a field is read as its low 4 bytes, an element of an array of them at the wrong stride + uint32_t elemBytes() const + { + switch (type) + { + case AttrType::Float: case AttrType::Int32: case AttrType::Uint32: return 4; + case AttrType::Int16: case AttrType::Uint16: return 2; + default: return 1; + } + } + + /// @brief True if the value is a whole number: an integer field with nothing still to scale + bool isIntegral() const + { + return (type != AttrType::Float) && ((divisor == 0.0f) || (divisor == 1.0f)) && (addend == 0.0f); + } + + /// @brief Get one element's value from a decoded record (one poll struct) + /// @param pRecord the decoded record + /// @param recordSize the record's size (the record type's pollStructSize) + /// @param elemIdx element index (0 for a scalar attribute) + /// @param value (out) the value, with the field's divisor and addend applied - what is still to be applied + /// (1 and 0 where the decode has scaled the value already, see DecodeGenerator get_field_desc_def) + /// @return false if the element is outside the record + bool getValue(const void* pRecord, uint32_t recordSize, uint16_t elemIdx, double& value) const + { + value = 0.0; + uint32_t bytes = elemBytes(); + uint32_t pos = offset + (uint32_t)elemIdx * bytes; + if (!pRecord || (elemIdx >= elemCount()) || (pos + bytes > recordSize)) + return false; + const uint8_t* p = static_cast(pRecord) + pos; + switch (type) + { + case AttrType::Float: { float v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Int32: { int32_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Uint32: { uint32_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Int16: { int16_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Uint16: { uint16_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Int8: { value = (int8_t)*p; break; } + case AttrType::Uint8: { value = *p; break; } + case AttrType::Bool: { value = (*p != 0) ? 1.0 : 0.0; break; } + } + if ((divisor != 0.0f) && (divisor != 1.0f)) + value /= divisor; + value += addend; + return true; + } }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/components/core/DeviceTypes/DeviceTypeRecords.cpp b/components/core/DeviceTypes/DeviceTypeRecords.cpp index 95894d5..72db4e8 100644 --- a/components/core/DeviceTypes/DeviceTypeRecords.cpp +++ b/components/core/DeviceTypes/DeviceTypeRecords.cpp @@ -1494,6 +1494,92 @@ DeviceTypeRecordDecodeFn DeviceTypeRecords::getPollDecodeFn(DeviceTypeIndexType return pRec ? __atomic_load_n(&pRec->pollResultDecodeFn, __ATOMIC_ACQUIRE) : nullptr; } +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data of a device type, padding a record cut short (see the header) +uint32_t DeviceTypeRecords::decodePollData(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, uint32_t dataLen, + void* pStructOut, uint32_t structOutSize, uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState) const +{ + // The decode function is published after the fields it uses (release), so a record whose function is seen + // has its sizes set + DeviceTypeRecordDecodeFn decodeFn = getPollDecodeFn(deviceTypeIndex); + const DeviceTypeRecord* pRec = getRecord(deviceTypeIndex); + if (!decodeFn || !pRec) + return 0; + return decodePollData(decodeFn, pRec->pollDataSizeBytes, pData, dataLen, pStructOut, structOutSize, + maxRecCount, decodeState); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data of a device type and keep only the newest record (see the header) +uint32_t DeviceTypeRecords::decodeNewestPollRecord(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, + uint32_t dataLen, void* pStructOut, uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState) const +{ + DeviceTypeRecordDecodeFn decodeFn = getPollDecodeFn(deviceTypeIndex); + const DeviceTypeRecord* pRec = getRecord(deviceTypeIndex); + if (!decodeFn || !pRec) + return 0; + return decodeNewestPollRecord(decodeFn, pRec->pollDataSizeBytes, pRec->pollStructSize, pData, dataLen, + pStructOut, structOutSize, decodeState); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data with a given decode function, padding a record cut short (see the header) +uint32_t DeviceTypeRecords::decodePollData(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + const uint8_t* pData, uint32_t dataLen, void* pStructOut, uint32_t structOutSize, + uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState) +{ + if (!decodeFn || !pData || !pStructOut || (dataLen == 0) || (maxRecCount == 0)) + return 0; + + // Whole records are decoded as they are + const uint32_t recBytes = pollDataSizeBytes + DevicePollingInfo::POLL_RESULT_TIMESTAMP_SIZE; + const uint32_t partBytes = dataLen % recBytes; + if (partBytes == 0) + return decodeFn(pData, dataLen, pStructOut, structOutSize, maxRecCount, decodeState); + + // A last record without even its timestamp isn't a record + if (partBytes < DevicePollingInfo::POLL_RESULT_TIMESTAMP_SIZE) + return (dataLen > partBytes) ? + decodeFn(pData, dataLen - partBytes, pStructOut, structOutSize, maxRecCount, decodeState) : 0; + + // Pad the last record with zeros to the record size, as the poll store does + std::vector padded(pData, pData + dataLen); + padded.resize(dataLen - partBytes + recBytes, 0); + return decodeFn(padded.data(), padded.size(), pStructOut, structOutSize, maxRecCount, decodeState); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data with a given decode function and keep only the newest record (see the header) +uint32_t DeviceTypeRecords::decodeNewestPollRecord(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + uint32_t pollStructSize, const uint8_t* pData, uint32_t dataLen, void* pStructOut, + uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState, uint8_t* pScratch, uint32_t scratchSize) +{ + if (!decodeFn || (pollStructSize == 0) || !pStructOut || (structOutSize < pollStructSize)) + return 0; + + // Working space for every record the data holds + std::vector allocated; + if (!pScratch) + { + uint32_t maxRecs = std::min(NEWEST_DECODE_MAX_RECORDS, + std::max(1, NEWEST_DECODE_MAX_BYTES / pollStructSize)); + allocated.resize(maxRecs * pollStructSize); + pScratch = allocated.data(); + scratchSize = allocated.size(); + } + uint32_t maxRecs = std::min(scratchSize / pollStructSize, UINT16_MAX); + if (maxRecs == 0) + return 0; + + // Decode, then keep the newest + uint32_t numRecs = decodePollData(decodeFn, pollDataSizeBytes, pData, dataLen, pScratch, + maxRecs * pollStructSize, (uint16_t)maxRecs, decodeState); + if ((numRecs == 0) || (numRecs > maxRecs)) + return 0; + memcpy(pStructOut, pScratch + (numRecs - 1) * pollStructSize, pollStructSize); + return numRecs; +} + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// @brief Prepare the runtime decoder of a record just added (not yet published) with no compiled decoder: /// with policy OnIdentify, check that it can be decoded from its attribute list and mark it Pending (its diff --git a/components/core/DeviceTypes/DeviceTypeRecords.h b/components/core/DeviceTypes/DeviceTypeRecords.h index 753956a..fc50b01 100644 --- a/components/core/DeviceTypes/DeviceTypeRecords.h +++ b/components/core/DeviceTypes/DeviceTypeRecords.h @@ -480,6 +480,56 @@ class DeviceTypeRecords /// @return poll decode function DeviceTypeRecordDecodeFn getPollDecodeFn(uint16_t deviceTypeIndex) const; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Decoding poll data. Decoders read whole records of the record type's size (resp.b plus the timestamp), as + // the bus's poll store keeps them. But a data subscriber (DeviceManager::registerForDeviceData) and the bus's + // latest value get each poll result at its actual length, which is shorter for a poll with a variable read + // length (e.g. a FIFO read, r{...}): given to a decoder as it is, it decodes to nothing. These functions pad a + // record cut short with zeros, as the poll store does. Every on-device consumer decodes through them + + /// @brief Decode poll data of a device type + /// @param deviceTypeIndex device type index + /// @param pData poll data: one or more poll results, the last of which may be short + /// @param dataLen length of the data + /// @param pStructOut (out) decoded records (the record type's poll struct) + /// @param structOutSize size of pStructOut in bytes + /// @param maxRecCount maximum number of records to decode (a poll result can hold several, e.g. a FIFO read) + /// @param decodeState decode state, kept between calls for the same device + /// @return number of records decoded (0 if the type has no decoder) + uint32_t decodePollData(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, uint32_t dataLen, + void* pStructOut, uint32_t structOutSize, uint16_t maxRecCount, + RaftBusDeviceDecodeState& decodeState) const; + + /// @brief Decode poll data and keep only the newest record + /// @param pStructOut (out) the newest record (one poll struct) + /// @param structOutSize size of pStructOut in bytes (at least the record type's pollStructSize) + /// @return number of records the data held (0 if none, or the type has no decoder); pStructOut has the newest + /// @note a poll result can hold up to NEWEST_DECODE_MAX_RECORDS records; beyond that the newest are dropped + uint32_t decodeNewestPollRecord(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, uint32_t dataLen, + void* pStructOut, uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState) const; + + /// @brief Decode poll data with a given decode function (for a consumer that keeps a record's decoder) + /// @param decodeFn the record type's decode function + /// @param pollDataSizeBytes the record type's pollDataSizeBytes (resp.b) + static uint32_t decodePollData(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + const uint8_t* pData, uint32_t dataLen, void* pStructOut, uint32_t structOutSize, + uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState); + + /// @brief Decode poll data with a given decode function and keep only the newest record + /// @param pollStructSize the record type's pollStructSize + /// @param pScratch working space for the records decoded (nullptr to allocate it); a consumer on the bus task + /// passes its own so that nothing is allocated there + /// @param scratchSize size of pScratch in bytes (records beyond what fits are dropped, the newest first) + static uint32_t decodeNewestPollRecord(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + uint32_t pollStructSize, const uint8_t* pData, uint32_t dataLen, void* pStructOut, + uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState, + uint8_t* pScratch = nullptr, uint32_t scratchSize = 0); + + // The working space decodeNewestPollRecord allocates: up to NEWEST_DECODE_MAX_RECORDS records (an LSM6DS FIFO + // read holds up to 20), within NEWEST_DECODE_MAX_BYTES + static constexpr uint32_t NEWEST_DECODE_MAX_RECORDS = 32; + static constexpr uint32_t NEWEST_DECODE_MAX_BYTES = 4096; + // Where a record's poll decoder comes from: compiled into the firmware (generated, or inherited by an override // with the same response layout), made at runtime from its attribute list (RAFT_RUNTIME_DECODE), or none enum class DecoderKind : uint8_t diff --git a/linux_unit_tests/Makefile b/linux_unit_tests/Makefile index d652549..3d42f0a 100644 --- a/linux_unit_tests/Makefile +++ b/linux_unit_tests/Makefile @@ -110,6 +110,7 @@ SOURCES_test_device_type_policy = test_device_type_policy.cpp $(CORE)/DeviceType SOURCES_test_device_type_file = test_device_type_file.cpp $(CORE)/DeviceTypes/DeviceTypeFileLoader.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) SOURCES_test_catalogue_index = test_catalogue_index.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(COMMON_SOURCES) SOURCES_test_runtime_decoder = test_runtime_decoder.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(CORE)/DeviceTypes/DeviceTypeFileLoader.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) +SOURCES_test_poll_data_decode = test_poll_data_decode.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(CORE)/DeviceTypes/DeviceTypeFileLoader.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) SOURCES_test_protocol_exchange = test_protocol_exchange.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_file_upload_okto = test_file_upload_okto.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_file_upload_background = test_file_upload_background.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) @@ -124,7 +125,7 @@ SOURCES_test_espnow_config = test_espnow_config.cpp $(ESPNOW_DIR)/ESPNowConfig.c # Tests run by "make check" TESTS = test_raft_json test_raft_utils test_device_names test_device_settings test_device_type_settings test_device_data_subscribers test_device_type_records \ - test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder + test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_poll_data_decode test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder # Extra device type catalogues for test_device_type_records to sweep, e.g. # make check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tests/TestDevTypeRecs.json" @@ -208,7 +209,7 @@ $(DEV_TYPE_GEN_HEADER): $(DEV_TYPE_CATALOGUE) ../scripts/ProcessDevTypeJsonToC.p python3 ../scripts/ProcessDevTypeJsonToC.py $(DEV_TYPE_CATALOGUE) $(DEV_TYPE_GEN_HEADER) \ $(DEV_TYPE_GEN_DIR)/DevicePollRecords_generated.h > $(DEV_TYPE_GEN_DIR)/generator.log -$(call objs,test_device_type_records.cpp test_device_type_table.cpp test_catalogue_index.cpp test_runtime_decoder.cpp test_device_type_grammar.cpp test_device_type_policy.cpp test_device_type_file.cpp test_poll_plan.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp): $(DEV_TYPE_GEN_HEADER) +$(call objs,test_device_type_records.cpp test_device_type_table.cpp test_catalogue_index.cpp test_runtime_decoder.cpp test_poll_data_decode.cpp test_device_type_grammar.cpp test_device_type_policy.cpp test_device_type_file.cpp test_poll_plan.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp): $(DEV_TYPE_GEN_HEADER) # The address map and scan lists the generator used to derive from the catalogue (for test_catalogue_index) $(CATALOGUE_INDEX_EXPECTED): $(DEV_TYPE_CATALOGUE) ../scripts/tests/reference_catalogue_index.py diff --git a/linux_unit_tests/README.md b/linux_unit_tests/README.md index ccb492a..8532e30 100644 --- a/linux_unit_tests/README.md +++ b/linux_unit_tests/README.md @@ -29,6 +29,7 @@ make -C linux_unit_tests check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tes | `test_device_names` | `DeviceNames`: device names and roles as used by DeviceManager | | `test_device_data_subscribers` | `DeviceDataSubscribers`: several subscribers per device, rate limits, unsubscribing while a callback runs | | `test_device_type_records` | Detection value parsing in `DeviceTypeRecords`, against hand-checked cases and every built-in device type record (generated from `devtypes/records/`, assembled into `gen/DeviceTypeRecords.json`) | +| `test_poll_data_decode` | Decoding poll data as on-device consumers get it (`DeviceTypeRecords::decodePollData`, `decodeNewestPollRecord`): for every built-in record, data with a last record cut short decodes as the zero-padded data does; a real LSM6DS3TR-C FIFO poll (shorter than its record) gives every sample, and the newest; and `AttrFieldDesc::getValue` reads every field, every element of array attributes, as its struct bytes | | `test_device_poll_timing` | `DevicePollTimingStats` (interval statistics, histogram, percentiles) and its recording at full poll starts in `DeviceStatus` | | `test_protocol_exchange` | `ProtocolExchange`: a real-time stream session from start to final block, the file stream activity hook, and refusal of a stream to an unknown endpoint | | `test_file_upload_okto` | `FileUploadOKTOProtocol`: when it acks - the first block, each batch, the last block, at once for the first block out of sequence (a lost block or one sent again), and on its timer - and that its acks stay in step with the host's batches after a loss or a timer ack | diff --git a/linux_unit_tests/test_poll_data_decode.cpp b/linux_unit_tests/test_poll_data_decode.cpp new file mode 100644 index 0000000..26247c1 --- /dev/null +++ b/linux_unit_tests/test_poll_data_decode.cpp @@ -0,0 +1,224 @@ +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// test_poll_data_decode +// +// Decoding poll data as on-device consumers get it (DeviceTypeRecords::decodePollData and +// decodeNewestPollRecord), and reading values from decoded records (AttrFieldDesc::getValue). +// +// A data subscriber and the bus's latest value get each poll result at its actual length. A poll with a variable +// read length (an LSM6DS FIFO read) is shorter than the record size the decoders read, so it must be padded as the +// poll store pads it. The decoders themselves are unchanged: test_runtime_decoder checks the compiled, runtime +// and interpreted decoders against each other, truncated buffers included. +// +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include "testhdr.h" +#include "DeviceTypeRecords.h" +#include "RaftJson.h" + +static uint8_t randomByte(std::mt19937& rng) +{ + static const uint8_t EDGES[] = {0x00, 0xff, 0x7f, 0x80, 0x01, 0xfe}; + return (rng() % 10 < 4) ? EDGES[rng() % sizeof(EDGES)] : (uint8_t)rng(); +} + +static const uint32_t TS_BYTES = DevicePollingInfo::POLL_RESULT_TIMESTAMP_SIZE; + +// Every compiled record: data with a short last record decodes as the same data padded with zeros (what the poll +// store holds) decodes with the record's decoder +static void testShortRecordsDecodeAsPadded() +{ + printf("Running testShortRecordsDecodeAsPadded...\n"); + DeviceTypeRecords table; + std::mt19937 rng(4321); + uint32_t numChecked = 0; + bool allSame = true; + for (uint32_t idx = 0; idx < DeviceTypeRecords::getNumBaseRecords(); idx++) + { + const DeviceTypeRecord* pRec = table.getRecord(idx); + if (!pRec || !pRec->pollResultDecodeFn || (pRec->pollStructSize == 0)) + continue; + const uint32_t recBytes = pRec->pollDataSizeBytes + TS_BYTES; + const uint32_t structSize = pRec->pollStructSize; + bool same = true; + for (int iter = 0; (iter < 500) && same; iter++) + { + // Whole records, then a last one cut short (or not) + uint32_t numWhole = rng() % 3; + uint32_t lastLen = (rng() % 4 == 0) ? recBytes : (rng() % (recBytes + 1)); + std::vector data(numWhole * recBytes + lastLen); + for (auto& b : data) + b = randomByte(rng); + // The padded reference: a last record shorter than the timestamp isn't a record + std::vector padded(data); + if (lastLen >= TS_BYTES) + padded.resize((numWhole + 1) * recBytes, 0); + else + padded.resize(numWhole * recBytes); + + const uint16_t maxRecs = 1 + rng() % 24; + // Zeroed as decodeNewestPollRecord's working space is: decoders leave a struct's padding bytes as they are + std::vector outA(structSize * maxRecs, 0), outB(structSize * maxRecs, 0); + RaftBusDeviceDecodeState stateA, stateB; + uint32_t nA = table.decodePollData(idx, data.data(), data.size(), outA.data(), outA.size(), maxRecs, stateA); + uint32_t nB = padded.empty() ? 0 : + pRec->pollResultDecodeFn(padded.data(), padded.size(), outB.data(), outB.size(), maxRecs, stateB); + same = (nA == nB) && (memcmp(outA.data(), outB.data(), nA * structSize) == 0) && + (stateA.lastReportTimestampUs == stateB.lastReportTimestampUs); + if (!same) + printf(" %s: %u records vs %u padded (iteration %d, length %u)\n", pRec->deviceType, nA, nB, iter, + (unsigned)data.size()); + + // The newest record is the last of them + std::vector newest(structSize, 0); + RaftBusDeviceDecodeState stateC; + uint32_t nC = table.decodeNewestPollRecord(idx, data.data(), data.size(), newest.data(), newest.size(), stateC); + if (nB > 0 && nB < maxRecs) + same = same && (nC == nB) && (memcmp(newest.data(), outB.data() + (nB - 1) * structSize, structSize) == 0); + } + TEST_ASSERT(same, (std::string(pRec->deviceType) + ": short records decode as padded").c_str()); + allSame = allSame && same; + numChecked++; + } + printf(" %u records checked\n", numChecked); + TEST_ASSERT(numChecked >= 55, "every compiled record checked"); + TEST_ASSERT(allSame, "all the same"); +} + +// A real LSM6DS3TR-C FIFO poll as a subscriber gets it: unpadded it decodes to nothing, through decodePollData it +// gives every sample, and decodeNewestPollRecord gives the newest +static void testLsm6dsFifoPoll() +{ + printf("Running testLsm6dsFifoPoll...\n"); + DeviceTypeRecords table; + DeviceTypeIndexType idx = 0; + const DeviceTypeRecord* pRec = table.getRecord("LSM6DS3TRC", idx); + TEST_ASSERT(pRec && pRec->pollResultDecodeFn, "LSM6DS3TRC compiled"); + if (!pRec || !pRec->pollResultDecodeFn) + return; + + // Poll: timestamp, OUT_TEMP (2), FIFO_STATUS1-4 (4), then 5 samples of 6 words (gx gy gz ax ay az) + const uint32_t NUM_SAMPLES = 5; + std::vector poll = {0x00, 0x64, // timestamp + 0x00, 0x02, // temperature: 512 / 256 + 25 = 27 degC + NUM_SAMPLES * 6, 0x00, // FIFO words + 0x00, 0x00}; // pattern 0 (aligned) + for (uint32_t i = 0; i < NUM_SAMPLES; i++) + { + const int16_t words[6] = {(int16_t)(100 * i), -50, 7, (int16_t)(-8192 + (int16_t)i), 0, 8192}; + for (int16_t w : words) + { + poll.push_back((uint8_t)(w & 0xff)); + poll.push_back((uint8_t)((uint16_t)w >> 8)); + } + } + TEST_ASSERT(poll.size() < pRec->pollDataSizeBytes + TS_BYTES, "the poll is shorter than the record size"); + + std::vector out(pRec->pollStructSize * 32, 0); + RaftBusDeviceDecodeState state1, state2, state3; + TEST_ASSERT(pRec->pollResultDecodeFn(poll.data(), poll.size(), out.data(), out.size(), 32, state1) == 0, + "the decoder alone decodes nothing from the unpadded poll"); + uint32_t n = table.decodePollData(idx, poll.data(), poll.size(), out.data(), out.size(), 32, state2); + TEST_ASSERT(n == NUM_SAMPLES, "decodePollData gives every sample"); + + // Field values from the newest sample + std::vector newest(pRec->pollStructSize, 0); + uint32_t nNewest = table.decodeNewestPollRecord(idx, poll.data(), poll.size(), newest.data(), newest.size(), state3); + TEST_ASSERT(nNewest == NUM_SAMPLES, "decodeNewestPollRecord counts the samples"); + auto value = [&](const char* name, const uint8_t* pRecord) { + for (uint32_t i = 0; i < pRec->pollFieldCount; i++) + { + double v = 0; + if ((strcmp(pRec->pollFieldDescs[i].name, name) == 0) && + pRec->pollFieldDescs[i].getValue(pRecord, pRec->pollStructSize, 0, v)) + return v; + } + return -9999.0; + }; + const double gyroLsb = 1.0 / 14.285714; + TEST_ASSERT(fabs(value("gx", newest.data()) - 400 * gyroLsb) < 1e-3, "newest gx (sample 4)"); + TEST_ASSERT(fabs(value("ax", newest.data()) - (-8188.0 / 8192)) < 1e-6, "newest ax"); + TEST_ASSERT(fabs(value("az", newest.data()) - 1.0) < 1e-6, "az 1 g"); + TEST_ASSERT(fabs(value("temp", newest.data()) - 27.0) < 1e-6, "temperature"); + TEST_ASSERT(fabs(value("gx", out.data()) - 0.0) < 1e-6, "first sample gx"); +} + +// Reading values: every compiled record's fields, every element of array attributes, against the struct bytes +static void testFieldValues() +{ + printf("Running testFieldValues...\n"); + DeviceTypeRecords table; + std::mt19937 rng(99); + uint32_t numArrays = 0, numFields = 0; + bool allRight = true; + for (uint32_t idx = 0; idx < DeviceTypeRecords::getNumBaseRecords(); idx++) + { + const DeviceTypeRecord* pRec = table.getRecord(idx); + if (!pRec || !pRec->pollFieldDescs || (pRec->pollStructSize == 0)) + continue; + std::vector rec(pRec->pollStructSize); + for (auto& b : rec) + b = (uint8_t)rng(); + for (uint32_t f = 0; f < pRec->pollFieldCount; f++) + { + const AttrFieldDesc& desc = pRec->pollFieldDescs[f]; + const uint32_t elemBytes = desc.elemBytes(); + for (uint16_t e = 0; e < desc.elemCount(); e++) + { + // The reference: the element's bytes as the field's type, then the divisor and addend + const uint8_t* p = rec.data() + desc.offset + e * elemBytes; + double ref = 0; + switch (desc.type) + { + case AttrType::Float: { float v; memcpy(&v, p, 4); ref = v; break; } + case AttrType::Int32: { int32_t v; memcpy(&v, p, 4); ref = v; break; } + case AttrType::Uint32: { uint32_t v; memcpy(&v, p, 4); ref = v; break; } + case AttrType::Int16: { int16_t v; memcpy(&v, p, 2); ref = v; break; } + case AttrType::Uint16: { uint16_t v; memcpy(&v, p, 2); ref = v; break; } + case AttrType::Int8: ref = (int8_t)*p; break; + case AttrType::Uint8: ref = *p; break; + case AttrType::Bool: ref = *p ? 1 : 0; break; + } + if ((desc.divisor != 0.0f) && (desc.divisor != 1.0f)) + ref /= desc.divisor; + ref += desc.addend; + double v = 0; + bool ok = desc.getValue(rec.data(), rec.size(), e, v); + bool right = ok && ((v == ref) || (std::isnan(v) && std::isnan(ref))); + if (!right) + printf(" %s %s[%u]: %g, expected %g\n", pRec->deviceType, desc.name, e, v, ref); + allRight = allRight && right; + numFields++; + } + if (desc.elemCount() > 1) + numArrays++; + double v = 0; + TEST_ASSERT(!desc.getValue(rec.data(), rec.size(), desc.elemCount(), v), "an element past the end is refused"); + TEST_ASSERT(!desc.getValue(rec.data(), desc.offset, 0, v), "a field past the record's end is refused"); + } + } + printf(" %u values read, %u array attributes\n", numFields, numArrays); + TEST_ASSERT(allRight, "every value as its struct bytes"); + TEST_ASSERT(numArrays >= 5, "array attributes read element by element"); + + // Integral: an integer with nothing to scale + AttrFieldDesc d = {"x", 0, AttrType::Uint16, "", 1.0f, 0.0f, 1}; + TEST_ASSERT(d.isIntegral(), "uint16 integral"); + d.divisor = 8192.0f; + TEST_ASSERT(!d.isIntegral(), "scaled not integral"); + d = {"x", 0, AttrType::Float, "", 1.0f, 0.0f, 1}; + TEST_ASSERT(!d.isIntegral(), "float not integral"); +} + +int main() +{ + testShortRecordsDecodeAsPadded(); + testLsm6dsFifoPoll(); + testFieldValues(); + return testSummary("test_poll_data_decode"); +} From 3b6a4ae40a03770ce4be5cfba041e4c365aa1f8e Mon Sep 17 00:00:00 2001 From: Rob Dobson Date: Mon, 9 Nov 2026 18:09:51 +0100 Subject: [PATCH 2/2] Named values: the newest record of the latest poll, through the shared value reader; name[i] for array elements getBusDeviceNamedValue reads the value with AttrFieldDesc::getValue, and an array attribute's element is named with an index (e.g. "diff[3]"). The latest decoded poll response is the newest record of a poll that holds several (the bus implementation decodes it with decodeNewestPollRecord). Co-Authored-By: Claude Opus 5.5 --- components/core/Bus/RaftBusDevicesIF.h | 5 +- .../core/DeviceManager/DeviceManager.cpp | 49 ++++++++----------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/components/core/Bus/RaftBusDevicesIF.h b/components/core/Bus/RaftBusDevicesIF.h index b01f2da..efc36ce 100644 --- a/components/core/Bus/RaftBusDevicesIF.h +++ b/components/core/Bus/RaftBusDevicesIF.h @@ -140,7 +140,8 @@ class RaftBusDevicesIF uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState) const = 0; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// @brief Get the latest decoded poll response if one has arrived since the previous call + /// @brief Get the latest decoded poll response (its newest record, for a poll holding several) if one has + /// arrived since the previous call /// (from any caller). Does not drain the stored results, but each value is reported once, /// so two readers of the same device see it only between them - use /// peekLatestDecodedPollResponse() for the current value regardless. @@ -157,7 +158,7 @@ class RaftBusDevicesIF } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// @brief Get the most recent decoded poll response whether or not it has been read before + /// @brief Get the most recent decoded poll response (its newest record) whether or not it has been read before /// (a true peek: it leaves what getLatestDecodedPollResponse() reports untouched) /// @param address address of device to get data from /// @param pStructOut pointer to structure to receive decoded data diff --git a/components/core/DeviceManager/DeviceManager.cpp b/components/core/DeviceManager/DeviceManager.cpp index e55bc07..0008c3f 100644 --- a/components/core/DeviceManager/DeviceManager.cpp +++ b/components/core/DeviceManager/DeviceManager.cpp @@ -2900,7 +2900,7 @@ bool DeviceManager::resolveDeviceNameToID(const String& name, RaftDeviceID& devi ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// @brief Get a named value from a bus-discovered device using its cached poll data /// @param deviceID Device identifier on the bus -/// @param paramName Attribute name (e.g. "ax", "temperature") +/// @param paramName Attribute name (e.g. "ax", "temperature"), or an array attribute's element ("diffs[3]") /// @param isValid (out) true if value was found and valid /// @return attribute value (with divisor/addend applied) double DeviceManager::getBusDeviceNamedValue(RaftDeviceID deviceID, const String& paramName, bool& isValid) @@ -2927,17 +2927,25 @@ double DeviceManager::getBusDeviceNamedValue(RaftDeviceID deviceID, const String if (!devTypeRec.pollFieldDescs || devTypeRec.pollFieldCount == 0 || devTypeRec.pollStructSize == 0) return 0.0; - // Find the matching field by name + // Find the matching field by name, with an element index for an array attribute ("diffs[3]") + String fieldName = paramName; + uint16_t elemIdx = 0; + int bracketPos = paramName.indexOf('['); + if ((bracketPos > 0) && paramName.endsWith("]")) + { + fieldName = paramName.substring(0, bracketPos); + elemIdx = (uint16_t)strtoul(paramName.substring(bracketPos + 1, paramName.length() - 1).c_str(), nullptr, 10); + } const AttrFieldDesc* pMatchedField = nullptr; for (uint16_t i = 0; i < devTypeRec.pollFieldCount; i++) { - if (paramName.equalsIgnoreCase(devTypeRec.pollFieldDescs[i].name)) + if (fieldName.equalsIgnoreCase(devTypeRec.pollFieldDescs[i].name)) { pMatchedField = &devTypeRec.pollFieldDescs[i]; break; } } - if (!pMatchedField) + if (!pMatchedField || (elemIdx >= pMatchedField->elemCount())) return 0.0; // Get decoded poll data from the bus @@ -2952,36 +2960,19 @@ double DeviceManager::getBusDeviceNamedValue(RaftDeviceID deviceID, const String uint8_t decodedBuf[devTypeRec.pollStructSize]; memset(decodedBuf, 0, devTypeRec.pollStructSize); - // Peek the latest value: neither drains the ring buffer (which would starve the publish - // pipeline) nor takes the "new" state another reader of getLatestDecodedPollResponse() uses, - // and gives the current value between polls rather than nothing + // Peek the latest value (the newest record of the latest poll): neither drains the ring buffer (which would + // starve the publish pipeline) nor takes the "new" state another reader of getLatestDecodedPollResponse() + // uses, and gives the current value between polls rather than nothing RaftBusDeviceDecodeState decodeState; if (!pDevicesIF->peekLatestDecodedPollResponse(addr, decodedBuf, devTypeRec.pollStructSize, decodeState)) return 0.0; - // Read the value from the decoded buffer at the field's offset - double rawVal = 0.0; - const uint8_t* pField = decodedBuf + pMatchedField->offset; - switch (pMatchedField->type) - { - case AttrType::Float: { float v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Int32: { int32_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Uint32: { uint32_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Int16: { int16_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Uint16: { uint16_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Int8: { int8_t v = *reinterpret_cast(pField); rawVal = v; break; } - case AttrType::Uint8: { rawVal = *pField; break; } - case AttrType::Bool: { rawVal = (*pField) ? 1.0 : 0.0; break; } - } - - // Apply what is still to be applied - the field table's divisor and addend are 1 and 0 - // where the decode has already scaled the value (see DecodeGenerator get_field_desc_def) - if (pMatchedField->divisor != 0.0f && pMatchedField->divisor != 1.0f) - rawVal /= pMatchedField->divisor; - rawVal += pMatchedField->addend; - + // The value, as every on-device consumer reads it + double value = 0.0; + if (!pMatchedField->getValue(decodedBuf, devTypeRec.pollStructSize, elemIdx, value)) + return 0.0; isValid = true; - return rawVal; + return value; } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////