diff --git a/components/core/Bus/RaftBusDevicesIF.h b/components/core/Bus/RaftBusDevicesIF.h index b01f2da..efc36ce 100644 --- a/components/core/Bus/RaftBusDevicesIF.h +++ b/components/core/Bus/RaftBusDevicesIF.h @@ -140,7 +140,8 @@ class RaftBusDevicesIF uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState) const = 0; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// @brief Get the latest decoded poll response if one has arrived since the previous call + /// @brief Get the latest decoded poll response (its newest record, for a poll holding several) if one has + /// arrived since the previous call /// (from any caller). Does not drain the stored results, but each value is reported once, /// so two readers of the same device see it only between them - use /// peekLatestDecodedPollResponse() for the current value regardless. @@ -157,7 +158,7 @@ class RaftBusDevicesIF } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// @brief Get the most recent decoded poll response whether or not it has been read before + /// @brief Get the most recent decoded poll response (its newest record) whether or not it has been read before /// (a true peek: it leaves what getLatestDecodedPollResponse() reports untouched) /// @param address address of device to get data from /// @param pStructOut pointer to structure to receive decoded data diff --git a/components/core/DeviceManager/DeviceManager.cpp b/components/core/DeviceManager/DeviceManager.cpp index e55bc07..0008c3f 100644 --- a/components/core/DeviceManager/DeviceManager.cpp +++ b/components/core/DeviceManager/DeviceManager.cpp @@ -2900,7 +2900,7 @@ bool DeviceManager::resolveDeviceNameToID(const String& name, RaftDeviceID& devi ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// @brief Get a named value from a bus-discovered device using its cached poll data /// @param deviceID Device identifier on the bus -/// @param paramName Attribute name (e.g. "ax", "temperature") +/// @param paramName Attribute name (e.g. "ax", "temperature"), or an array attribute's element ("diffs[3]") /// @param isValid (out) true if value was found and valid /// @return attribute value (with divisor/addend applied) double DeviceManager::getBusDeviceNamedValue(RaftDeviceID deviceID, const String& paramName, bool& isValid) @@ -2927,17 +2927,25 @@ double DeviceManager::getBusDeviceNamedValue(RaftDeviceID deviceID, const String if (!devTypeRec.pollFieldDescs || devTypeRec.pollFieldCount == 0 || devTypeRec.pollStructSize == 0) return 0.0; - // Find the matching field by name + // Find the matching field by name, with an element index for an array attribute ("diffs[3]") + String fieldName = paramName; + uint16_t elemIdx = 0; + int bracketPos = paramName.indexOf('['); + if ((bracketPos > 0) && paramName.endsWith("]")) + { + fieldName = paramName.substring(0, bracketPos); + elemIdx = (uint16_t)strtoul(paramName.substring(bracketPos + 1, paramName.length() - 1).c_str(), nullptr, 10); + } const AttrFieldDesc* pMatchedField = nullptr; for (uint16_t i = 0; i < devTypeRec.pollFieldCount; i++) { - if (paramName.equalsIgnoreCase(devTypeRec.pollFieldDescs[i].name)) + if (fieldName.equalsIgnoreCase(devTypeRec.pollFieldDescs[i].name)) { pMatchedField = &devTypeRec.pollFieldDescs[i]; break; } } - if (!pMatchedField) + if (!pMatchedField || (elemIdx >= pMatchedField->elemCount())) return 0.0; // Get decoded poll data from the bus @@ -2952,36 +2960,19 @@ double DeviceManager::getBusDeviceNamedValue(RaftDeviceID deviceID, const String uint8_t decodedBuf[devTypeRec.pollStructSize]; memset(decodedBuf, 0, devTypeRec.pollStructSize); - // Peek the latest value: neither drains the ring buffer (which would starve the publish - // pipeline) nor takes the "new" state another reader of getLatestDecodedPollResponse() uses, - // and gives the current value between polls rather than nothing + // Peek the latest value (the newest record of the latest poll): neither drains the ring buffer (which would + // starve the publish pipeline) nor takes the "new" state another reader of getLatestDecodedPollResponse() + // uses, and gives the current value between polls rather than nothing RaftBusDeviceDecodeState decodeState; if (!pDevicesIF->peekLatestDecodedPollResponse(addr, decodedBuf, devTypeRec.pollStructSize, decodeState)) return 0.0; - // Read the value from the decoded buffer at the field's offset - double rawVal = 0.0; - const uint8_t* pField = decodedBuf + pMatchedField->offset; - switch (pMatchedField->type) - { - case AttrType::Float: { float v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Int32: { int32_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Uint32: { uint32_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Int16: { int16_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Uint16: { uint16_t v; memcpy(&v, pField, sizeof(v)); rawVal = v; break; } - case AttrType::Int8: { int8_t v = *reinterpret_cast(pField); rawVal = v; break; } - case AttrType::Uint8: { rawVal = *pField; break; } - case AttrType::Bool: { rawVal = (*pField) ? 1.0 : 0.0; break; } - } - - // Apply what is still to be applied - the field table's divisor and addend are 1 and 0 - // where the decode has already scaled the value (see DecodeGenerator get_field_desc_def) - if (pMatchedField->divisor != 0.0f && pMatchedField->divisor != 1.0f) - rawVal /= pMatchedField->divisor; - rawVal += pMatchedField->addend; - + // The value, as every on-device consumer reads it + double value = 0.0; + if (!pMatchedField->getValue(decodedBuf, devTypeRec.pollStructSize, elemIdx, value)) + return 0.0; isValid = true; - return rawVal; + return value; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/components/core/DeviceTypes/DeviceTypeRecord.h b/components/core/DeviceTypes/DeviceTypeRecord.h index 5b0e0d7..726898a 100644 --- a/components/core/DeviceTypes/DeviceTypeRecord.h +++ b/components/core/DeviceTypes/DeviceTypeRecord.h @@ -53,6 +53,64 @@ struct AttrFieldDesc float divisor; // Divisor from resp.a[].d (1.0 = no division) float addend; // Addend from resp.a[].a (applied after division) uint16_t count; // Elements in the field (>1 for array attributes; 0 treated as 1) + + // Reading a value from a decoded record. Every on-device consumer (DeviceManager named values, RaftMicroPy, + // RaftROS) uses these so that they all give the same value for the same field + + /// @brief Number of elements (1 for a scalar attribute) + uint16_t elemCount() const { return count ? count : 1; } + + /// @brief Bytes per element in the decoded struct + /// @note the type is the struct field's C type except for the few wider fields: int64 and uint64 are described + /// as Int32 and Uint32 and double as Float (no RaftCore record has a double; BLEBTHome has a uint64). + /// Such a field is read as its low 4 bytes, an element of an array of them at the wrong stride + uint32_t elemBytes() const + { + switch (type) + { + case AttrType::Float: case AttrType::Int32: case AttrType::Uint32: return 4; + case AttrType::Int16: case AttrType::Uint16: return 2; + default: return 1; + } + } + + /// @brief True if the value is a whole number: an integer field with nothing still to scale + bool isIntegral() const + { + return (type != AttrType::Float) && ((divisor == 0.0f) || (divisor == 1.0f)) && (addend == 0.0f); + } + + /// @brief Get one element's value from a decoded record (one poll struct) + /// @param pRecord the decoded record + /// @param recordSize the record's size (the record type's pollStructSize) + /// @param elemIdx element index (0 for a scalar attribute) + /// @param value (out) the value, with the field's divisor and addend applied - what is still to be applied + /// (1 and 0 where the decode has scaled the value already, see DecodeGenerator get_field_desc_def) + /// @return false if the element is outside the record + bool getValue(const void* pRecord, uint32_t recordSize, uint16_t elemIdx, double& value) const + { + value = 0.0; + uint32_t bytes = elemBytes(); + uint32_t pos = offset + (uint32_t)elemIdx * bytes; + if (!pRecord || (elemIdx >= elemCount()) || (pos + bytes > recordSize)) + return false; + const uint8_t* p = static_cast(pRecord) + pos; + switch (type) + { + case AttrType::Float: { float v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Int32: { int32_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Uint32: { uint32_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Int16: { int16_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Uint16: { uint16_t v; memcpy(&v, p, sizeof(v)); value = v; break; } + case AttrType::Int8: { value = (int8_t)*p; break; } + case AttrType::Uint8: { value = *p; break; } + case AttrType::Bool: { value = (*p != 0) ? 1.0 : 0.0; break; } + } + if ((divisor != 0.0f) && (divisor != 1.0f)) + value /= divisor; + value += addend; + return true; + } }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/components/core/DeviceTypes/DeviceTypeRecords.cpp b/components/core/DeviceTypes/DeviceTypeRecords.cpp index 95894d5..72db4e8 100644 --- a/components/core/DeviceTypes/DeviceTypeRecords.cpp +++ b/components/core/DeviceTypes/DeviceTypeRecords.cpp @@ -1494,6 +1494,92 @@ DeviceTypeRecordDecodeFn DeviceTypeRecords::getPollDecodeFn(DeviceTypeIndexType return pRec ? __atomic_load_n(&pRec->pollResultDecodeFn, __ATOMIC_ACQUIRE) : nullptr; } +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data of a device type, padding a record cut short (see the header) +uint32_t DeviceTypeRecords::decodePollData(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, uint32_t dataLen, + void* pStructOut, uint32_t structOutSize, uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState) const +{ + // The decode function is published after the fields it uses (release), so a record whose function is seen + // has its sizes set + DeviceTypeRecordDecodeFn decodeFn = getPollDecodeFn(deviceTypeIndex); + const DeviceTypeRecord* pRec = getRecord(deviceTypeIndex); + if (!decodeFn || !pRec) + return 0; + return decodePollData(decodeFn, pRec->pollDataSizeBytes, pData, dataLen, pStructOut, structOutSize, + maxRecCount, decodeState); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data of a device type and keep only the newest record (see the header) +uint32_t DeviceTypeRecords::decodeNewestPollRecord(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, + uint32_t dataLen, void* pStructOut, uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState) const +{ + DeviceTypeRecordDecodeFn decodeFn = getPollDecodeFn(deviceTypeIndex); + const DeviceTypeRecord* pRec = getRecord(deviceTypeIndex); + if (!decodeFn || !pRec) + return 0; + return decodeNewestPollRecord(decodeFn, pRec->pollDataSizeBytes, pRec->pollStructSize, pData, dataLen, + pStructOut, structOutSize, decodeState); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data with a given decode function, padding a record cut short (see the header) +uint32_t DeviceTypeRecords::decodePollData(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + const uint8_t* pData, uint32_t dataLen, void* pStructOut, uint32_t structOutSize, + uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState) +{ + if (!decodeFn || !pData || !pStructOut || (dataLen == 0) || (maxRecCount == 0)) + return 0; + + // Whole records are decoded as they are + const uint32_t recBytes = pollDataSizeBytes + DevicePollingInfo::POLL_RESULT_TIMESTAMP_SIZE; + const uint32_t partBytes = dataLen % recBytes; + if (partBytes == 0) + return decodeFn(pData, dataLen, pStructOut, structOutSize, maxRecCount, decodeState); + + // A last record without even its timestamp isn't a record + if (partBytes < DevicePollingInfo::POLL_RESULT_TIMESTAMP_SIZE) + return (dataLen > partBytes) ? + decodeFn(pData, dataLen - partBytes, pStructOut, structOutSize, maxRecCount, decodeState) : 0; + + // Pad the last record with zeros to the record size, as the poll store does + std::vector padded(pData, pData + dataLen); + padded.resize(dataLen - partBytes + recBytes, 0); + return decodeFn(padded.data(), padded.size(), pStructOut, structOutSize, maxRecCount, decodeState); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/// @brief Decode poll data with a given decode function and keep only the newest record (see the header) +uint32_t DeviceTypeRecords::decodeNewestPollRecord(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + uint32_t pollStructSize, const uint8_t* pData, uint32_t dataLen, void* pStructOut, + uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState, uint8_t* pScratch, uint32_t scratchSize) +{ + if (!decodeFn || (pollStructSize == 0) || !pStructOut || (structOutSize < pollStructSize)) + return 0; + + // Working space for every record the data holds + std::vector allocated; + if (!pScratch) + { + uint32_t maxRecs = std::min(NEWEST_DECODE_MAX_RECORDS, + std::max(1, NEWEST_DECODE_MAX_BYTES / pollStructSize)); + allocated.resize(maxRecs * pollStructSize); + pScratch = allocated.data(); + scratchSize = allocated.size(); + } + uint32_t maxRecs = std::min(scratchSize / pollStructSize, UINT16_MAX); + if (maxRecs == 0) + return 0; + + // Decode, then keep the newest + uint32_t numRecs = decodePollData(decodeFn, pollDataSizeBytes, pData, dataLen, pScratch, + maxRecs * pollStructSize, (uint16_t)maxRecs, decodeState); + if ((numRecs == 0) || (numRecs > maxRecs)) + return 0; + memcpy(pStructOut, pScratch + (numRecs - 1) * pollStructSize, pollStructSize); + return numRecs; +} + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// @brief Prepare the runtime decoder of a record just added (not yet published) with no compiled decoder: /// with policy OnIdentify, check that it can be decoded from its attribute list and mark it Pending (its diff --git a/components/core/DeviceTypes/DeviceTypeRecords.h b/components/core/DeviceTypes/DeviceTypeRecords.h index 753956a..fc50b01 100644 --- a/components/core/DeviceTypes/DeviceTypeRecords.h +++ b/components/core/DeviceTypes/DeviceTypeRecords.h @@ -480,6 +480,56 @@ class DeviceTypeRecords /// @return poll decode function DeviceTypeRecordDecodeFn getPollDecodeFn(uint16_t deviceTypeIndex) const; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Decoding poll data. Decoders read whole records of the record type's size (resp.b plus the timestamp), as + // the bus's poll store keeps them. But a data subscriber (DeviceManager::registerForDeviceData) and the bus's + // latest value get each poll result at its actual length, which is shorter for a poll with a variable read + // length (e.g. a FIFO read, r{...}): given to a decoder as it is, it decodes to nothing. These functions pad a + // record cut short with zeros, as the poll store does. Every on-device consumer decodes through them + + /// @brief Decode poll data of a device type + /// @param deviceTypeIndex device type index + /// @param pData poll data: one or more poll results, the last of which may be short + /// @param dataLen length of the data + /// @param pStructOut (out) decoded records (the record type's poll struct) + /// @param structOutSize size of pStructOut in bytes + /// @param maxRecCount maximum number of records to decode (a poll result can hold several, e.g. a FIFO read) + /// @param decodeState decode state, kept between calls for the same device + /// @return number of records decoded (0 if the type has no decoder) + uint32_t decodePollData(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, uint32_t dataLen, + void* pStructOut, uint32_t structOutSize, uint16_t maxRecCount, + RaftBusDeviceDecodeState& decodeState) const; + + /// @brief Decode poll data and keep only the newest record + /// @param pStructOut (out) the newest record (one poll struct) + /// @param structOutSize size of pStructOut in bytes (at least the record type's pollStructSize) + /// @return number of records the data held (0 if none, or the type has no decoder); pStructOut has the newest + /// @note a poll result can hold up to NEWEST_DECODE_MAX_RECORDS records; beyond that the newest are dropped + uint32_t decodeNewestPollRecord(DeviceTypeIndexType deviceTypeIndex, const uint8_t* pData, uint32_t dataLen, + void* pStructOut, uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState) const; + + /// @brief Decode poll data with a given decode function (for a consumer that keeps a record's decoder) + /// @param decodeFn the record type's decode function + /// @param pollDataSizeBytes the record type's pollDataSizeBytes (resp.b) + static uint32_t decodePollData(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + const uint8_t* pData, uint32_t dataLen, void* pStructOut, uint32_t structOutSize, + uint16_t maxRecCount, RaftBusDeviceDecodeState& decodeState); + + /// @brief Decode poll data with a given decode function and keep only the newest record + /// @param pollStructSize the record type's pollStructSize + /// @param pScratch working space for the records decoded (nullptr to allocate it); a consumer on the bus task + /// passes its own so that nothing is allocated there + /// @param scratchSize size of pScratch in bytes (records beyond what fits are dropped, the newest first) + static uint32_t decodeNewestPollRecord(DeviceTypeRecordDecodeFn decodeFn, uint32_t pollDataSizeBytes, + uint32_t pollStructSize, const uint8_t* pData, uint32_t dataLen, void* pStructOut, + uint32_t structOutSize, RaftBusDeviceDecodeState& decodeState, + uint8_t* pScratch = nullptr, uint32_t scratchSize = 0); + + // The working space decodeNewestPollRecord allocates: up to NEWEST_DECODE_MAX_RECORDS records (an LSM6DS FIFO + // read holds up to 20), within NEWEST_DECODE_MAX_BYTES + static constexpr uint32_t NEWEST_DECODE_MAX_RECORDS = 32; + static constexpr uint32_t NEWEST_DECODE_MAX_BYTES = 4096; + // Where a record's poll decoder comes from: compiled into the firmware (generated, or inherited by an override // with the same response layout), made at runtime from its attribute list (RAFT_RUNTIME_DECODE), or none enum class DecoderKind : uint8_t diff --git a/linux_unit_tests/Makefile b/linux_unit_tests/Makefile index d652549..3d42f0a 100644 --- a/linux_unit_tests/Makefile +++ b/linux_unit_tests/Makefile @@ -110,6 +110,7 @@ SOURCES_test_device_type_policy = test_device_type_policy.cpp $(CORE)/DeviceType SOURCES_test_device_type_file = test_device_type_file.cpp $(CORE)/DeviceTypes/DeviceTypeFileLoader.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) SOURCES_test_catalogue_index = test_catalogue_index.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(COMMON_SOURCES) SOURCES_test_runtime_decoder = test_runtime_decoder.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(CORE)/DeviceTypes/DeviceTypeFileLoader.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) +SOURCES_test_poll_data_decode = test_poll_data_decode.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp $(CORE)/DeviceTypes/DeviceTypeGrammar.cpp $(CORE)/DeviceTypes/DeviceTypeRuntimeDecoder.cpp $(CORE)/DeviceTypes/DeviceTypeCodeInterpreter.cpp $(CORE)/DeviceTypes/DeviceTypeDecodeVerifier.cpp $(CORE)/DeviceTypes/DeviceTypeFileLoader.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) SOURCES_test_protocol_exchange = test_protocol_exchange.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_file_upload_okto = test_file_upload_okto.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_file_upload_background = test_file_upload_background.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) @@ -124,7 +125,7 @@ SOURCES_test_espnow_config = test_espnow_config.cpp $(ESPNOW_DIR)/ESPNowConfig.c # Tests run by "make check" TESTS = test_raft_json test_raft_utils test_device_names test_device_settings test_device_type_settings test_device_data_subscribers test_device_type_records \ - test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder + test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_poll_data_decode test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder # Extra device type catalogues for test_device_type_records to sweep, e.g. # make check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tests/TestDevTypeRecs.json" @@ -208,7 +209,7 @@ $(DEV_TYPE_GEN_HEADER): $(DEV_TYPE_CATALOGUE) ../scripts/ProcessDevTypeJsonToC.p python3 ../scripts/ProcessDevTypeJsonToC.py $(DEV_TYPE_CATALOGUE) $(DEV_TYPE_GEN_HEADER) \ $(DEV_TYPE_GEN_DIR)/DevicePollRecords_generated.h > $(DEV_TYPE_GEN_DIR)/generator.log -$(call objs,test_device_type_records.cpp test_device_type_table.cpp test_catalogue_index.cpp test_runtime_decoder.cpp test_device_type_grammar.cpp test_device_type_policy.cpp test_device_type_file.cpp test_poll_plan.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp): $(DEV_TYPE_GEN_HEADER) +$(call objs,test_device_type_records.cpp test_device_type_table.cpp test_catalogue_index.cpp test_runtime_decoder.cpp test_poll_data_decode.cpp test_device_type_grammar.cpp test_device_type_policy.cpp test_device_type_file.cpp test_poll_plan.cpp $(CORE)/DeviceTypes/DeviceTypeRecords.cpp): $(DEV_TYPE_GEN_HEADER) # The address map and scan lists the generator used to derive from the catalogue (for test_catalogue_index) $(CATALOGUE_INDEX_EXPECTED): $(DEV_TYPE_CATALOGUE) ../scripts/tests/reference_catalogue_index.py diff --git a/linux_unit_tests/README.md b/linux_unit_tests/README.md index ccb492a..8532e30 100644 --- a/linux_unit_tests/README.md +++ b/linux_unit_tests/README.md @@ -29,6 +29,7 @@ make -C linux_unit_tests check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tes | `test_device_names` | `DeviceNames`: device names and roles as used by DeviceManager | | `test_device_data_subscribers` | `DeviceDataSubscribers`: several subscribers per device, rate limits, unsubscribing while a callback runs | | `test_device_type_records` | Detection value parsing in `DeviceTypeRecords`, against hand-checked cases and every built-in device type record (generated from `devtypes/records/`, assembled into `gen/DeviceTypeRecords.json`) | +| `test_poll_data_decode` | Decoding poll data as on-device consumers get it (`DeviceTypeRecords::decodePollData`, `decodeNewestPollRecord`): for every built-in record, data with a last record cut short decodes as the zero-padded data does; a real LSM6DS3TR-C FIFO poll (shorter than its record) gives every sample, and the newest; and `AttrFieldDesc::getValue` reads every field, every element of array attributes, as its struct bytes | | `test_device_poll_timing` | `DevicePollTimingStats` (interval statistics, histogram, percentiles) and its recording at full poll starts in `DeviceStatus` | | `test_protocol_exchange` | `ProtocolExchange`: a real-time stream session from start to final block, the file stream activity hook, and refusal of a stream to an unknown endpoint | | `test_file_upload_okto` | `FileUploadOKTOProtocol`: when it acks - the first block, each batch, the last block, at once for the first block out of sequence (a lost block or one sent again), and on its timer - and that its acks stay in step with the host's batches after a loss or a timer ack | diff --git a/linux_unit_tests/test_poll_data_decode.cpp b/linux_unit_tests/test_poll_data_decode.cpp new file mode 100644 index 0000000..26247c1 --- /dev/null +++ b/linux_unit_tests/test_poll_data_decode.cpp @@ -0,0 +1,224 @@ +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// test_poll_data_decode +// +// Decoding poll data as on-device consumers get it (DeviceTypeRecords::decodePollData and +// decodeNewestPollRecord), and reading values from decoded records (AttrFieldDesc::getValue). +// +// A data subscriber and the bus's latest value get each poll result at its actual length. A poll with a variable +// read length (an LSM6DS FIFO read) is shorter than the record size the decoders read, so it must be padded as the +// poll store pads it. The decoders themselves are unchanged: test_runtime_decoder checks the compiled, runtime +// and interpreted decoders against each other, truncated buffers included. +// +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include "testhdr.h" +#include "DeviceTypeRecords.h" +#include "RaftJson.h" + +static uint8_t randomByte(std::mt19937& rng) +{ + static const uint8_t EDGES[] = {0x00, 0xff, 0x7f, 0x80, 0x01, 0xfe}; + return (rng() % 10 < 4) ? EDGES[rng() % sizeof(EDGES)] : (uint8_t)rng(); +} + +static const uint32_t TS_BYTES = DevicePollingInfo::POLL_RESULT_TIMESTAMP_SIZE; + +// Every compiled record: data with a short last record decodes as the same data padded with zeros (what the poll +// store holds) decodes with the record's decoder +static void testShortRecordsDecodeAsPadded() +{ + printf("Running testShortRecordsDecodeAsPadded...\n"); + DeviceTypeRecords table; + std::mt19937 rng(4321); + uint32_t numChecked = 0; + bool allSame = true; + for (uint32_t idx = 0; idx < DeviceTypeRecords::getNumBaseRecords(); idx++) + { + const DeviceTypeRecord* pRec = table.getRecord(idx); + if (!pRec || !pRec->pollResultDecodeFn || (pRec->pollStructSize == 0)) + continue; + const uint32_t recBytes = pRec->pollDataSizeBytes + TS_BYTES; + const uint32_t structSize = pRec->pollStructSize; + bool same = true; + for (int iter = 0; (iter < 500) && same; iter++) + { + // Whole records, then a last one cut short (or not) + uint32_t numWhole = rng() % 3; + uint32_t lastLen = (rng() % 4 == 0) ? recBytes : (rng() % (recBytes + 1)); + std::vector data(numWhole * recBytes + lastLen); + for (auto& b : data) + b = randomByte(rng); + // The padded reference: a last record shorter than the timestamp isn't a record + std::vector padded(data); + if (lastLen >= TS_BYTES) + padded.resize((numWhole + 1) * recBytes, 0); + else + padded.resize(numWhole * recBytes); + + const uint16_t maxRecs = 1 + rng() % 24; + // Zeroed as decodeNewestPollRecord's working space is: decoders leave a struct's padding bytes as they are + std::vector outA(structSize * maxRecs, 0), outB(structSize * maxRecs, 0); + RaftBusDeviceDecodeState stateA, stateB; + uint32_t nA = table.decodePollData(idx, data.data(), data.size(), outA.data(), outA.size(), maxRecs, stateA); + uint32_t nB = padded.empty() ? 0 : + pRec->pollResultDecodeFn(padded.data(), padded.size(), outB.data(), outB.size(), maxRecs, stateB); + same = (nA == nB) && (memcmp(outA.data(), outB.data(), nA * structSize) == 0) && + (stateA.lastReportTimestampUs == stateB.lastReportTimestampUs); + if (!same) + printf(" %s: %u records vs %u padded (iteration %d, length %u)\n", pRec->deviceType, nA, nB, iter, + (unsigned)data.size()); + + // The newest record is the last of them + std::vector newest(structSize, 0); + RaftBusDeviceDecodeState stateC; + uint32_t nC = table.decodeNewestPollRecord(idx, data.data(), data.size(), newest.data(), newest.size(), stateC); + if (nB > 0 && nB < maxRecs) + same = same && (nC == nB) && (memcmp(newest.data(), outB.data() + (nB - 1) * structSize, structSize) == 0); + } + TEST_ASSERT(same, (std::string(pRec->deviceType) + ": short records decode as padded").c_str()); + allSame = allSame && same; + numChecked++; + } + printf(" %u records checked\n", numChecked); + TEST_ASSERT(numChecked >= 55, "every compiled record checked"); + TEST_ASSERT(allSame, "all the same"); +} + +// A real LSM6DS3TR-C FIFO poll as a subscriber gets it: unpadded it decodes to nothing, through decodePollData it +// gives every sample, and decodeNewestPollRecord gives the newest +static void testLsm6dsFifoPoll() +{ + printf("Running testLsm6dsFifoPoll...\n"); + DeviceTypeRecords table; + DeviceTypeIndexType idx = 0; + const DeviceTypeRecord* pRec = table.getRecord("LSM6DS3TRC", idx); + TEST_ASSERT(pRec && pRec->pollResultDecodeFn, "LSM6DS3TRC compiled"); + if (!pRec || !pRec->pollResultDecodeFn) + return; + + // Poll: timestamp, OUT_TEMP (2), FIFO_STATUS1-4 (4), then 5 samples of 6 words (gx gy gz ax ay az) + const uint32_t NUM_SAMPLES = 5; + std::vector poll = {0x00, 0x64, // timestamp + 0x00, 0x02, // temperature: 512 / 256 + 25 = 27 degC + NUM_SAMPLES * 6, 0x00, // FIFO words + 0x00, 0x00}; // pattern 0 (aligned) + for (uint32_t i = 0; i < NUM_SAMPLES; i++) + { + const int16_t words[6] = {(int16_t)(100 * i), -50, 7, (int16_t)(-8192 + (int16_t)i), 0, 8192}; + for (int16_t w : words) + { + poll.push_back((uint8_t)(w & 0xff)); + poll.push_back((uint8_t)((uint16_t)w >> 8)); + } + } + TEST_ASSERT(poll.size() < pRec->pollDataSizeBytes + TS_BYTES, "the poll is shorter than the record size"); + + std::vector out(pRec->pollStructSize * 32, 0); + RaftBusDeviceDecodeState state1, state2, state3; + TEST_ASSERT(pRec->pollResultDecodeFn(poll.data(), poll.size(), out.data(), out.size(), 32, state1) == 0, + "the decoder alone decodes nothing from the unpadded poll"); + uint32_t n = table.decodePollData(idx, poll.data(), poll.size(), out.data(), out.size(), 32, state2); + TEST_ASSERT(n == NUM_SAMPLES, "decodePollData gives every sample"); + + // Field values from the newest sample + std::vector newest(pRec->pollStructSize, 0); + uint32_t nNewest = table.decodeNewestPollRecord(idx, poll.data(), poll.size(), newest.data(), newest.size(), state3); + TEST_ASSERT(nNewest == NUM_SAMPLES, "decodeNewestPollRecord counts the samples"); + auto value = [&](const char* name, const uint8_t* pRecord) { + for (uint32_t i = 0; i < pRec->pollFieldCount; i++) + { + double v = 0; + if ((strcmp(pRec->pollFieldDescs[i].name, name) == 0) && + pRec->pollFieldDescs[i].getValue(pRecord, pRec->pollStructSize, 0, v)) + return v; + } + return -9999.0; + }; + const double gyroLsb = 1.0 / 14.285714; + TEST_ASSERT(fabs(value("gx", newest.data()) - 400 * gyroLsb) < 1e-3, "newest gx (sample 4)"); + TEST_ASSERT(fabs(value("ax", newest.data()) - (-8188.0 / 8192)) < 1e-6, "newest ax"); + TEST_ASSERT(fabs(value("az", newest.data()) - 1.0) < 1e-6, "az 1 g"); + TEST_ASSERT(fabs(value("temp", newest.data()) - 27.0) < 1e-6, "temperature"); + TEST_ASSERT(fabs(value("gx", out.data()) - 0.0) < 1e-6, "first sample gx"); +} + +// Reading values: every compiled record's fields, every element of array attributes, against the struct bytes +static void testFieldValues() +{ + printf("Running testFieldValues...\n"); + DeviceTypeRecords table; + std::mt19937 rng(99); + uint32_t numArrays = 0, numFields = 0; + bool allRight = true; + for (uint32_t idx = 0; idx < DeviceTypeRecords::getNumBaseRecords(); idx++) + { + const DeviceTypeRecord* pRec = table.getRecord(idx); + if (!pRec || !pRec->pollFieldDescs || (pRec->pollStructSize == 0)) + continue; + std::vector rec(pRec->pollStructSize); + for (auto& b : rec) + b = (uint8_t)rng(); + for (uint32_t f = 0; f < pRec->pollFieldCount; f++) + { + const AttrFieldDesc& desc = pRec->pollFieldDescs[f]; + const uint32_t elemBytes = desc.elemBytes(); + for (uint16_t e = 0; e < desc.elemCount(); e++) + { + // The reference: the element's bytes as the field's type, then the divisor and addend + const uint8_t* p = rec.data() + desc.offset + e * elemBytes; + double ref = 0; + switch (desc.type) + { + case AttrType::Float: { float v; memcpy(&v, p, 4); ref = v; break; } + case AttrType::Int32: { int32_t v; memcpy(&v, p, 4); ref = v; break; } + case AttrType::Uint32: { uint32_t v; memcpy(&v, p, 4); ref = v; break; } + case AttrType::Int16: { int16_t v; memcpy(&v, p, 2); ref = v; break; } + case AttrType::Uint16: { uint16_t v; memcpy(&v, p, 2); ref = v; break; } + case AttrType::Int8: ref = (int8_t)*p; break; + case AttrType::Uint8: ref = *p; break; + case AttrType::Bool: ref = *p ? 1 : 0; break; + } + if ((desc.divisor != 0.0f) && (desc.divisor != 1.0f)) + ref /= desc.divisor; + ref += desc.addend; + double v = 0; + bool ok = desc.getValue(rec.data(), rec.size(), e, v); + bool right = ok && ((v == ref) || (std::isnan(v) && std::isnan(ref))); + if (!right) + printf(" %s %s[%u]: %g, expected %g\n", pRec->deviceType, desc.name, e, v, ref); + allRight = allRight && right; + numFields++; + } + if (desc.elemCount() > 1) + numArrays++; + double v = 0; + TEST_ASSERT(!desc.getValue(rec.data(), rec.size(), desc.elemCount(), v), "an element past the end is refused"); + TEST_ASSERT(!desc.getValue(rec.data(), desc.offset, 0, v), "a field past the record's end is refused"); + } + } + printf(" %u values read, %u array attributes\n", numFields, numArrays); + TEST_ASSERT(allRight, "every value as its struct bytes"); + TEST_ASSERT(numArrays >= 5, "array attributes read element by element"); + + // Integral: an integer with nothing to scale + AttrFieldDesc d = {"x", 0, AttrType::Uint16, "", 1.0f, 0.0f, 1}; + TEST_ASSERT(d.isIntegral(), "uint16 integral"); + d.divisor = 8192.0f; + TEST_ASSERT(!d.isIntegral(), "scaled not integral"); + d = {"x", 0, AttrType::Float, "", 1.0f, 0.0f, 1}; + TEST_ASSERT(!d.isIntegral(), "float not integral"); +} + +int main() +{ + testShortRecordsDecodeAsPadded(); + testLsm6dsFifoPoll(); + testFieldValues(); + return testSummary("test_poll_data_decode"); +}