From cc09573dc2c674da2b4c65143b35dea75569a02e Mon Sep 17 00:00:00 2001 From: Rob Dobson Date: Wed, 11 Nov 2026 22:26:27 +0100 Subject: [PATCH 1/2] REST requests: decode any %XX escape, once unencodeHTTPChars replaced a fixed list of printable escapes one after another, so %27 became ^ (not '), %25 was decoded before the others ("%2526" gave "&"), and control characters (%0D, %0A, %03), %40 and UTF-8 were not decoded - MicroPython REPL input sent with encodeURIComponent() arrived with escapes left in it. It is now a single-pass decoder of any %XX (either hex case); a % that isn't an escape, and %00, are kept. test_rest_api_params (Linux unit tests) checks the decoding and that query values are still trimmed. Co-Authored-By: Claude Opus 5.5 --- .../RestAPIEndpointManager.cpp | 64 ++++++++-------- linux_unit_tests/Makefile | 3 +- linux_unit_tests/README.md | 1 + linux_unit_tests/test_rest_api_params.cpp | 74 +++++++++++++++++++ .../main/RestAPIEndpointManager_test.cpp | 27 +++++++ 5 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 linux_unit_tests/test_rest_api_params.cpp diff --git a/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp b/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp index b2c56de8..5a10827f 100644 --- a/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp +++ b/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp @@ -373,38 +373,38 @@ int RestAPIEndpointManager::getNumArgs(const char *argStr) /// @return Decoded string String RestAPIEndpointManager::unencodeHTTPChars(String &inStr) { - inStr.replace("%20", " "); - inStr.replace("%21", "!"); - inStr.replace("%22", "\""); - inStr.replace("%23", "#"); - inStr.replace("%24", "$"); - inStr.replace("%25", "%"); - inStr.replace("%26", "&"); - inStr.replace("%27", "^"); - inStr.replace("%28", "("); - inStr.replace("%29", ")"); - inStr.replace("%2A", "*"); - inStr.replace("%2B", "+"); - inStr.replace("%2C", ","); - inStr.replace("%2D", "-"); - inStr.replace("%2E", "."); - inStr.replace("%2F", "/"); - inStr.replace("%3A", ":"); - inStr.replace("%3B", ";"); - inStr.replace("%3C", "<"); - inStr.replace("%3D", "="); - inStr.replace("%3E", ">"); - inStr.replace("%3F", "?"); - inStr.replace("%5B", "["); - inStr.replace("%5C", "\\"); - inStr.replace("%5D", "]"); - inStr.replace("%5E", "^"); - inStr.replace("%5F", "_"); - inStr.replace("%60", "`"); - inStr.replace("%7B", "{"); - inStr.replace("%7C", "|"); - inStr.replace("%7D", "}"); - inStr.replace("%7E", "~"); + // Decode every %XX escape (either hex case) in a single pass so that decoded characters are never decoded + // again (e.g. "%2526" gives "%26", not "&"). Control characters such as %0D/%0A and UTF-8 bytes are decoded + // too. A "%" that isn't followed by two hex digits is kept as it is, and %00 is kept as it is because the + // result is a C string. "+" is not changed (encodeURIComponent() encodes a space as %20). + const char* pIn = inStr.c_str(); + if (!strchr(pIn, '%')) + return inStr; + String outStr; + outStr.reserve(inStr.length()); + auto hexVal = [](char c) -> int { + if ((c >= '0') && (c <= '9')) return c - '0'; + if ((c >= 'a') && (c <= 'f')) return c - 'a' + 10; + if ((c >= 'A') && (c <= 'F')) return c - 'A' + 10; + return -1; + }; + while (*pIn) + { + if (*pIn == '%') + { + int hi = hexVal(pIn[1]); + int lo = (hi >= 0) ? hexVal(pIn[2]) : -1; + if ((hi >= 0) && (lo >= 0) && ((hi | lo) != 0)) + { + outStr.concat((char)((hi << 4) | lo)); + pIn += 3; + continue; + } + } + outStr.concat(*pIn); + pIn++; + } + inStr = outStr; return inStr; } diff --git a/linux_unit_tests/Makefile b/linux_unit_tests/Makefile index 3d42f0a7..26417e08 100644 --- a/linux_unit_tests/Makefile +++ b/linux_unit_tests/Makefile @@ -118,6 +118,7 @@ SOURCES_test_thread_safe_queue = test_thread_safe_queue.cpp $(COMMON_SOURCES) SOURCES_test_simple_buffer = test_simple_buffer.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) SOURCES_test_file_system_paths = test_file_system_paths.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_mini_hdlc = test_mini_hdlc.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) +SOURCES_test_rest_api_params = test_rest_api_params.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_device_action_encoder = test_device_action_encoder.cpp $(CORE)/DeviceTypes/DeviceActionEncoder.cpp $(COMMON_SOURCES) SOURCES_test_mqtt_protocol = test_mqtt_protocol.cpp $(CORE)/MQTT/MQTTProtocol.cpp $(COMMON_SOURCES) @@ -125,7 +126,7 @@ SOURCES_test_espnow_config = test_espnow_config.cpp $(ESPNOW_DIR)/ESPNowConfig.c # Tests run by "make check" TESTS = test_raft_json test_raft_utils test_device_names test_device_settings test_device_type_settings test_device_data_subscribers test_device_type_records \ - test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_poll_data_decode test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder + test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_poll_data_decode test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder test_rest_api_params # Extra device type catalogues for test_device_type_records to sweep, e.g. # make check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tests/TestDevTypeRecs.json" diff --git a/linux_unit_tests/README.md b/linux_unit_tests/README.md index 8532e30d..119d005d 100644 --- a/linux_unit_tests/README.md +++ b/linux_unit_tests/README.md @@ -35,6 +35,7 @@ make -C linux_unit_tests check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tes | `test_file_upload_okto` | `FileUploadOKTOProtocol`: when it acks - the first block, each batch, the last block, at once for the first block out of sequence (a lost block or one sent again), and on its timer - and that its acks stay in step with the host's batches after a loss or a timer ack | | `test_led_strip_rmt_plan` | `LEDStripRMTPlan`: the RMT memory (or DMA buffer) an LED strip's channel asks for - the whole frame in RMT memory when it fits in the strip's share, DMA where the chip has it when it doesn't, then smaller sizes; shares between strips, a configured size, DMA modes and when a frame is refilled | | `test_mini_hdlc` | `MiniHDLC`: frames encoded and decoded again up to 70000 bytes (past 64 KB), the frame length limit, and a 200000-byte limit (as ProtocolRICSerial sets with PSRAM) kept rather than truncated to 16 bits | +| `test_rest_api_params` | `RestAPIEndpointManager`: `unencodeHTTPChars` decodes any `%XX` escape once (control characters, `'`, UTF-8, either hex case) and leaves non-escapes alone; query values are decoded and trimmed, path segments decoded after splitting | | `test_mqtt_protocol` | `MQTTProtocol`: CONNECT (with and without credentials), PUBLISH at QoS 0 and 1 with and without retain, long remaining lengths, SUBSCRIBE and PUBACK encoded as MQTT 3.1.1 specifies; broker packets decoded whether they arrive a byte at a time, several to a read or split across reads; malformed and overlong packets rejected; topic filter wildcards | | `test_thread_safe_queue` | `ThreadSafeQueue` (with its `SpiramAwareAllocator` deque): order, length limits, peek/pop/clear, many elements, and a producer and consumer on two threads | | `test_simple_buffer` | `SimpleBuffer` (MiniHDLC's frame buffers): grows geometrically and only when the memory is there (fails instead of crashing), `clear()` releases a buffer grown by a long frame; frames through MiniHDLC, including after a run longer than the limit | diff --git a/linux_unit_tests/test_rest_api_params.cpp b/linux_unit_tests/test_rest_api_params.cpp new file mode 100644 index 00000000..2d3cdccf --- /dev/null +++ b/linux_unit_tests/test_rest_api_params.cpp @@ -0,0 +1,74 @@ +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// RestAPIEndpointManager request parsing tests +// +// unencodeHTTPChars must decode any %XX escape exactly once: it previously replaced a fixed list of printable +// escapes one after another, so %27 became ^, "%2526" became "&" (decoded twice), and %0D, %0A, %40 and UTF-8 +// were not decoded at all - which broke MicroPython REPL input sent with encodeURIComponent(). Query values from +// getParamsAndNameValues are decoded the same way and are still whitespace-trimmed. +// +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include "testhdr.h" +#include "ArduinoWString.h" +#include "RaftJson.h" +#include "RestAPIEndpointManager.h" + +static void testUnencode() +{ + struct { const char* in; const char* out; } cases[] = { + { "hello%20world", "hello world" }, + { "c%2B%2B", "c++" }, + { "%27quoted%27", "'quoted'" }, + { "%2526", "%26" }, + { "a%0Db%0Ac%09d", "a\rb\nc\td" }, + { "%03", "\x03" }, + { "%40decorator", "@decorator" }, + { "caf%C3%A9", "caf\xC3\xA9" }, + { "%2f%2F", "//" }, + { "50%", "50%" }, + { "50%zz", "50%zz" }, + { "%4", "%4" }, + { "%", "%" }, + { "x%00y", "x%00y" }, + { "plain", "plain" }, + { "", "" }, + }; + for (auto& c : cases) + { + String s = c.in; + String r = RestAPIEndpointManager::unencodeHTTPChars(s); + String name = String("unencode \"") + c.in + "\""; + TEST_ASSERT(strcmp(r.c_str(), c.out) == 0, name.c_str()); + TEST_ASSERT(strcmp(s.c_str(), c.out) == 0, (name + " (in place)").c_str()); + } +} + +static void testQueryValues() +{ + std::vector params; + std::vector nameValues; + RestAPIEndpointManager::getParamsAndNameValues( + "/upy/repl/input?data=print('a%20%26%20b%3F')%0D&newline=0", params, nameValues); + TEST_ASSERT(params.size() == 3, "three path params"); + TEST_ASSERT(nameValues.size() == 2, "two name/values"); + if (nameValues.size() == 2) + { + // Trimming of values is unchanged: the trailing \r is removed + TEST_ASSERT(nameValues[0].value == "print('a & b?')", "data decoded once and trimmed"); + TEST_ASSERT(nameValues[1].value == "0", "newline value"); + } + params.clear(); + nameValues.clear(); + RestAPIEndpointManager::getParamsAndNameValues("/files/my%20dir%2Fa.txt", params, nameValues); + TEST_ASSERT(params.size() == 2 && params[1] == "my dir/a.txt", "path segment decoded after splitting"); +} + +int main() +{ + testUnencode(); + testQueryValues(); + return testSummary("test_rest_api_params"); +} diff --git a/unit_tests/main/RestAPIEndpointManager_test.cpp b/unit_tests/main/RestAPIEndpointManager_test.cpp index 05163e47..4779bf1a 100644 --- a/unit_tests/main/RestAPIEndpointManager_test.cpp +++ b/unit_tests/main/RestAPIEndpointManager_test.cpp @@ -210,6 +210,33 @@ TEST_CASE("getParamsAndNameValues - URL encoded", "[restapi]") TEST_ASSERT_EQUAL_STRING("c++", nameValues[1].value.c_str()); } +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Test unencodeHTTPChars decodes any %XX escape once only +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +TEST_CASE("unencodeHTTPChars - any escape, single pass", "[restapi]") +{ + struct { const char* in; const char* out; } cases[] = { + { "hello%20world", "hello world" }, + { "c%2B%2B", "c++" }, + { "%27quoted%27", "'quoted'" }, // was decoded as ^ + { "%2526", "%26" }, // decoded once only (was "&") + { "a%0Db%0Ac%09d", "a\rb\nc\td" }, // control characters + { "%40decorator", "@decorator" }, + { "caf%C3%A9", "caf\xC3\xA9" }, // UTF-8 + { "%2f%2F", "//" }, // either hex case + { "50%", "50%" }, // not an escape + { "50%zz", "50%zz" }, + { "x%00y", "x%00y" }, // NUL not decoded + }; + for (auto& c : cases) + { + String s = c.in; + String r = RestAPIEndpointManager::unencodeHTTPChars(s); + TEST_ASSERT_EQUAL_STRING(c.out, r.c_str()); + } +} + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Test getParamsAndNameValues with empty values ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 9541bc33da2bbbc6a920cb6ca2ef8356ae42bcf4 Mon Sep 17 00:00:00 2001 From: Rob Dobson Date: Wed, 11 Nov 2026 22:26:27 +0100 Subject: [PATCH 2/2] devman/setrole: correct the comment - save=1 keeps the role over a restart Co-Authored-By: Claude Opus 5.5 --- components/core/DeviceManager/DeviceManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/DeviceManager/DeviceManager.cpp b/components/core/DeviceManager/DeviceManager.cpp index 0008c3ff..8a4b61b1 100644 --- a/components/core/DeviceManager/DeviceManager.cpp +++ b/components/core/DeviceManager/DeviceManager.cpp @@ -3017,7 +3017,7 @@ RaftRetCode DeviceManager::apiDevManSetName(const String &reqStr, String &respSt /// @param respStr (out) response string /// @param jsonParams JSON object containing "deviceid" and "role" fields /// @return RaftRetCode -/// @note In-memory only; persistence is not handled by this endpoint +/// @note With save=1 the role is also saved in the device settings file (kept over a restart) RaftRetCode DeviceManager::apiDevManSetRole(const String &reqStr, String &respStr, const RaftJson& jsonParams) { String deviceIDStr = jsonParams.getString("deviceid", "");