diff --git a/components/core/DeviceManager/DeviceManager.cpp b/components/core/DeviceManager/DeviceManager.cpp index 0008c3ff..8a4b61b1 100644 --- a/components/core/DeviceManager/DeviceManager.cpp +++ b/components/core/DeviceManager/DeviceManager.cpp @@ -3017,7 +3017,7 @@ RaftRetCode DeviceManager::apiDevManSetName(const String &reqStr, String &respSt /// @param respStr (out) response string /// @param jsonParams JSON object containing "deviceid" and "role" fields /// @return RaftRetCode -/// @note In-memory only; persistence is not handled by this endpoint +/// @note With save=1 the role is also saved in the device settings file (kept over a restart) RaftRetCode DeviceManager::apiDevManSetRole(const String &reqStr, String &respStr, const RaftJson& jsonParams) { String deviceIDStr = jsonParams.getString("deviceid", ""); diff --git a/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp b/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp index b2c56de8..5a10827f 100644 --- a/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp +++ b/components/core/RestAPIEndpoints/RestAPIEndpointManager.cpp @@ -373,38 +373,38 @@ int RestAPIEndpointManager::getNumArgs(const char *argStr) /// @return Decoded string String RestAPIEndpointManager::unencodeHTTPChars(String &inStr) { - inStr.replace("%20", " "); - inStr.replace("%21", "!"); - inStr.replace("%22", "\""); - inStr.replace("%23", "#"); - inStr.replace("%24", "$"); - inStr.replace("%25", "%"); - inStr.replace("%26", "&"); - inStr.replace("%27", "^"); - inStr.replace("%28", "("); - inStr.replace("%29", ")"); - inStr.replace("%2A", "*"); - inStr.replace("%2B", "+"); - inStr.replace("%2C", ","); - inStr.replace("%2D", "-"); - inStr.replace("%2E", "."); - inStr.replace("%2F", "/"); - inStr.replace("%3A", ":"); - inStr.replace("%3B", ";"); - inStr.replace("%3C", "<"); - inStr.replace("%3D", "="); - inStr.replace("%3E", ">"); - inStr.replace("%3F", "?"); - inStr.replace("%5B", "["); - inStr.replace("%5C", "\\"); - inStr.replace("%5D", "]"); - inStr.replace("%5E", "^"); - inStr.replace("%5F", "_"); - inStr.replace("%60", "`"); - inStr.replace("%7B", "{"); - inStr.replace("%7C", "|"); - inStr.replace("%7D", "}"); - inStr.replace("%7E", "~"); + // Decode every %XX escape (either hex case) in a single pass so that decoded characters are never decoded + // again (e.g. "%2526" gives "%26", not "&"). Control characters such as %0D/%0A and UTF-8 bytes are decoded + // too. A "%" that isn't followed by two hex digits is kept as it is, and %00 is kept as it is because the + // result is a C string. "+" is not changed (encodeURIComponent() encodes a space as %20). + const char* pIn = inStr.c_str(); + if (!strchr(pIn, '%')) + return inStr; + String outStr; + outStr.reserve(inStr.length()); + auto hexVal = [](char c) -> int { + if ((c >= '0') && (c <= '9')) return c - '0'; + if ((c >= 'a') && (c <= 'f')) return c - 'a' + 10; + if ((c >= 'A') && (c <= 'F')) return c - 'A' + 10; + return -1; + }; + while (*pIn) + { + if (*pIn == '%') + { + int hi = hexVal(pIn[1]); + int lo = (hi >= 0) ? hexVal(pIn[2]) : -1; + if ((hi >= 0) && (lo >= 0) && ((hi | lo) != 0)) + { + outStr.concat((char)((hi << 4) | lo)); + pIn += 3; + continue; + } + } + outStr.concat(*pIn); + pIn++; + } + inStr = outStr; return inStr; } diff --git a/linux_unit_tests/Makefile b/linux_unit_tests/Makefile index 011a1661..ea701189 100644 --- a/linux_unit_tests/Makefile +++ b/linux_unit_tests/Makefile @@ -118,6 +118,7 @@ SOURCES_test_thread_safe_queue = test_thread_safe_queue.cpp $(COMMON_SOURCES) SOURCES_test_simple_buffer = test_simple_buffer.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) SOURCES_test_file_system_paths = test_file_system_paths.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_mini_hdlc = test_mini_hdlc.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMON_SOURCES) +SOURCES_test_rest_api_params = test_rest_api_params.cpp $(COMMS_SOURCES) $(COMMON_SOURCES) SOURCES_test_comms_framing_vectors = test_comms_framing_vectors.cpp $(CORE)/MiniHDLC/MiniHDLC.cpp $(COMMS)/ProtocolOverAscii/ProtocolOverAscii.cpp $(COMMON_SOURCES) SOURCES_test_device_action_encoder = test_device_action_encoder.cpp $(CORE)/DeviceTypes/DeviceActionEncoder.cpp $(COMMON_SOURCES) @@ -126,7 +127,7 @@ SOURCES_test_espnow_config = test_espnow_config.cpp $(ESPNOW_DIR)/ESPNowConfig.c # Tests run by "make check" TESTS = test_raft_json test_raft_utils test_device_names test_device_settings test_device_type_settings test_device_data_subscribers test_device_type_records \ - test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_poll_data_decode test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder test_comms_framing_vectors + test_device_type_table test_device_type_grammar test_device_type_policy test_device_type_file test_catalogue_index test_runtime_decoder test_poll_data_decode test_bus_ident test_bus_scan_schedule test_device_poll_timing test_device_poll_scheduler test_poll_plan test_protocol_exchange test_file_upload_okto test_file_upload_background test_mini_hdlc test_mqtt_protocol test_thread_safe_queue test_simple_buffer test_file_system_paths test_led_strip_rmt_plan test_device_action_encoder test_rest_api_params test_comms_framing_vectors # Extra device type catalogues for test_device_type_records to sweep, e.g. # make check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tests/TestDevTypeRecs.json" diff --git a/linux_unit_tests/README.md b/linux_unit_tests/README.md index 25d8931e..c7ba5f19 100644 --- a/linux_unit_tests/README.md +++ b/linux_unit_tests/README.md @@ -35,6 +35,7 @@ make -C linux_unit_tests check DEV_TYPE_CATALOGUES="../../RaftI2C/linux_unit_tes | `test_file_upload_okto` | `FileUploadOKTOProtocol`: when it acks - the first block, each batch, the last block, at once for the first block out of sequence (a lost block or one sent again), and on its timer - and that its acks stay in step with the host's batches after a loss or a timer ack | | `test_led_strip_rmt_plan` | `LEDStripRMTPlan`: the RMT memory (or DMA buffer) an LED strip's channel asks for - the whole frame in RMT memory when it fits in the strip's share, DMA where the chip has it when it doesn't, then smaller sizes; shares between strips, a configured size, DMA modes and when a frame is refilled | | `test_mini_hdlc` | `MiniHDLC`: frames encoded and decoded again up to 70000 bytes (past 64 KB), the frame length limit, and a 200000-byte limit (as ProtocolRICSerial sets with PSRAM) kept rather than truncated to 16 bits | +| `test_rest_api_params` | `RestAPIEndpointManager`: `unencodeHTTPChars` decodes any `%XX` escape once (control characters, `'`, UTF-8, either hex case) and leaves non-escapes alone; query values are decoded and trimmed, path segments decoded after splitting | | `test_comms_framing_vectors` | Makes the shared vectors `scripts/tests/hdlc_vectors.json` and `overascii_vectors.json` from `MiniHDLC` and `ProtocolOverAscii` and fails if the files differ (the RICRESTProtocol wiki examples are checked too). Client libraries (raftjs, raftpy) test their framing against these files. After an intended change to the framing, run `./test_comms_framing_vectors --write` and copy the files to the clients | | `test_mqtt_protocol` | `MQTTProtocol`: CONNECT (with and without credentials), PUBLISH at QoS 0 and 1 with and without retain, long remaining lengths, SUBSCRIBE and PUBACK encoded as MQTT 3.1.1 specifies; broker packets decoded whether they arrive a byte at a time, several to a read or split across reads; malformed and overlong packets rejected; topic filter wildcards | | `test_thread_safe_queue` | `ThreadSafeQueue` (with its `SpiramAwareAllocator` deque): order, length limits, peek/pop/clear, many elements, and a producer and consumer on two threads | diff --git a/linux_unit_tests/test_rest_api_params.cpp b/linux_unit_tests/test_rest_api_params.cpp new file mode 100644 index 00000000..2d3cdccf --- /dev/null +++ b/linux_unit_tests/test_rest_api_params.cpp @@ -0,0 +1,74 @@ +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// RestAPIEndpointManager request parsing tests +// +// unencodeHTTPChars must decode any %XX escape exactly once: it previously replaced a fixed list of printable +// escapes one after another, so %27 became ^, "%2526" became "&" (decoded twice), and %0D, %0A, %40 and UTF-8 +// were not decoded at all - which broke MicroPython REPL input sent with encodeURIComponent(). Query values from +// getParamsAndNameValues are decoded the same way and are still whitespace-trimmed. +// +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include "testhdr.h" +#include "ArduinoWString.h" +#include "RaftJson.h" +#include "RestAPIEndpointManager.h" + +static void testUnencode() +{ + struct { const char* in; const char* out; } cases[] = { + { "hello%20world", "hello world" }, + { "c%2B%2B", "c++" }, + { "%27quoted%27", "'quoted'" }, + { "%2526", "%26" }, + { "a%0Db%0Ac%09d", "a\rb\nc\td" }, + { "%03", "\x03" }, + { "%40decorator", "@decorator" }, + { "caf%C3%A9", "caf\xC3\xA9" }, + { "%2f%2F", "//" }, + { "50%", "50%" }, + { "50%zz", "50%zz" }, + { "%4", "%4" }, + { "%", "%" }, + { "x%00y", "x%00y" }, + { "plain", "plain" }, + { "", "" }, + }; + for (auto& c : cases) + { + String s = c.in; + String r = RestAPIEndpointManager::unencodeHTTPChars(s); + String name = String("unencode \"") + c.in + "\""; + TEST_ASSERT(strcmp(r.c_str(), c.out) == 0, name.c_str()); + TEST_ASSERT(strcmp(s.c_str(), c.out) == 0, (name + " (in place)").c_str()); + } +} + +static void testQueryValues() +{ + std::vector params; + std::vector nameValues; + RestAPIEndpointManager::getParamsAndNameValues( + "/upy/repl/input?data=print('a%20%26%20b%3F')%0D&newline=0", params, nameValues); + TEST_ASSERT(params.size() == 3, "three path params"); + TEST_ASSERT(nameValues.size() == 2, "two name/values"); + if (nameValues.size() == 2) + { + // Trimming of values is unchanged: the trailing \r is removed + TEST_ASSERT(nameValues[0].value == "print('a & b?')", "data decoded once and trimmed"); + TEST_ASSERT(nameValues[1].value == "0", "newline value"); + } + params.clear(); + nameValues.clear(); + RestAPIEndpointManager::getParamsAndNameValues("/files/my%20dir%2Fa.txt", params, nameValues); + TEST_ASSERT(params.size() == 2 && params[1] == "my dir/a.txt", "path segment decoded after splitting"); +} + +int main() +{ + testUnencode(); + testQueryValues(); + return testSummary("test_rest_api_params"); +} diff --git a/unit_tests/main/RestAPIEndpointManager_test.cpp b/unit_tests/main/RestAPIEndpointManager_test.cpp index 05163e47..4779bf1a 100644 --- a/unit_tests/main/RestAPIEndpointManager_test.cpp +++ b/unit_tests/main/RestAPIEndpointManager_test.cpp @@ -210,6 +210,33 @@ TEST_CASE("getParamsAndNameValues - URL encoded", "[restapi]") TEST_ASSERT_EQUAL_STRING("c++", nameValues[1].value.c_str()); } +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Test unencodeHTTPChars decodes any %XX escape once only +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +TEST_CASE("unencodeHTTPChars - any escape, single pass", "[restapi]") +{ + struct { const char* in; const char* out; } cases[] = { + { "hello%20world", "hello world" }, + { "c%2B%2B", "c++" }, + { "%27quoted%27", "'quoted'" }, // was decoded as ^ + { "%2526", "%26" }, // decoded once only (was "&") + { "a%0Db%0Ac%09d", "a\rb\nc\td" }, // control characters + { "%40decorator", "@decorator" }, + { "caf%C3%A9", "caf\xC3\xA9" }, // UTF-8 + { "%2f%2F", "//" }, // either hex case + { "50%", "50%" }, // not an escape + { "50%zz", "50%zz" }, + { "x%00y", "x%00y" }, // NUL not decoded + }; + for (auto& c : cases) + { + String s = c.in; + String r = RestAPIEndpointManager::unencodeHTTPChars(s); + TEST_ASSERT_EQUAL_STRING(c.out, r.c_str()); + } +} + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Test getParamsAndNameValues with empty values /////////////////////////////////////////////////////////////////////////////////////////////////////////////////