From 9952a63240f8789299752e5a992b8c30b3ef7a02 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 28 Mar 2026 17:22:28 -0400 Subject: [PATCH 1/4] Serialize fxdata without ArduinoJSON Eliminates size limit. Fixes #5458 --- wled00/json.cpp | 65 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/wled00/json.cpp b/wled00/json.cpp index db8e7dfcfd..a653755462 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -1160,21 +1160,6 @@ void serializePins(JsonObject root) } } -// deserializes mode data string into JsonArray -void serializeModeData(JsonArray fxdata) -{ - char lineBuffer[256]; - for (size_t i = 0; i < strip.getModeCount(); i++) { - strncpy_P(lineBuffer, strip.getModeData(i), sizeof(lineBuffer)/sizeof(char)-1); - lineBuffer[sizeof(lineBuffer)/sizeof(char)-1] = '\0'; // terminate string - if (lineBuffer[0] != 0) { - char* dataPtr = strchr(lineBuffer,'@'); - if (dataPtr) fxdata.add(dataPtr+1); - else fxdata.add(""); - } - } -} - // deserializes mode names string into JsonArray // also removes effect data extensions (@...) from deserialised names void serializeModeNames(JsonArray arr) @@ -1191,6 +1176,48 @@ void serializeModeNames(JsonArray arr) } } +// Generate a streamed JSON response for the mode data +// This uses sendChunked to send the reply in blocks based on how much fit in the outbound +// packet buffer, minimizing the required state (ie. just the next index to send). This +// allows us to send an arbitrarily large response without using any significant amount of +// memory (so no worries about buffer limits). +void respondModeData(AsyncWebServerRequest* request) { + size_t fx_index = 0; + request->sendChunked(FPSTR(CONTENT_TYPE_JSON), + [fx_index](uint8_t* data, size_t len, size_t) mutable { + size_t bytes_written = 0; + char lineBuffer[256]; + while (fx_index < strip.getModeCount() && (len > 5)) { + strncpy_P(lineBuffer, strip.getModeData(fx_index), sizeof(lineBuffer)/sizeof(char)-1); // Copy to stack buffer for strchr + if (lineBuffer[0] != 0) { + lineBuffer[sizeof(lineBuffer)/sizeof(char)-1] = '\0'; // terminate string + char* dataPtr = strchr(lineBuffer,'@'); // Find '@', if there is one + size_t mode_bytes; + if (dataPtr) { + mode_bytes = snprintf_P((char*) data, len, PSTR(",\"%s\""), dataPtr + 1); + if (mode_bytes > len) break; // didn't fit; break loop and try again next packet + } else { + strncpy_P((char*)data, PSTR(",\"\""), len); + mode_bytes = 3; + } + if (fx_index == 0) *data = '['; + data += mode_bytes; + len -= mode_bytes; + bytes_written += mode_bytes; + } + ++fx_index; + } + + if ((fx_index == strip.getModeCount()) && (len >= 1)) { + *data = ']'; + ++bytes_written; + ++fx_index; // we're really done + } + + return bytes_written; + }); +} + // Global buffer locking response helper class (to make sure lock is released when AsyncJsonResponse is destroyed) class LockedJsonResponse: public AsyncJsonResponse { bool _holding_lock; @@ -1218,7 +1245,7 @@ class LockedJsonResponse: public AsyncJsonResponse { void serveJson(AsyncWebServerRequest* request) { enum class json_target { - all, state, info, state_info, nodes, effects, palettes, fxdata, networks, config, pins + all, state, info, state_info, nodes, effects, palettes, networks, config, pins }; json_target subJson = json_target::all; @@ -1229,7 +1256,7 @@ void serveJson(AsyncWebServerRequest* request) else if (url.indexOf(F("nodes")) > 0) subJson = json_target::nodes; else if (url.indexOf(F("eff")) > 0) subJson = json_target::effects; else if (url.indexOf(F("palx")) > 0) subJson = json_target::palettes; - else if (url.indexOf(F("fxda")) > 0) subJson = json_target::fxdata; + else if (url.indexOf(F("fxda")) > 0) { respondModeData(request); return; } else if (url.indexOf(F("net")) > 0) subJson = json_target::networks; else if (url.indexOf(F("cfg")) > 0) subJson = json_target::config; else if (url.indexOf(F("pins")) > 0) subJson = json_target::pins; @@ -1254,7 +1281,7 @@ void serveJson(AsyncWebServerRequest* request) } // releaseJSONBufferLock() will be called when "response" is destroyed (from AsyncWebServer) // make sure you delete "response" if no "request->send(response);" is made - LockedJsonResponse *response = new LockedJsonResponse(pDoc, subJson==json_target::fxdata || subJson==json_target::effects); // will clear and convert JsonDocument into JsonArray if necessary + LockedJsonResponse *response = new LockedJsonResponse(pDoc, subJson==json_target::effects); // will clear and convert JsonDocument into JsonArray if necessary JsonVariant lDoc = response->getRoot(); @@ -1270,8 +1297,6 @@ void serveJson(AsyncWebServerRequest* request) serializePalettes(lDoc, request->hasParam(F("page")) ? request->getParam(F("page"))->value().toInt() : 0); break; case json_target::effects: serializeModeNames(lDoc); break; - case json_target::fxdata: - serializeModeData(lDoc); break; case json_target::networks: serializeNetworks(lDoc); break; case json_target::config: From 177e099d6583ce30b696da2efbe1b75b324b8291 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 29 Mar 2026 23:13:22 -0400 Subject: [PATCH 2/4] Fix off-by-one in fxdata serializer Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- wled00/json.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wled00/json.cpp b/wled00/json.cpp index a653755462..53b1723d78 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -1195,7 +1195,7 @@ void respondModeData(AsyncWebServerRequest* request) { size_t mode_bytes; if (dataPtr) { mode_bytes = snprintf_P((char*) data, len, PSTR(",\"%s\""), dataPtr + 1); - if (mode_bytes > len) break; // didn't fit; break loop and try again next packet + if (mode_bytes >= len) break; // didn't fit; break loop and try again next packet } else { strncpy_P((char*)data, PSTR(",\"\""), len); mode_bytes = 3; From 67be8352fbb64fb0045454e79281654b5768b7bd Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 29 Mar 2026 23:45:41 -0400 Subject: [PATCH 3/4] Remove declaration of removed function Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- wled00/fcn_declare.h | 1 - 1 file changed, 1 deletion(-) diff --git a/wled00/fcn_declare.h b/wled00/fcn_declare.h index 746d6142a6..ffb2c1202f 100644 --- a/wled00/fcn_declare.h +++ b/wled00/fcn_declare.h @@ -176,7 +176,6 @@ void serializeSegment(const JsonObject& root, const Segment& seg, byte id, bool void serializeState(JsonObject root, bool forPreset = false, bool includeBri = true, bool segmentBounds = true, bool selectedSegmentsOnly = false); void serializeInfo(JsonObject root); void serializeModeNames(JsonArray arr); -void serializeModeData(JsonArray fxdata); void serializePins(JsonObject root); void serveJson(AsyncWebServerRequest* request); #ifdef WLED_ENABLE_JSONLIVE From 64ebd4170ce09519e012b2c4e49299eeea438f82 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 30 Mar 2026 18:58:25 -0400 Subject: [PATCH 4/4] Factor out streaming JSON primitives Co-Authored-By: Claude Sonnet 4.6 --- wled00/json.cpp | 54 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/wled00/json.cpp b/wled00/json.cpp index 53b1723d78..a8b37d3757 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -1176,6 +1176,42 @@ void serializeModeNames(JsonArray arr) } } +// Writes a JSON-escaped string (with surrounding quotes) into dest[0..maxLen-1]. +// Returns bytes written, or 0 if the buffer was too small. +static size_t writeJSONString(uint8_t* dest, size_t maxLen, const char* src) { + size_t pos = 0; + + auto emit = [&](char c) -> bool { + if (pos >= maxLen) return false; + dest[pos++] = (uint8_t)c; + return true; + }; + + if (!emit('"')) return 0; + + for (const char* p = src; *p; ++p) { + char esc = ARDUINOJSON_NAMESPACE::EscapeSequence::escapeChar(*p); + if (esc) { + if (!emit('\\') || !emit(esc)) return 0; + } else { + if (!emit(*p)) return 0; + } + } + + if (!emit('"')) return 0; + return pos; +} + +// Writes ,"" into dest[0..maxLen-1] (no null terminator). +// Returns bytes written, or 0 if the buffer was too small. +static size_t writeJSONStringElement(uint8_t* dest, size_t maxLen, const char* src) { + if (maxLen == 0) return 0; + dest[0] = ','; + size_t n = writeJSONString(dest + 1, maxLen - 1, src); + if (n == 0) return 0; + return 1 + n; +} + // Generate a streamed JSON response for the mode data // This uses sendChunked to send the reply in blocks based on how much fit in the outbound // packet buffer, minimizing the required state (ie. just the next index to send). This @@ -1187,19 +1223,13 @@ void respondModeData(AsyncWebServerRequest* request) { [fx_index](uint8_t* data, size_t len, size_t) mutable { size_t bytes_written = 0; char lineBuffer[256]; - while (fx_index < strip.getModeCount() && (len > 5)) { - strncpy_P(lineBuffer, strip.getModeData(fx_index), sizeof(lineBuffer)/sizeof(char)-1); // Copy to stack buffer for strchr + while (fx_index < strip.getModeCount()) { + strncpy_P(lineBuffer, strip.getModeData(fx_index), sizeof(lineBuffer)-1); // Copy to stack buffer for strchr if (lineBuffer[0] != 0) { - lineBuffer[sizeof(lineBuffer)/sizeof(char)-1] = '\0'; // terminate string - char* dataPtr = strchr(lineBuffer,'@'); // Find '@', if there is one - size_t mode_bytes; - if (dataPtr) { - mode_bytes = snprintf_P((char*) data, len, PSTR(",\"%s\""), dataPtr + 1); - if (mode_bytes >= len) break; // didn't fit; break loop and try again next packet - } else { - strncpy_P((char*)data, PSTR(",\"\""), len); - mode_bytes = 3; - } + lineBuffer[sizeof(lineBuffer)-1] = '\0'; // terminate string (only needed if strncpy filled the buffer) + const char* dataPtr = strchr(lineBuffer,'@'); // Find '@', if there is one + size_t mode_bytes = writeJSONStringElement(data, len, dataPtr ? dataPtr + 1 : ""); + if (mode_bytes == 0) break; // didn't fit; break loop and try again next packet if (fx_index == 0) *data = '['; data += mode_bytes; len -= mode_bytes;