From 6e4ec81e992d324fe4c1d4322d253405ffade2a4 Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Fri, 18 Sep 2026 09:36:28 +0200 Subject: [PATCH 1/7] pack: bound msgpack2json() recursion depth to prevent stack overflow msgpack2json() in flb_pack.c recurses once per nesting level while converting a msgpack object into a JSON string, with no limit on how deep it will go for MSGPACK_OBJECT_ARRAY and MSGPACK_OBJECT_MAP values. A msgpack_object tree that is nested deeply enough (for example one built from a deeply nested JSON log line by a JSON-to-msgpack conversion elsewhere in the pipeline, which is not bound by the wire-level unpacker's own MSGPACK_EMBED_STACK_SIZE guard) will recurse past the available stack and crash the process with SIGSEGV. This was observed in production: the out_loki output plugin calls flb_msgpack_to_json_str() -> flb_msgpack_to_json() -> msgpack2json() from pack_record() while flushing records, and a deeply nested record reliably crashed the process with a stack overflow inside msgpack2json(), taking down the whole Fluent Bit instance and losing the in-flight flush batch. Add a FLB_PACK_JSON_MAX_DEPTH (512) guard: once msgpack2json() would recurse past this depth, stop descending and encode the remaining structure as a JSON null instead, so the conversion still completes and returns a valid (truncated) JSON string rather than crashing. 512 levels is far beyond any reasonably structured log record while keeping stack usage negligible. A non-empty map whose key/value pairs would land exactly one level past the limit needs special care: a JSON object key must always be a quoted string, so truncating an individual key to a bare null would produce invalid JSON such as {null:...}. Both MSGPACK_OBJECT_ARRAY and MSGPACK_OBJECT_MAP now check depth one level ahead of opening their bracket/brace, and render the whole container as null instead of partially opening it when their children would exceed the limit. The truncation warning is now logged at most once per top-level conversion (via a warned flag threaded through the recursion) instead of once per truncated branch, to avoid flooding the log when a single record has many separate over-depth branches. The msgpack_object *p pointer used while walking an array's elements is now declared with the function's other local variables instead of inside the loop != 0 block, matching this project's coding style of declaring variables at the top of the function; only the assignment stays inside the conditional. Verified locally with a small reproduction harness that builds an in-memory msgpack_object tree several hundred thousand levels deep and calls flb_msgpack_to_json_str() directly: - Before this patch: SIGSEGV (stack overflow), matching the exact production crash signature (out_loki's pack_record() call site). - After this patch: the call returns a valid, truncated JSON string and the process does not crash. Added two regression tests to tests/internal/pack.c: json_pack_deep_map_boundary builds a non-empty map exactly at the 512-level boundary and checks the exact output shape (length, and the null literal starting at the exact offset right after the opening brackets) instead of just checking that a null appears somewhere in the output; json_pack_deep_map_below_boundary builds the same map shape comfortably under the limit and checks its real content is still serialized. Both tests declare their local variables (including out_len and expected_len) at the top of the function per this project's coding style. Full tests/internal/pack.c suite (27 tests) passes, and the two new tests run clean under Valgrind (0 errors, 0 leaks, all blocks freed). Signed-off-by: Ugur Guenduez --- src/flb_pack.c | 82 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 7 deletions(-) diff --git a/src/flb_pack.c b/src/flb_pack.c index 581255f328b..66b498a1d82 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -51,6 +51,17 @@ #define try_to_write_str flb_utils_write_str +/* + * Maximum recursion depth allowed while converting a msgpack object into a + * JSON string (see msgpack2json() below). Msgpack arrays and maps can be + * nested arbitrarily deep, and msgpack2json() recurses once per nesting + * level. Without a bound, a deeply nested (malformed, corrupted or + * maliciously crafted) record can recurse deep enough to overflow the + * thread stack and crash the process. 512 levels is far beyond any + * reasonably structured log record while keeping stack usage negligible. + */ +#define FLB_PACK_JSON_MAX_DEPTH 512 + static int convert_nan_to_null = FLB_FALSE; static int flb_pack_set_null_as_nan(int b) { @@ -981,14 +992,42 @@ static inline int key_exists_in_map(msgpack_object key, msgpack_object map, int return FLB_FALSE; } +/* + * Log the maximum-nesting-depth truncation warning at most once per + * top-level msgpack2json() conversion, regardless of how many separate + * branches in the structure end up being truncated. + */ +static void msgpack2json_depth_warn(int *warned) +{ + if (warned != NULL && *warned == FLB_FALSE) { + flb_warn("[pack] msgpack to JSON conversion exceeded the maximum " + "nesting depth (%d), truncating remaining structure", + FLB_PACK_JSON_MAX_DEPTH); + *warned = FLB_TRUE; + } +} + static int msgpack2json(char *buf, int *off, size_t left, - const msgpack_object *o, int escape_unicode) + const msgpack_object *o, int escape_unicode, + int depth, int *warned) { int i; int dup; int ret = FLB_FALSE; int loop; int packed; + msgpack_object *p; + + /* + * Stop descending once the maximum nesting depth is reached and encode + * the remaining structure as a JSON null instead of recursing further. + * This keeps the conversion bounded and avoids a stack overflow on + * pathologically nested input, see FLB_PACK_JSON_MAX_DEPTH above. + */ + if (depth > FLB_PACK_JSON_MAX_DEPTH) { + msgpack2json_depth_warn(warned); + return try_to_write(buf, off, left, "null", 4); + } switch(o->type) { case MSGPACK_OBJECT_NIL: @@ -1078,17 +1117,30 @@ static int msgpack2json(char *buf, int *off, size_t left, case MSGPACK_OBJECT_ARRAY: loop = o->via.array.size; + if (loop != 0 && depth + 1 > FLB_PACK_JSON_MAX_DEPTH) { + /* + * The array is non-empty but its elements would exceed the + * maximum nesting depth. Render the whole array as null + * instead of opening it and only then truncating an element, + * keeping the output symmetric with the MSGPACK_OBJECT_MAP + * case below. + */ + msgpack2json_depth_warn(warned); + ret = try_to_write(buf, off, left, "null", 4); + break; + } + if (!try_to_write(buf, off, left, "[", 1)) { goto msg2json_end; } if (loop != 0) { - msgpack_object* p = o->via.array.ptr; - if (!msgpack2json(buf, off, left, p, escape_unicode)) { + p = o->via.array.ptr; + if (!msgpack2json(buf, off, left, p, escape_unicode, depth + 1, warned)) { goto msg2json_end; } for (i=1; ivia.map.size; + + if (loop != 0 && depth + 1 > FLB_PACK_JSON_MAX_DEPTH) { + /* + * The map is non-empty but its keys/values would exceed the + * maximum nesting depth. A JSON object key must always be a + * quoted string; truncating an individual key to a bare + * "null" (as the generic depth guard above would do) produces + * invalid JSON such as {null:...}. Render the whole map as + * null instead of opening it. + */ + msgpack2json_depth_warn(warned); + ret = try_to_write(buf, off, left, "null", 4); + break; + } + if (!try_to_write(buf, off, left, "{", 1)) { goto msg2json_end; } @@ -1124,9 +1191,9 @@ static int msgpack2json(char *buf, int *off, size_t left, } if ( - !msgpack2json(buf, off, left, &(p+i)->key, escape_unicode) || + !msgpack2json(buf, off, left, &(p+i)->key, escape_unicode, depth + 1, warned) || !try_to_write(buf, off, left, ":", 1) || - !msgpack2json(buf, off, left, &(p+i)->val, escape_unicode) ) { + !msgpack2json(buf, off, left, &(p+i)->val, escape_unicode, depth + 1, warned) ) { goto msg2json_end; } packed++; @@ -1158,12 +1225,13 @@ int flb_msgpack_to_json(char *json_str, size_t json_size, { int ret = -1; int off = 0; + int warned = FLB_FALSE; if (json_str == NULL || obj == NULL) { return -1; } - ret = msgpack2json(json_str, &off, json_size - 1, obj, escape_unicode); + ret = msgpack2json(json_str, &off, json_size - 1, obj, escape_unicode, 0, &warned); json_str[off] = '\0'; return ret ? off: ret; } From ee0138b3acd52aa3238f207b56af2256932816f3 Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Fri, 18 Sep 2026 09:36:30 +0200 Subject: [PATCH 2/7] tests: internal: pack: add msgpack2json() depth limit regression tests Signed-off-by: Ugur Guenduez --- tests/internal/pack.c | 156 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/tests/internal/pack.c b/tests/internal/pack.c index 60f79238286..a7f6fe1314b 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -1077,6 +1077,160 @@ void test_json_pack_bug5336() } /* Ensure empty arrays inside nested objects are handled */ +/* + * Must mirror FLB_PACK_JSON_MAX_DEPTH in src/flb_pack.c (not exposed via a + * public header since it is an internal implementation detail of + * msgpack2json()). + */ +#define TEST_PACK_MAX_DEPTH 512 + +/* + * Regression test: a non-empty msgpack map whose key/value pairs would be + * evaluated exactly one level past FLB_PACK_JSON_MAX_DEPTH must still be + * rendered as valid JSON. The whole map is expected to collapse to a JSON + * null literal instead of emitting a bare, unquoted null in place of a map + * key (which would produce invalid JSON such as {null:"v"}). + */ +void test_json_pack_deep_map_boundary() +{ + int i; + char *out; + char *p; + msgpack_object *chain; + msgpack_object_kv kv; + size_t out_len; + size_t expected_len; + + /* + * Build TEST_PACK_MAX_DEPTH nested single-element arrays with a + * non-empty map ({"k":"v"}) as the innermost element, constructed + * directly in memory. chain[0] is evaluated at depth 0, chain[i] at + * depth i, so the map at chain[TEST_PACK_MAX_DEPTH] is evaluated at + * depth == TEST_PACK_MAX_DEPTH: its own guard passes, but its key/value + * pair would be one level past the limit. + */ + chain = flb_malloc(sizeof(msgpack_object) * (TEST_PACK_MAX_DEPTH + 1)); + if (!TEST_CHECK(chain != NULL)) { + TEST_MSG("could not allocate test msgpack_object chain"); + return; + } + + for (i = 0; i < TEST_PACK_MAX_DEPTH; i++) { + chain[i].type = MSGPACK_OBJECT_ARRAY; + chain[i].via.array.size = 1; + chain[i].via.array.ptr = &chain[i + 1]; + } + + kv.key.type = MSGPACK_OBJECT_STR; + kv.key.via.str.size = 1; + kv.key.via.str.ptr = "k"; + kv.val.type = MSGPACK_OBJECT_STR; + kv.val.via.str.size = 1; + kv.val.via.str.ptr = "v"; + + chain[TEST_PACK_MAX_DEPTH].type = MSGPACK_OBJECT_MAP; + chain[TEST_PACK_MAX_DEPTH].via.map.size = 1; + chain[TEST_PACK_MAX_DEPTH].via.map.ptr = &kv; + + out = flb_msgpack_to_json_str(1024, &chain[0], FLB_FALSE); + flb_free(chain); + + if (!TEST_CHECK(out != NULL)) { + TEST_MSG("flb_msgpack_to_json_str returned NULL"); + return; + } + + /* a map key must never be truncated to an unquoted null */ + p = strstr(out, "null:"); + if (!TEST_CHECK(p == NULL)) { + TEST_MSG("map key was rendered as an unquoted null: %s", out); + } + + /* + * Exact shape check: TEST_PACK_MAX_DEPTH opening brackets, then the + * truncated map as a bare "null", then TEST_PACK_MAX_DEPTH closing + * brackets. Verify both the total length and that the null literal + * begins exactly at offset TEST_PACK_MAX_DEPTH, rather than accepting + * "null" anywhere in the output. + */ + { + out_len = strlen(out); + expected_len = (size_t) TEST_PACK_MAX_DEPTH * 2 + 4; + + if (!TEST_CHECK(out_len == expected_len)) { + TEST_MSG("unexpected output length: expected=%zu got=%zu out=%s", + expected_len, out_len, out); + } + + if (!TEST_CHECK(out_len > (size_t) TEST_PACK_MAX_DEPTH + 4 && + strncmp(out + TEST_PACK_MAX_DEPTH, "null", 4) == 0)) { + TEST_MSG("expected a null literal at offset %d: %s", + TEST_PACK_MAX_DEPTH, out); + } + } + + /* the original key/value content must not appear: it was truncated */ + p = strstr(out, "\"k\":\"v\""); + if (!TEST_CHECK(p == NULL)) { + TEST_MSG("map content should have been truncated: %s", out); + } + + flb_free(out); +} + +/* + * Companion check: the same map shape placed comfortably below the depth + * limit must still serialize its real content (i.e. the pre-check added for + * the boundary case above must not fire early for valid, shallower input). + */ +void test_json_pack_deep_map_below_boundary() +{ + int i; + int shallow_depth = TEST_PACK_MAX_DEPTH - 5; + char *out; + char *p; + msgpack_object *chain; + msgpack_object_kv kv; + + chain = flb_malloc(sizeof(msgpack_object) * (shallow_depth + 1)); + if (!TEST_CHECK(chain != NULL)) { + TEST_MSG("could not allocate test msgpack_object chain"); + return; + } + + for (i = 0; i < shallow_depth; i++) { + chain[i].type = MSGPACK_OBJECT_ARRAY; + chain[i].via.array.size = 1; + chain[i].via.array.ptr = &chain[i + 1]; + } + + kv.key.type = MSGPACK_OBJECT_STR; + kv.key.via.str.size = 1; + kv.key.via.str.ptr = "k"; + kv.val.type = MSGPACK_OBJECT_STR; + kv.val.via.str.size = 1; + kv.val.via.str.ptr = "v"; + + chain[shallow_depth].type = MSGPACK_OBJECT_MAP; + chain[shallow_depth].via.map.size = 1; + chain[shallow_depth].via.map.ptr = &kv; + + out = flb_msgpack_to_json_str(1024, &chain[0], FLB_FALSE); + flb_free(chain); + + if (!TEST_CHECK(out != NULL)) { + TEST_MSG("flb_msgpack_to_json_str returned NULL"); + return; + } + + p = strstr(out, "\"k\":\"v\""); + if (!TEST_CHECK(p != NULL)) { + TEST_MSG("map content below the depth limit should be preserved: %s", out); + } + + flb_free(out); +} + void test_json_pack_empty_array() { int ret; @@ -1305,6 +1459,8 @@ TEST_LIST = { { "json_pack_nan" , test_json_pack_nan}, { "json_pack_bug5336" , test_json_pack_bug5336}, { "json_pack_empty_array", test_json_pack_empty_array}, + { "json_pack_deep_map_boundary", test_json_pack_deep_map_boundary}, + { "json_pack_deep_map_below_boundary", test_json_pack_deep_map_below_boundary}, { "json_date_iso8601" , test_json_date_iso8601}, { "json_date_double" , test_json_date_double}, { "json_date_java_sql" , test_json_date_java_sql}, From 7baf3f70183ba71d2afceafad06630baa46c1ff1 Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Fri, 18 Sep 2026 14:25:42 +0200 Subject: [PATCH 3/7] pack: serialize nested objects without recursive stack growth Walk maps and arrays with a bounded heap-backed traversal stack. Preserve depth-limit truncation and duplicate-key behavior. Separate terminal conversion errors from output-buffer retries. Signed-off-by: Ugur Guenduez --- src/flb_pack.c | 272 +++++++++++++++++++++++++++++-------------------- 1 file changed, 163 insertions(+), 109 deletions(-) diff --git a/src/flb_pack.c b/src/flb_pack.c index 66b498a1d82..2f65ff770f6 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -52,13 +52,8 @@ #define try_to_write_str flb_utils_write_str /* - * Maximum recursion depth allowed while converting a msgpack object into a - * JSON string (see msgpack2json() below). Msgpack arrays and maps can be - * nested arbitrarily deep, and msgpack2json() recurses once per nesting - * level. Without a bound, a deeply nested (malformed, corrupted or - * maliciously crafted) record can recurse deep enough to overflow the - * thread stack and crash the process. 512 levels is far beyond any - * reasonably structured log record while keeping stack usage negligible. + * Keep traversal state on the heap: output coroutines have small stacks. + * Preserve the nesting limit and truncate deeper containers to JSON null. */ #define FLB_PACK_JSON_MAX_DEPTH 512 @@ -984,7 +979,9 @@ static inline int key_exists_in_map(msgpack_object key, msgpack_object map, int continue; } - if (memcmp(key.via.str.ptr, p.via.str.ptr, p.via.str.size) == 0) { + if (p.via.str.size == 0 || + (p.via.str.ptr != NULL && + memcmp(key.via.str.ptr, p.via.str.ptr, p.via.str.size) == 0)) { return FLB_TRUE; } } @@ -1007,27 +1004,12 @@ static void msgpack2json_depth_warn(int *warned) } } -static int msgpack2json(char *buf, int *off, size_t left, - const msgpack_object *o, int escape_unicode, - int depth, int *warned) +static int msgpack2json_scalar(char *buf, int *off, size_t left, + const msgpack_object *o, int escape_unicode) { int i; - int dup; int ret = FLB_FALSE; int loop; - int packed; - msgpack_object *p; - - /* - * Stop descending once the maximum nesting depth is reached and encode - * the remaining structure as a JSON null instead of recursing further. - * This keeps the conversion bounded and avoids a stack overflow on - * pathologically nested input, see FLB_PACK_JSON_MAX_DEPTH above. - */ - if (depth > FLB_PACK_JSON_MAX_DEPTH) { - msgpack2json_depth_warn(warned); - return try_to_write(buf, off, left, "null", 4); - } switch(o->type) { case MSGPACK_OBJECT_NIL: @@ -1073,6 +1055,10 @@ static int msgpack2json(char *buf, int *off, size_t left, break; case MSGPACK_OBJECT_STR: + if (o->via.str.size != 0 && o->via.str.ptr == NULL) { + errno = EINVAL; + return -1; + } if (try_to_write(buf, off, left, "\"", 1) && (o->via.str.size > 0 ? try_to_write_str(buf, off, left, o->via.str.ptr, o->via.str.size, escape_unicode) @@ -1083,6 +1069,10 @@ static int msgpack2json(char *buf, int *off, size_t left, break; case MSGPACK_OBJECT_BIN: + if (o->via.bin.size != 0 && o->via.bin.ptr == NULL) { + errno = EINVAL; + return -1; + } if (try_to_write(buf, off, left, "\"", 1) && (o->via.bin.size > 0 ? try_to_write_str(buf, off, left, o->via.bin.ptr, o->via.bin.size, escape_unicode) @@ -1093,6 +1083,11 @@ static int msgpack2json(char *buf, int *off, size_t left, break; case MSGPACK_OBJECT_EXT: + if (o->via.ext.size > INT_MAX || + (o->via.ext.size != 0 && o->via.ext.ptr == NULL)) { + errno = EINVAL; + return -1; + } if (!try_to_write(buf, off, left, "\"", 1)) { goto msg2json_end; } @@ -1114,100 +1109,149 @@ static int msgpack2json(char *buf, int *off, size_t left, ret = FLB_TRUE; break; - case MSGPACK_OBJECT_ARRAY: - loop = o->via.array.size; + default: + flb_warn("[%s] unknown msgpack type %i", __FUNCTION__, o->type); + errno = EINVAL; + return -1; + } - if (loop != 0 && depth + 1 > FLB_PACK_JSON_MAX_DEPTH) { - /* - * The array is non-empty but its elements would exceed the - * maximum nesting depth. Render the whole array as null - * instead of opening it and only then truncating an element, - * keeping the output symmetric with the MSGPACK_OBJECT_MAP - * case below. - */ - msgpack2json_depth_warn(warned); - ret = try_to_write(buf, off, left, "null", 4); - break; - } + msg2json_end: + return ret; +} - if (!try_to_write(buf, off, left, "[", 1)) { - goto msg2json_end; - } - if (loop != 0) { - p = o->via.array.ptr; - if (!msgpack2json(buf, off, left, p, escape_unicode, depth + 1, warned)) { - goto msg2json_end; +struct msgpack_json_frame { + const msgpack_object *object; + uint32_t index; + int packed; + int value; +}; + +/* 1: complete, 0: output buffer too small, -1: terminal conversion error. */ +static int msgpack2json(char *buf, int *off, size_t left, + const msgpack_object *o, int escape_unicode) +{ + struct msgpack_json_frame *frames; + struct msgpack_json_frame *frame; + const msgpack_object *container; + const msgpack_object *key; + uint32_t count; + int depth = 0; + int warned = FLB_FALSE; + int ret; + + if (o->type != MSGPACK_OBJECT_ARRAY && o->type != MSGPACK_OBJECT_MAP) { + return msgpack2json_scalar(buf, off, left, o, escape_unicode); + } + + frames = flb_calloc(FLB_PACK_JSON_MAX_DEPTH, sizeof(*frames)); + if (frames == NULL) { + errno = ENOMEM; + return -1; + } + + while (1) { + if (o->type == MSGPACK_OBJECT_ARRAY || o->type == MSGPACK_OBJECT_MAP) { + count = o->type == MSGPACK_OBJECT_ARRAY ? o->via.array.size : o->via.map.size; + if (count > INT_MAX || + (count != 0 && o->type == MSGPACK_OBJECT_ARRAY && o->via.array.ptr == NULL) || + (count != 0 && o->type == MSGPACK_OBJECT_MAP && o->via.map.ptr == NULL)) { + errno = EINVAL; + ret = -1; + break; } - for (i=1; itype == MSGPACK_OBJECT_ARRAY ? "[" : "{", 1); + if (ret == 0) { + break; + } + if (count != 0) { + frame = &frames[depth++]; + frame->object = o; + frame->index = 0; + frame->packed = 0; + frame->value = FLB_FALSE; + goto next_child; } + ret = try_to_write(buf, off, left, + o->type == MSGPACK_OBJECT_ARRAY ? "]" : "}", 1); } } - - ret = try_to_write(buf, off, left, "]", 1); - break; - - case MSGPACK_OBJECT_MAP: - loop = o->via.map.size; - - if (loop != 0 && depth + 1 > FLB_PACK_JSON_MAX_DEPTH) { - /* - * The map is non-empty but its keys/values would exceed the - * maximum nesting depth. A JSON object key must always be a - * quoted string; truncating an individual key to a bare - * "null" (as the generic depth guard above would do) produces - * invalid JSON such as {null:...}. Render the whole map as - * null instead of opening it. - */ - msgpack2json_depth_warn(warned); - ret = try_to_write(buf, off, left, "null", 4); - break; + else { + ret = msgpack2json_scalar(buf, off, left, o, escape_unicode); } - - if (!try_to_write(buf, off, left, "{", 1)) { - goto msg2json_end; + if (ret <= 0) { + break; } - if (loop != 0) { - msgpack_object k; - msgpack_object_kv *p = o->via.map.ptr; - packed = 0; - dup = FLB_FALSE; - - k = o->via.map.ptr[0].key; - for (i = 0; i < loop; i++) { - k = o->via.map.ptr[i].key; - dup = key_exists_in_map(k, *o, i + 1); - if (dup == FLB_TRUE) { - continue; + /* Resume the parent without consuming another C stack frame. */ + while (depth > 0) { + frame = &frames[depth - 1]; + if (frame->object->type == MSGPACK_OBJECT_MAP && !frame->value) { + ret = try_to_write(buf, off, left, ":", 1); + if (ret == 0) { + goto done; } - - if (packed > 0) { - if (!try_to_write(buf, off, left, ",", 1)) { - goto msg2json_end; + frame->value = FLB_TRUE; + o = &frame->object->via.map.ptr[frame->index].val; + goto visit; + } + frame->index++; + frame->packed++; + frame->value = FLB_FALSE; + +next_child: + container = frame->object; + count = container->type == MSGPACK_OBJECT_ARRAY ? + container->via.array.size : container->via.map.size; + if (container->type == MSGPACK_OBJECT_MAP) { + while (frame->index < count) { + key = &container->via.map.ptr[frame->index].key; + if (key->type != MSGPACK_OBJECT_STR && key->type != MSGPACK_OBJECT_BIN && + key->type != MSGPACK_OBJECT_EXT) { + errno = EINVAL; + ret = -1; + goto done; + } + if (key->type == MSGPACK_OBJECT_STR && key->via.str.size != 0 && + key->via.str.ptr == NULL) { + errno = EINVAL; + ret = -1; + goto done; + } + if (!key_exists_in_map(*key, *container, frame->index + 1)) { + break; } + frame->index++; } - - if ( - !msgpack2json(buf, off, left, &(p+i)->key, escape_unicode, depth + 1, warned) || - !try_to_write(buf, off, left, ":", 1) || - !msgpack2json(buf, off, left, &(p+i)->val, escape_unicode, depth + 1, warned) ) { - goto msg2json_end; + } + if (frame->index < count) { + if (frame->packed > 0 && !try_to_write(buf, off, left, ",", 1)) { + ret = 0; + goto done; } - packed++; + o = container->type == MSGPACK_OBJECT_ARRAY ? + &container->via.array.ptr[frame->index] : + &container->via.map.ptr[frame->index].key; + goto visit; } + ret = try_to_write(buf, off, left, + container->type == MSGPACK_OBJECT_ARRAY ? "]" : "}", 1); + if (ret == 0) { + goto done; + } + depth--; } - - ret = try_to_write(buf, off, left, "}", 1); break; - - default: - flb_warn("[%s] unknown msgpack type %i", __FUNCTION__, o->type); +visit: + continue; } - - msg2json_end: +done: + flb_free(frames); return ret; } @@ -1218,22 +1262,23 @@ static int msgpack2json(char *buf, int *off, size_t left, * @param json_str The buffer to fill JSON string. * @param json_size The size of json_str. * @param data The msgpack_unpacked data. - * @return success ? a number characters filled : negative value + * @return characters written on success, zero if the buffer is too small, + * or a negative value for a terminal conversion error. */ int flb_msgpack_to_json(char *json_str, size_t json_size, const msgpack_object *obj, int escape_unicode) { int ret = -1; int off = 0; - int warned = FLB_FALSE; - if (json_str == NULL || obj == NULL) { + if (json_str == NULL || obj == NULL || json_size == 0 || json_size > INT_MAX) { + errno = EINVAL; return -1; } - ret = msgpack2json(json_str, &off, json_size - 1, obj, escape_unicode, 0, &warned); + ret = msgpack2json(json_str, &off, json_size - 1, obj, escape_unicode); json_str[off] = '\0'; - return ret ? off: ret; + return ret > 0 ? off : ret; } flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int escape_unicode) @@ -1272,7 +1317,12 @@ flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int es root = &result.data; while (1) { ret = flb_msgpack_to_json(out_buf, out_size, root, escape_unicode); - if (ret <= 0) { + if (ret < 0) { + flb_sds_destroy(out_buf); + msgpack_unpacked_destroy(&result); + return NULL; + } + if (ret == 0) { realloc_size *= 2; tmp_buf = flb_sds_increase(out_buf, realloc_size); if (tmp_buf) { @@ -1697,7 +1747,11 @@ char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj, int escape while (1) { ret = flb_msgpack_to_json(buf, size, obj, escape_unicode); - if (ret <= 0) { + if (ret < 0) { + flb_free(buf); + return NULL; + } + if (ret == 0) { /* buffer is small. retry.*/ size *= 2; tmp = flb_realloc(buf, size); From 335d9ed8e5b355e6cdf0aa3387a643d2cb35b3cf Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Fri, 18 Sep 2026 14:26:35 +0200 Subject: [PATCH 4/7] tests: internal: cover JSON conversion on small coroutine stacks Signed-off-by: Ugur Guenduez --- tests/internal/pack.c | 178 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/tests/internal/pack.c b/tests/internal/pack.c index a7f6fe1314b..648c25cf14b 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -1441,7 +1442,184 @@ void test_json_pack_token_count_overflow() flb_pack_state_reset(&state); } +static cothread_t pack_coro_caller; +static msgpack_object *pack_coro_object; +static char *pack_coro_json; + +static void pack_small_stack_entry(void) +{ + pack_coro_json = flb_msgpack_to_json_str(1, pack_coro_object, FLB_FALSE); + co_switch(pack_coro_caller); +} + +/* Match the small coroutine stacks used by older Linux output workers. */ +static void check_json_pack_small_stack(int depth, int mixed) +{ + msgpack_object *chain; + msgpack_object_kv *pairs; + cothread_t callee; + char *expected; + int offset = 0; + int retained; + size_t stack_size; +#ifdef FLB_HAVE_VALGRIND + unsigned int stack_id; +#endif + int i; + + chain = flb_calloc(depth + 1, sizeof(*chain)); + TEST_ASSERT(chain != NULL); + pairs = flb_calloc(depth, sizeof(*pairs)); + TEST_ASSERT(pairs != NULL); + expected = flb_malloc(depth * 6 + 5); + TEST_ASSERT(expected != NULL); + retained = depth > 512 ? 512 : depth; + for (i = 0; i < depth; i++) { + if (mixed && i % 2 == 0) { + chain[i].type = MSGPACK_OBJECT_MAP; + chain[i].via.map.size = 1; + chain[i].via.map.ptr = &pairs[i]; + pairs[i].key.type = MSGPACK_OBJECT_STR; + pairs[i].key.via.str.ptr = "k"; + pairs[i].key.via.str.size = 1; + } + else { + chain[i].type = MSGPACK_OBJECT_ARRAY; + chain[i].via.array.size = 1; + chain[i].via.array.ptr = &chain[i + 1]; + } + if (i < retained) { + offset += sprintf(expected + offset, mixed && i % 2 == 0 ? "{\"k\":" : "["); + } + } + chain[depth].type = MSGPACK_OBJECT_NIL; + for (i = 0; i < depth; i++) { + if (mixed && i % 2 == 0) { + pairs[i].val = chain[i + 1]; + } + } + offset += sprintf(expected + offset, "null"); + for (i = retained - 1; i >= 0; i--) { + expected[offset++] = mixed && i % 2 == 0 ? '}' : ']'; + } + expected[offset] = '\0'; + pack_coro_object = chain; + pack_coro_json = NULL; + pack_coro_caller = co_active(); + callee = co_create(24576, pack_small_stack_entry, &stack_size); + TEST_ASSERT(callee != NULL); +#ifdef FLB_HAVE_VALGRIND + stack_id = VALGRIND_STACK_REGISTER(callee, ((char *) callee) + stack_size); +#endif + co_switch(callee); +#ifdef FLB_HAVE_VALGRIND + VALGRIND_STACK_DEREGISTER(stack_id); +#endif + co_delete(callee); + TEST_CHECK(pack_coro_json != NULL); + if (pack_coro_json != NULL) { + TEST_CHECK(strcmp(pack_coro_json, expected) == 0); + flb_free(pack_coro_json); + } + flb_free(chain); + flb_free(pairs); + flb_free(expected); +} + +void test_json_pack_small_stack(void) +{ + check_json_pack_small_stack(200, FLB_FALSE); + check_json_pack_small_stack(200, FLB_TRUE); + check_json_pack_small_stack(2000, FLB_TRUE); +} + +void test_json_pack_terminal_error(void) +{ + msgpack_object object; + msgpack_object_kv pairs[2]; + char *json; + char buffer[16]; + const char invalid_key[] = {0x81, 0x01, 0xc0}; + + memset(&object, 0, sizeof(object)); + object.type = MSGPACK_OBJECT_STR; + object.via.str.size = 1; + object.via.str.ptr = NULL; + TEST_CHECK(flb_msgpack_to_json(buffer, sizeof(buffer), &object, FLB_FALSE) == -1); + json = flb_msgpack_to_json_str(1, &object, FLB_FALSE); + TEST_CHECK(json == NULL); + flb_free(json); + + /* A later invalid key must not be dereferenced during duplicate lookup. */ + memset(pairs, 0, sizeof(pairs)); + pairs[0].key.type = MSGPACK_OBJECT_STR; + pairs[0].key.via.str.ptr = "k"; + pairs[0].key.via.str.size = 1; + pairs[0].val.type = MSGPACK_OBJECT_NIL; + pairs[1] = pairs[0]; + pairs[1].key.via.str.ptr = NULL; + object.type = MSGPACK_OBJECT_MAP; + object.via.map.ptr = pairs; + object.via.map.size = 2; + json = flb_msgpack_to_json_str(1, &object, FLB_FALSE); + TEST_CHECK(json == NULL); + flb_free(json); + + /* Both allocating wrappers must stop on terminal conversion errors. */ + json = flb_msgpack_raw_to_json_sds(invalid_key, sizeof(invalid_key), FLB_FALSE); + TEST_CHECK(json == NULL); + flb_sds_destroy(json); +} + +void test_json_pack_iterative_buffer_retry(void) +{ + const char *input = "{\"dup\":0,\"a\":[{},[],{\"b\":[true,null,\"x\\ny\"]}]," + "\"dup\":2,\"last\":{\"empty\":[]}}"; + const char *expected = "{\"a\":[{},[],{\"b\":[true,null,\"x\\ny\"]}]," + "\"dup\":2,\"last\":{\"empty\":[]}}"; + char *packed; + char *json; + char buffer[256]; + size_t packed_size; + size_t offset = 0; + size_t capacity; + int root_type; + int ret; + msgpack_unpacked result; + + ret = flb_pack_json(input, strlen(input), &packed, &packed_size, &root_type, NULL); + TEST_ASSERT(ret == 0); + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, packed, packed_size, &offset); + TEST_ASSERT(ret == MSGPACK_UNPACK_SUCCESS); + + /* Exercise early exits at every bracket, key, separator and scalar. */ + for (capacity = 0; capacity < strlen(expected) + 3; capacity++) { + memset(buffer, 'X', sizeof(buffer)); + ret = flb_msgpack_to_json(buffer, capacity, &result.data, FLB_FALSE); + TEST_CHECK(buffer[capacity] == 'X'); + if (capacity == 0) { + TEST_CHECK(ret == -1); + } + else if (ret > 0) { + TEST_CHECK(strcmp(buffer, expected) == 0); + } + else { + TEST_CHECK(ret == 0); + } + } + json = flb_msgpack_to_json_str(1, &result.data, FLB_FALSE); + TEST_ASSERT(json != NULL); + TEST_CHECK(strcmp(json, expected) == 0); + flb_free(json); + msgpack_unpacked_destroy(&result); + flb_free(packed); +} + TEST_LIST = { + { "json_pack_small_stack", test_json_pack_small_stack }, + { "json_pack_terminal_error", test_json_pack_terminal_error }, + { "json_pack_iterative_buffer_retry", test_json_pack_iterative_buffer_retry }, /* JSON maps iteration */ { "json_pack" , test_json_pack }, { "json_pack_ext_default_backend", test_json_pack_ext_default_backend }, From 2c0ef1258023e606e1247ff6e75f37f1f1ac9077 Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Fri, 18 Sep 2026 14:26:36 +0200 Subject: [PATCH 5/7] tests: integration: verify nested Loki records on small stacks Signed-off-by: Ugur Guenduez --- .../config/out_loki_long_unicode.yaml | 1 + .../out_loki/tests/test_out_loki_001.py | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/integration/scenarios/out_loki/config/out_loki_long_unicode.yaml b/tests/integration/scenarios/out_loki/config/out_loki_long_unicode.yaml index 4bd4d59c78c..d3d0ce7eb74 100644 --- a/tests/integration/scenarios/out_loki/config/out_loki_long_unicode.yaml +++ b/tests/integration/scenarios/out_loki/config/out_loki_long_unicode.yaml @@ -1,4 +1,5 @@ service: + coro_stack_size: 24576 flush: 1 grace: 1 log_level: info diff --git a/tests/integration/scenarios/out_loki/tests/test_out_loki_001.py b/tests/integration/scenarios/out_loki/tests/test_out_loki_001.py index 63a52f0082b..8e2f6fd5de5 100644 --- a/tests/integration/scenarios/out_loki/tests/test_out_loki_001.py +++ b/tests/integration/scenarios/out_loki/tests/test_out_loki_001.py @@ -131,3 +131,29 @@ def test_out_loki_preserves_long_unicode_json_strings(): assert len(records) == RECORD_COUNT assert all(set(record) == {"msg"} for record in records) assert sorted(record["msg"] for record in records) == sorted(expected_messages) + + +def test_out_loki_preserves_nested_records_on_small_stack(): + nested = {"leaf": "preserved"} + # Stay below the MessagePack decoder limit while exceeding the old + # recursive converter's capacity on a 24 KiB coroutine stack. + for _ in range(24): + nested = {"child": [nested]} + + with tempfile.TemporaryDirectory(prefix="flb-out-loki-depth-") as temp_directory: + input_path = Path(temp_directory) / "input.log" + with input_path.open("w", encoding="utf-8") as input_file: + for sequence in range(RECORD_COUNT): + json.dump({"seq": sequence, "msg": nested}, input_file) + input_file.write("\n") + service = Service(input_path) + try: + service.start() + service.wait_for_records() + finally: + service.stop() + records = collect_loki_records() + + assert len(records) == RECORD_COUNT + for record in records: + assert record == {"msg": nested} From 767e3de52d9515c086ae789fcce8a6742265dcfc Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Wed, 23 Sep 2026 21:27:01 +0000 Subject: [PATCH 6/7] pack: keep negative return for insufficient JSON buffer flb_msgpack_to_json() reports a buffer that is too small as a negative value, and fixed-buffer callers such as out_s3 only check ret < 0. Restore that public contract. Keep the distinction between buffer exhaustion and terminal conversion errors in a static helper used by flb_msgpack_raw_to_json_sds() and flb_msgpack_to_json_str(), so they still grow the buffer only when it is too small and stop on terminal errors. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WR8wEtu6GRYCv2pBsRZRxG Signed-off-by: Ugur Guenduez --- src/flb_pack.c | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/src/flb_pack.c b/src/flb_pack.c index 2f65ff770f6..b544454bd80 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -1255,18 +1255,13 @@ static int msgpack2json(char *buf, int *off, size_t left, return ret; } -/** - * convert msgpack to JSON string. - * This API is similar to snprintf. - * - * @param json_str The buffer to fill JSON string. - * @param json_size The size of json_str. - * @param data The msgpack_unpacked data. - * @return characters written on success, zero if the buffer is too small, - * or a negative value for a terminal conversion error. +/* + * Internal variant used by the growing-buffer wrappers: characters written + * on success, zero if the buffer is too small, or a negative value for a + * terminal conversion error. */ -int flb_msgpack_to_json(char *json_str, size_t json_size, - const msgpack_object *obj, int escape_unicode) +static int msgpack_to_json_buf(char *json_str, size_t json_size, + const msgpack_object *obj, int escape_unicode) { int ret = -1; int off = 0; @@ -1281,6 +1276,25 @@ int flb_msgpack_to_json(char *json_str, size_t json_size, return ret > 0 ? off : ret; } +/** + * convert msgpack to JSON string. + * This API is similar to snprintf. + * + * @param json_str The buffer to fill JSON string. + * @param json_size The size of json_str. + * @param data The msgpack_unpacked data. + * @return success ? a number characters filled : negative value + * (including when the buffer is too small) + */ +int flb_msgpack_to_json(char *json_str, size_t json_size, + const msgpack_object *obj, int escape_unicode) +{ + int ret; + + ret = msgpack_to_json_buf(json_str, json_size, obj, escape_unicode); + return ret > 0 ? ret : -1; +} + flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int escape_unicode) { int ret; @@ -1316,7 +1330,7 @@ flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int es root = &result.data; while (1) { - ret = flb_msgpack_to_json(out_buf, out_size, root, escape_unicode); + ret = msgpack_to_json_buf(out_buf, out_size, root, escape_unicode); if (ret < 0) { flb_sds_destroy(out_buf); msgpack_unpacked_destroy(&result); @@ -1746,7 +1760,7 @@ char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj, int escape } while (1) { - ret = flb_msgpack_to_json(buf, size, obj, escape_unicode); + ret = msgpack_to_json_buf(buf, size, obj, escape_unicode); if (ret < 0) { flb_free(buf); return NULL; From 64d2ed6600f3ddf17058f1a9db4a008bacb9b34d Mon Sep 17 00:00:00 2001 From: Ugur Guenduez Date: Wed, 23 Sep 2026 21:27:01 +0000 Subject: [PATCH 7/7] tests: internal: pack: expect negative return on buffer overflow Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WR8wEtu6GRYCv2pBsRZRxG Signed-off-by: Ugur Guenduez --- tests/internal/pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/internal/pack.c b/tests/internal/pack.c index 648c25cf14b..e5fa1b22135 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -1605,7 +1605,8 @@ void test_json_pack_iterative_buffer_retry(void) TEST_CHECK(strcmp(buffer, expected) == 0); } else { - TEST_CHECK(ret == 0); + /* insufficient space must be reported as a negative value */ + TEST_CHECK(ret < 0); } } json = flb_msgpack_to_json_str(1, &result.data, FLB_FALSE);